Merge pull request #59 from vulcanize/schema_updates

reflect changes in integrated service

README.md

@@ -23,14 +23,13 @@ Available RPC methods are:
   * `statediff_streamCodeAndCodeHash()`
   * `statediff_stateDiffAt()`
   * `statediff_writeStateDiffAt()`
+  * `statediff_writeStateDiffsInRange()`
 
-### `write`
+e.g. `curl -X POST -H 'Content-Type: application/json' --data '{"jsonrpc":"2.0","method":"statediff_writeStateDiffsInRange","params":['"$BEGIN"', '"$END"', {"intermediateStateNodes":true,"intermediateStorageNodes":true,"includeBlock":true,"includeReceipts":true,"includeTD":true,"includeCode":true}],"id":1}' "$HOST":"$PORT"`
 
-To write state diffs directly to a database:
-
-`eth-statediff-service write --config=<config path>`
-
-This depends on the `database` settings being properly configured.
+The process can be configured locally with sets of ranges to process as a "prerun" to processing directed by the server endpoints.
+This is done by turning "prerun" on in the config (`statediff.prerun = true`) and defining ranged and params in the
+`prerun` section of the config as shown below.
 
 ## Configuration
 
@@ -42,27 +41,56 @@ An example config file:
     ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient"
 
 [server]
-    ipcPath = "~/.vulcanize/vulcanize.ipc"
+    ipcPath = ".ipc"
     httpPath = "127.0.0.1:8545"
 
-[write]
-    ranges = [[1, 2], [3, 4]]
-[write.params]
-    IntermediateStateNodes   = true
-    IntermediateStorageNodes = false
-    IncludeBlock             = true
-    IncludeReceipts          = true
-    IncludeTD                = true
-    IncludeCode              = false
-
 [statediff]
-    workers = 4
+    prerun = true
+    serviceWorkers = 1
+    workerQueueSize = 1024
+    trieWorkers = 4
+
+[prerun]
+    only = false
+    ranges = [
+        [0, 1000]
+    ]
+    [prerun.params]
+        intermediateStateNodes = true
+        intermediateStorageNodes = true
+        includeBlock = true
+        includeReceipts = true
+        includeTD = true
+        includeCode = true
+        watchedAddresses = []
+        watchedStorageKeys = []
 
 [log]
-    file = "~/.vulcanize/statediff.log"
+    file = ""
     level = "info"
 
 [eth]
     chainID = 1
 
+[database]
+    name     = "vulcanize_test"
+    hostname = "localhost"
+    port     = 5432
+    user     = "vulcanize"
+    password = "..."
+    type = "postgres"
+    driver = "sqlx"
+    dumpDestination = ""
+    filePath = ""
+
+[cache]
+    database = 1024
+    trie = 1024
+
+[prom]
+    dbStats = false
+    metrics = true
+    http = true
+    httpAddr = "localhost"
+    httpPort = "8889"
 ```

cmd/env.go

@@ -62,6 +62,7 @@ const (
 	DATABASE_TYPE        = "DATABASE_TYPE"
 	DATABASE_DRIVER_TYPE = "DATABASE_DRIVER_TYPE"
 	DATABASE_DUMP_DST    = "DATABASE_DUMP_DST"
+	DATABASE_FILE_PATH   = "DATABASE_FILE_PATH"
 
 	DATABASE_MAX_IDLE_CONNECTIONS = "DATABASE_MAX_IDLE_CONNECTIONS"
 	DATABASE_MAX_OPEN_CONNECTIONS = "DATABASE_MAX_OPEN_CONNECTIONS"
@@ -98,6 +99,7 @@ func init() {
 	viper.BindEnv("database.type", DATABASE_TYPE)
 	viper.BindEnv("database.driver", DATABASE_DRIVER_TYPE)
 	viper.BindEnv("database.dumpDestination", DATABASE_DUMP_DST)
+	viper.BindEnv("database.filePath", DATABASE_FILE_PATH)
 
 	viper.BindEnv("cache.database", DB_CACHE_SIZE_MB)
 	viper.BindEnv("cache.trie", TRIE_CACHE_SIZE_MB)

cmd/root.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/statediff/indexer/database/dump"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
 	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
 	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
 	"github.com/ethereum/go-ethereum/statediff/indexer/node"
@@ -137,6 +138,7 @@ func init() {
 	rootCmd.PersistentFlags().String("database-type", "postgres", "database type (currently supported: postgres, dump)")
 	rootCmd.PersistentFlags().String("database-driver", "sqlx", "database driver type (currently supported: sqlx, pgx)")
 	rootCmd.PersistentFlags().String("database-dump-dst", "stdout", "dump destination (for database-type=dump; options: stdout, stderr, discard)")
+	rootCmd.PersistentFlags().String("database-file-path", "", "full file path (for database-type=file)")
 
 	rootCmd.PersistentFlags().String("eth-node-id", "", "eth node id")
 	rootCmd.PersistentFlags().String("eth-client-name", "eth-statediff-service", "eth client name")
@@ -280,7 +282,15 @@ func getConfig(nodeInfo node.Info) (interfaces.Config, error) {
 	logWithCommand.Infof("configuring service for database type: %s", dbType)
 	var indexerConfig interfaces.Config
 	switch dbType {
+	case shared.FILE:
+		logWithCommand.Info("starting in sql file writing mode")
+		filePathStr := viper.GetString("database.filePath")
+		if filePathStr == "" {
+			logWithCommand.Fatal("when operating in sql file writing mode a file path must be provided")
+		}
+		indexerConfig = file.Config{FilePath: filePathStr}
 	case shared.DUMP:
+		logWithCommand.Info("starting in data dump mode")
 		dumpDstStr := viper.GetString("database.dumpDestination")
 		dumpDst, err := dump.ResolveDumpType(dumpDstStr)
 		if err != nil {
@@ -297,6 +307,7 @@ func getConfig(nodeInfo node.Info) (interfaces.Config, error) {
 			return nil, fmt.Errorf("unrecognized dump destination: %s", dumpDst)
 		}
 	case shared.POSTGRES:
+		logWithCommand.Info("starting in postgres mode")
 		driverTypeStr := viper.GetString("database.driver")
 		driverType, err := postgres.ResolveDriverType(driverTypeStr)
 		if err != nil {

environments/config.toml

@@ -41,6 +41,7 @@
     type = "postgres"
     driver = "sqlx"
     dumpDestination = ""
+    filePath = ""
 
 [cache]
     database = 1024

environments/example.toml

@@ -43,6 +43,7 @@
     type = "postgres"
     driver = "sqlx"
     dumpDestination = ""
+    filePath = ""
 
 [cache]
     database = 1024

go.mod

@@ -12,4 +12,4 @@ require (
 	github.com/vulcanize/go-eth-state-node-iterator v0.0.1-alpha.0.20211014064906-d23d01ed8191
 )
 
-replace github.com/ethereum/go-ethereum v1.10.9 => github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27b
+replace github.com/ethereum/go-ethereum v1.10.9 => github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27c

go.sum

@@ -653,8 +653,8 @@ github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPU
 github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
 github.com/vulcanize/go-eth-state-node-iterator v0.0.1-alpha.0.20211014064906-d23d01ed8191 h1:+xSwb0xOLCAhejoai2BQZPTJW5Z+tU9xkTaOBwGnPoU=
 github.com/vulcanize/go-eth-state-node-iterator v0.0.1-alpha.0.20211014064906-d23d01ed8191/go.mod h1:FBMQp69PTnRg3kLx9qeWGfKYfVq4fpfbr69X8lEvvfs=
-github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27b h1:TM6qCbWQCwBFTm2ZhZDKRtyjGPRU5ccSbDRp9tWHulY=
-github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27b/go.mod h1:XO9WLkNXfwoJN05BZj0//xgOWHJyUrUPdnudbQfKlUo=
+github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27c h1:EnKLxZU0y1he4s/kmfbnLH5kr+QOaCUOf+yrBVfsq4Y=
+github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27c/go.mod h1:XO9WLkNXfwoJN05BZj0//xgOWHJyUrUPdnudbQfKlUo=
 github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo=
 github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
 github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=