From a4062f19c0639bd672a807eb24102120bd30d6b2 Mon Sep 17 00:00:00 2001 From: Rob Mulholand Date: Tue, 19 Nov 2019 16:20:49 -0600 Subject: [PATCH 1/2] Make config preparation synchronous - Wait until config preparation has finished before moving forward with command execution --- cmd/compose.go | 44 ++++++++++++++++++++-------------------- cmd/composeAndExecute.go | 42 ++++++++++++++++++-------------------- cmd/execute.go | 33 +++++++++++++++--------------- go.mod | 7 ++++--- go.sum | 20 ++++++++++++++---- 5 files changed, 78 insertions(+), 68 deletions(-) diff --git a/cmd/compose.go b/cmd/compose.go index c3620486d..015f70a46 100644 --- a/cmd/compose.go +++ b/cmd/compose.go @@ -110,27 +110,26 @@ Specify config location when executing the command: func compose() { // Build plugin generator config - prepConfig() + configErr := prepConfig() + if configErr != nil { + LogWithCommand.Fatalf("failed to prepare config: %s", configErr.Error()) + } // Generate code to build the plugin according to the config file LogWithCommand.Info("generating plugin") - generator, err := p2.NewGenerator(genConfig, databaseConfig) - if err != nil { - LogWithCommand.Debug("initializing plugin generator failed") - LogWithCommand.Fatal(err) + generator, constructorErr := p2.NewGenerator(genConfig, databaseConfig) + if constructorErr != nil { + LogWithCommand.Fatalf("initializing plugin generator failed: %s", constructorErr.Error()) } - err = generator.GenerateExporterPlugin() - if err != nil { - LogWithCommand.Debug("generating plugin failed") - LogWithCommand.Fatal(err) + generateErr := generator.GenerateExporterPlugin() + if generateErr != nil { + LogWithCommand.Fatalf("generating plugin failed: %s", generateErr.Error()) } // TODO: Embed versioning info in the .so files so we know which version of vulcanizedb to run them with - _, pluginPath, err := genConfig.GetPluginPaths() - if err != nil { - LogWithCommand.Debug("getting plugin path failed") - LogWithCommand.Fatal(err) + _, pluginPath, pathErr := genConfig.GetPluginPaths() + if pathErr != nil { + LogWithCommand.Fatalf("getting plugin path failed: %s", pathErr.Error()) } - fmt.Printf("Composed plugin %s", pluginPath) LogWithCommand.Info("plugin .so file output to ", pluginPath) } @@ -138,7 +137,7 @@ func init() { rootCmd.AddCommand(composeCmd) } -func prepConfig() { +func prepConfig() error { LogWithCommand.Info("configuring plugin") names := viper.GetStringSlice("exporter.transformerNames") transformers := make(map[string]config.Transformer) @@ -146,31 +145,31 @@ func prepConfig() { transformer := viper.GetStringMapString("exporter." + name) p, pOK := transformer["path"] if !pOK || p == "" { - LogWithCommand.Fatal(name, " transformer config is missing `path` value") + return fmt.Errorf("transformer config is missing `path` value: %s", name) } r, rOK := transformer["repository"] if !rOK || r == "" { - LogWithCommand.Fatal(name, " transformer config is missing `repository` value") + return fmt.Errorf("transformer config is missing `repository` value: %s", name) } m, mOK := transformer["migrations"] if !mOK || m == "" { - LogWithCommand.Fatal(name, " transformer config is missing `migrations` value") + return fmt.Errorf("transformer config is missing `migrations` value: %s", name) } mr, mrOK := transformer["rank"] if !mrOK || mr == "" { - LogWithCommand.Fatal(name, " transformer config is missing `rank` value") + return fmt.Errorf("transformer config is missing `rank` value: %s", name) } rank, err := strconv.ParseUint(mr, 10, 64) if err != nil { - LogWithCommand.Fatal(name, " migration `rank` can't be converted to an unsigned integer") + return fmt.Errorf("migration `rank` can't be converted to an unsigned integer: %s", name) } t, tOK := transformer["type"] if !tOK { - LogWithCommand.Fatal(name, " transformer config is missing `type` value") + return fmt.Errorf("transformer config is missing `type` value: %s", name) } transformerType := config.GetTransformerType(t) if transformerType == config.UnknownTransformerType { - LogWithCommand.Fatal(errors.New(`unknown transformer type in exporter config accepted types are "eth_event", "eth_storage"`)) + return errors.New(`unknown transformer type in exporter config accepted types are "eth_event", "eth_storage"`) } transformers[name] = config.Transformer{ @@ -189,4 +188,5 @@ func prepConfig() { Save: viper.GetBool("exporter.save"), Home: viper.GetString("exporter.home"), } + return nil } diff --git a/cmd/composeAndExecute.go b/cmd/composeAndExecute.go index 149dd05a1..573b7b153 100644 --- a/cmd/composeAndExecute.go +++ b/cmd/composeAndExecute.go @@ -21,7 +21,6 @@ import ( "github.com/makerdao/vulcanizedb/libraries/shared/fetcher" "github.com/makerdao/vulcanizedb/libraries/shared/streamer" "github.com/makerdao/vulcanizedb/pkg/fs" - "os" "plugin" syn "sync" "time" @@ -116,48 +115,47 @@ Specify config location when executing the command: func composeAndExecute() { // Build plugin generator config - prepConfig() + configErr := prepConfig() + if configErr != nil { + LogWithCommand.Fatalf("failed to prepare config: %s", configErr.Error()) + } // Generate code to build the plugin according to the config file LogWithCommand.Info("generating plugin") - generator, err := p2.NewGenerator(genConfig, databaseConfig) - if err != nil { - LogWithCommand.Fatal(err) + generator, constructorErr := p2.NewGenerator(genConfig, databaseConfig) + if constructorErr != nil { + LogWithCommand.Fatalf("failed to initialize generator: %s", constructorErr.Error()) } - err = generator.GenerateExporterPlugin() - if err != nil { - LogWithCommand.Debug("generating plugin failed") - LogWithCommand.Fatal(err) + generateErr := generator.GenerateExporterPlugin() + if generateErr != nil { + LogWithCommand.Fatalf("generating plugin failed: %s", generateErr.Error()) } // Get the plugin path and load the plugin - _, pluginPath, err := genConfig.GetPluginPaths() - if err != nil { - LogWithCommand.Fatal(err) + _, pluginPath, pathErr := genConfig.GetPluginPaths() + if pathErr != nil { + LogWithCommand.Fatalf("failed to get plugin paths: %s", pathErr.Error()) } if !genConfig.Save { defer helpers.ClearFiles(pluginPath) } LogWithCommand.Info("linking plugin ", pluginPath) - plug, err := plugin.Open(pluginPath) - if err != nil { - LogWithCommand.Debug("linking plugin failed") - LogWithCommand.Fatal(err) + plug, openErr := plugin.Open(pluginPath) + if openErr != nil { + LogWithCommand.Fatalf("linking plugin failed: %s", openErr.Error()) } // Load the `Exporter` symbol from the plugin LogWithCommand.Info("loading transformers from plugin") - symExporter, err := plug.Lookup("Exporter") - if err != nil { - LogWithCommand.Debug("loading Exporter symbol failed") - LogWithCommand.Fatal(err) + symExporter, lookupErr := plug.Lookup("Exporter") + if lookupErr != nil { + LogWithCommand.Fatalf("loading Exporter symbol failed: %s", lookupErr.Error()) } // Assert that the symbol is of type Exporter exporter, ok := symExporter.(Exporter) if !ok { - LogWithCommand.Debug("plugged-in symbol not of type Exporter") - os.Exit(1) + LogWithCommand.Fatal("plugged-in symbol not of type Exporter") } // Use the Exporters export method to load the EventTransformerInitializer, StorageTransformerInitializer, and ContractTransformerInitializer sets diff --git a/cmd/execute.go b/cmd/execute.go index f06b96ded..5230cec89 100644 --- a/cmd/execute.go +++ b/cmd/execute.go @@ -17,7 +17,6 @@ package cmd import ( - "fmt" "github.com/ethereum/go-ethereum/statediff" "github.com/makerdao/vulcanizedb/libraries/shared/fetcher" "github.com/makerdao/vulcanizedb/libraries/shared/streamer" @@ -74,28 +73,28 @@ Specify config location when executing the command: func execute() { // Build plugin generator config - prepConfig() + configErr := prepConfig() + if configErr != nil { + LogWithCommand.Fatalf("failed to prepare config: %s", configErr.Error()) + } // Get the plugin path and load the plugin - _, pluginPath, err := genConfig.GetPluginPaths() - if err != nil { - LogWithCommand.Fatal(err) + _, pluginPath, pathErr := genConfig.GetPluginPaths() + if pathErr != nil { + LogWithCommand.Fatalf("failed to get plugin paths: %s", pathErr.Error()) } - fmt.Printf("Executing plugin %s", pluginPath) LogWithCommand.Info("linking plugin ", pluginPath) - plug, err := plugin.Open(pluginPath) - if err != nil { - LogWithCommand.Warn("linking plugin failed") - LogWithCommand.Fatal(err) + plug, openErr := plugin.Open(pluginPath) + if openErr != nil { + LogWithCommand.Fatalf("linking plugin failed: %s", openErr.Error()) } // Load the `Exporter` symbol from the plugin LogWithCommand.Info("loading transformers from plugin") - symExporter, err := plug.Lookup("Exporter") - if err != nil { - LogWithCommand.Warn("loading Exporter symbol failed") - LogWithCommand.Fatal(err) + symExporter, lookupErr := plug.Lookup("Exporter") + if lookupErr != nil { + LogWithCommand.Fatalf("loading Exporter symbol failed: %s", lookupErr.Error()) } // Assert that the symbol is of type Exporter @@ -116,9 +115,9 @@ func execute() { var wg syn.WaitGroup if len(ethEventInitializers) > 0 { ew := watcher.NewEventWatcher(&db, blockChain) - err = ew.AddTransformers(ethEventInitializers) - if err != nil { - LogWithCommand.Fatalf("failed to add event transformer initializers to watcher: %s", err.Error()) + addErr := ew.AddTransformers(ethEventInitializers) + if addErr != nil { + LogWithCommand.Fatalf("failed to add event transformer initializers to watcher: %s", addErr.Error()) } wg.Add(1) go watchEthEvents(&ew, &wg) diff --git a/go.mod b/go.mod index 6c1b0ba36..ee1842676 100644 --- a/go.mod +++ b/go.mod @@ -7,14 +7,14 @@ require ( github.com/allegro/bigcache v1.2.1 // indirect github.com/apilayer/freegeoip v3.5.0+incompatible // indirect github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015 // indirect - github.com/btcsuite/btcd v0.20.0-beta // indirect + github.com/btcsuite/btcd v0.20.1-beta // indirect github.com/cespare/cp v1.1.1 // indirect github.com/dave/jennifer v1.3.0 github.com/deckarep/golang-set v1.7.1 // indirect github.com/docker/docker v1.13.1 // indirect github.com/edsrzf/mmap-go v1.0.0 // indirect github.com/elastic/gosigar v0.10.4 // indirect - github.com/ethereum/go-ethereum v1.9.5 + github.com/ethereum/go-ethereum v1.9.6 github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect github.com/go-sql-driver/mysql v1.4.1 // indirect @@ -55,9 +55,10 @@ require ( github.com/syndtr/goleveldb v1.0.0 // indirect github.com/tyler-smith/go-bip39 v1.0.2 // indirect github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 // indirect - golang.org/x/crypto v0.0.0-20190926114937-fa1a29108794 // indirect + golang.org/x/crypto v0.0.0-20191119213627-4f8c1d86b1ba // indirect golang.org/x/net v0.0.0-20190603091049-60506f45cf65 golang.org/x/sync v0.0.0-20190423024810-112230192c58 + golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 // indirect google.golang.org/appengine v1.6.5 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190709231704-1e4459ed25ff // indirect diff --git a/go.sum b/go.sum index d3f10d552..2d3462427 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,7 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/aead/siphash v1.0.1 h1:FwHfE/T45KPKYuuSAKyyvE+oPWcaQ+CUmFW0bPlM+kg= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -13,14 +14,21 @@ github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015/go.mod h1: github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/btcsuite/btcd v0.20.0-beta h1:PamBMopnHxO2nEIsU89ibVVnqnXR2yFTgGNc+PdG68o= -github.com/btcsuite/btcd v0.20.0-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= +github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= +github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= +github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= +github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d h1:yJzD/yFppdVCf6ApMkVy8cUxV0XrxdP9rVf6D87/Mng= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd h1:R/opQEbFEy9JGkIguV40SvRY1uliPX8ifOvi6ICsFCw= github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= +github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd h1:qdGvebPBDuYDPGi1WCPjy1tGyMpmDK8IEapSsszn7HE= github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= +github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723 h1:ZA/jbKoGcVAnER6pCHPEkGdZOV7U1oLUedErBHCUMs0= github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= +github.com/btcsuite/winsvc v1.0.0 h1:J9B4L7e3oqhXOcm+2IuNApwzQec85lE+QaikUcCs+dk= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/cespare/cp v1.1.1 h1:nCb6ZLdB7NRaqsm91JtQTAme2SKJzXVsdPIPkyJr1MU= github.com/cespare/cp v1.1.1/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= @@ -93,15 +101,18 @@ github.com/influxdata/influxdb v1.7.9 h1:uSeBTNO4rBkbp1Be5FKRsAmglM9nlx25TzVQRQt github.com/influxdata/influxdb v1.7.9/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= github.com/jackpal/go-nat-pmp v1.0.1 h1:i0LektDkO1QlrTm/cSuP+PyBCDnYvjPLGl4LdWEMiaA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89 h1:12K8AlpT0/6QUXSfV0yi4Q0jkbq8NDtIKFtF61AoqV0= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmoiron/sqlx v0.0.0-20181024163419-82935fac6c1a h1:Jyg5PpIc1nLGrNDM5blVkiSySmRhaD/IiXkvaHzBYnw= github.com/jmoiron/sqlx v0.0.0-20181024163419-82935fac6c1a/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/jrick/logrotate v1.0.0 h1:lQ1bL/n9mBNeIXoTUoYRlK4dHuNJVofX9oWqBtPnSzI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/karalabe/usb v0.0.0-20190819132248-550797b1cad8 h1:VhnqxaTIudc9IWKx8uXRLnpdSb9noCEj+vHacjmhp68= github.com/karalabe/usb v0.0.0-20190819132248-550797b1cad8/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23 h1:FOOIBWrEkLgmlgGfMuZT83xIwfPDxEI2OHu6xUmJMFE= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -210,8 +221,8 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190926114937-fa1a29108794 h1:4Yo9XtTfxfBCecLiBW8TYsFIdN7TkDhjGLWetFo4JSo= -golang.org/x/crypto v0.0.0-20190926114937-fa1a29108794/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= +golang.org/x/crypto v0.0.0-20191119213627-4f8c1d86b1ba h1:9bFeDpN3gTqNanMVqNcoR/pJQuP5uroC3t1D7eXozTE= +golang.org/x/crypto v0.0.0-20191119213627-4f8c1d86b1ba/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U= @@ -234,6 +245,7 @@ golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 h1:rOhMmluY6kLMhdnrivzec6lLgaVbMHMn2ISQXJeJ5EM= From 6e0f4f50589aeb09773a117dae08413e84bb5068 Mon Sep 17 00:00:00 2001 From: Rob Mulholand Date: Wed, 20 Nov 2019 10:32:23 -0600 Subject: [PATCH 2/2] Move prepConfig to root - Since it's used by multiple commands - Also clean up imports with goimports --- cmd/coldImport.go | 11 +++-- cmd/compose.go | 69 ++----------------------------- cmd/composeAndExecute.go | 19 ++++----- cmd/contractWatcher.go | 9 ++-- cmd/execute.go | 19 ++++----- cmd/fullSync.go | 7 ++-- cmd/headerSync.go | 7 ++-- cmd/root.go | 89 ++++++++++++++++++++++++++++++++-------- 8 files changed, 109 insertions(+), 121 deletions(-) diff --git a/cmd/coldImport.go b/cmd/coldImport.go index 9648ff60d..cdcea0a6a 100644 --- a/cmd/coldImport.go +++ b/cmd/coldImport.go @@ -17,17 +17,16 @@ package cmd import ( - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "github.com/makerdao/vulcanizedb/pkg/crypto" "github.com/makerdao/vulcanizedb/pkg/datastore/ethereum" "github.com/makerdao/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/makerdao/vulcanizedb/pkg/eth/cold_import" "github.com/makerdao/vulcanizedb/pkg/eth/converters/cold_db" - vulcCommon "github.com/makerdao/vulcanizedb/pkg/eth/converters/common" + "github.com/makerdao/vulcanizedb/pkg/eth/converters/common" "github.com/makerdao/vulcanizedb/pkg/fs" "github.com/makerdao/vulcanizedb/utils" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" ) var coldImportCmd = &cobra.Command{ @@ -40,7 +39,7 @@ var coldImportCmd = &cobra.Command{ Geth must be synced over all of the desired blocks and must not be running in order to execute this command.`, Run: func(cmd *cobra.Command, args []string) { SubCommand = cmd.CalledAs() - LogWithCommand = *log.WithField("SubCommand", SubCommand) + LogWithCommand = *logrus.WithField("SubCommand", SubCommand) coldImport() }, } @@ -86,7 +85,7 @@ func coldImport() { blockRepository := repositories.NewBlockRepository(&pgDB) receiptRepository := repositories.FullSyncReceiptRepository{DB: &pgDB} transactionConverter := cold_db.NewColdDbTransactionConverter() - blockConverter := vulcCommon.NewBlockConverter(transactionConverter) + blockConverter := common.NewBlockConverter(transactionConverter) // init and execute cold importer coldImporter := cold_import.NewColdImporter(ethDB, blockRepository, receiptRepository, blockConverter) diff --git a/cmd/compose.go b/cmd/compose.go index 015f70a46..3d71f2c55 100644 --- a/cmd/compose.go +++ b/cmd/compose.go @@ -17,16 +17,9 @@ package cmd import ( - "errors" - "fmt" - "strconv" - - log "github.com/sirupsen/logrus" + "github.com/makerdao/vulcanizedb/pkg/plugin" + "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "github.com/spf13/viper" - - "github.com/makerdao/vulcanizedb/pkg/config" - p2 "github.com/makerdao/vulcanizedb/pkg/plugin" ) // composeCmd represents the compose command @@ -103,7 +96,7 @@ Specify config location when executing the command: ./vulcanizedb compose --config=./environments/config_name.toml`, Run: func(cmd *cobra.Command, args []string) { SubCommand = cmd.CalledAs() - LogWithCommand = *log.WithField("SubCommand", SubCommand) + LogWithCommand = *logrus.WithField("SubCommand", SubCommand) compose() }, } @@ -117,7 +110,7 @@ func compose() { // Generate code to build the plugin according to the config file LogWithCommand.Info("generating plugin") - generator, constructorErr := p2.NewGenerator(genConfig, databaseConfig) + generator, constructorErr := plugin.NewGenerator(genConfig, databaseConfig) if constructorErr != nil { LogWithCommand.Fatalf("initializing plugin generator failed: %s", constructorErr.Error()) } @@ -136,57 +129,3 @@ func compose() { func init() { rootCmd.AddCommand(composeCmd) } - -func prepConfig() error { - LogWithCommand.Info("configuring plugin") - names := viper.GetStringSlice("exporter.transformerNames") - transformers := make(map[string]config.Transformer) - for _, name := range names { - transformer := viper.GetStringMapString("exporter." + name) - p, pOK := transformer["path"] - if !pOK || p == "" { - return fmt.Errorf("transformer config is missing `path` value: %s", name) - } - r, rOK := transformer["repository"] - if !rOK || r == "" { - return fmt.Errorf("transformer config is missing `repository` value: %s", name) - } - m, mOK := transformer["migrations"] - if !mOK || m == "" { - return fmt.Errorf("transformer config is missing `migrations` value: %s", name) - } - mr, mrOK := transformer["rank"] - if !mrOK || mr == "" { - return fmt.Errorf("transformer config is missing `rank` value: %s", name) - } - rank, err := strconv.ParseUint(mr, 10, 64) - if err != nil { - return fmt.Errorf("migration `rank` can't be converted to an unsigned integer: %s", name) - } - t, tOK := transformer["type"] - if !tOK { - return fmt.Errorf("transformer config is missing `type` value: %s", name) - } - transformerType := config.GetTransformerType(t) - if transformerType == config.UnknownTransformerType { - return errors.New(`unknown transformer type in exporter config accepted types are "eth_event", "eth_storage"`) - } - - transformers[name] = config.Transformer{ - Path: p, - Type: transformerType, - RepositoryPath: r, - MigrationPath: m, - MigrationRank: rank, - } - } - - genConfig = config.Plugin{ - Transformers: transformers, - FilePath: "$GOPATH/src/github.com/makerdao/vulcanizedb/plugins", - FileName: viper.GetString("exporter.name"), - Save: viper.GetBool("exporter.save"), - Home: viper.GetString("exporter.home"), - } - return nil -} diff --git a/cmd/composeAndExecute.go b/cmd/composeAndExecute.go index 573b7b153..0938096b2 100644 --- a/cmd/composeAndExecute.go +++ b/cmd/composeAndExecute.go @@ -17,21 +17,20 @@ package cmd import ( - "github.com/ethereum/go-ethereum/statediff" - "github.com/makerdao/vulcanizedb/libraries/shared/fetcher" - "github.com/makerdao/vulcanizedb/libraries/shared/streamer" - "github.com/makerdao/vulcanizedb/pkg/fs" "plugin" syn "sync" "time" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - + "github.com/ethereum/go-ethereum/statediff" + "github.com/makerdao/vulcanizedb/libraries/shared/fetcher" + "github.com/makerdao/vulcanizedb/libraries/shared/streamer" "github.com/makerdao/vulcanizedb/libraries/shared/watcher" + "github.com/makerdao/vulcanizedb/pkg/fs" p2 "github.com/makerdao/vulcanizedb/pkg/plugin" "github.com/makerdao/vulcanizedb/pkg/plugin/helpers" "github.com/makerdao/vulcanizedb/utils" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" ) // composeAndExecuteCmd represents the composeAndExecute command @@ -108,7 +107,7 @@ Specify config location when executing the command: ./vulcanizedb composeAndExecute --config=./environments/config_name.toml`, Run: func(cmd *cobra.Command, args []string) { SubCommand = cmd.CalledAs() - LogWithCommand = *log.WithField("SubCommand", SubCommand) + LogWithCommand = *logrus.WithField("SubCommand", SubCommand) composeAndExecute() }, } @@ -181,7 +180,7 @@ func composeAndExecute() { if len(ethStorageInitializers) > 0 { switch storageDiffsSource { case "geth": - log.Debug("fetching storage diffs from geth pub sub") + logrus.Debug("fetching storage diffs from geth pub sub") rpcClient, _ := getClients() stateDiffStreamer := streamer.NewStateDiffStreamer(rpcClient) payloadChan := make(chan statediff.Payload) @@ -191,7 +190,7 @@ func composeAndExecute() { wg.Add(1) go watchEthStorage(&sw, &wg) default: - log.Debug("fetching storage diffs from csv") + logrus.Debug("fetching storage diffs from csv") tailer := fs.FileTailer{Path: storageDiffsPath} storageFetcher := fetcher.NewCsvTailStorageFetcher(tailer) sw := watcher.NewStorageWatcher(storageFetcher, &db) diff --git a/cmd/contractWatcher.go b/cmd/contractWatcher.go index 07ade1227..bbd669895 100644 --- a/cmd/contractWatcher.go +++ b/cmd/contractWatcher.go @@ -18,16 +18,15 @@ package cmd import ( "fmt" - "github.com/makerdao/vulcanizedb/pkg/config" "time" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - st "github.com/makerdao/vulcanizedb/libraries/shared/transformer" + "github.com/makerdao/vulcanizedb/pkg/config" ft "github.com/makerdao/vulcanizedb/pkg/contract_watcher/full/transformer" ht "github.com/makerdao/vulcanizedb/pkg/contract_watcher/header/transformer" "github.com/makerdao/vulcanizedb/utils" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" ) // contractWatcherCmd represents the contractWatcher command @@ -80,7 +79,7 @@ Requires a .toml config file: `, Run: func(cmd *cobra.Command, args []string) { SubCommand = cmd.CalledAs() - LogWithCommand = *log.WithField("SubCommand", SubCommand) + LogWithCommand = *logrus.WithField("SubCommand", SubCommand) contractWatcher() }, } diff --git a/cmd/execute.go b/cmd/execute.go index 5230cec89..4f6f79996 100644 --- a/cmd/execute.go +++ b/cmd/execute.go @@ -17,22 +17,21 @@ package cmd import ( - "github.com/ethereum/go-ethereum/statediff" - "github.com/makerdao/vulcanizedb/libraries/shared/fetcher" - "github.com/makerdao/vulcanizedb/libraries/shared/streamer" - "github.com/makerdao/vulcanizedb/pkg/fs" "plugin" syn "sync" "time" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - + "github.com/ethereum/go-ethereum/statediff" "github.com/makerdao/vulcanizedb/libraries/shared/constants" + "github.com/makerdao/vulcanizedb/libraries/shared/fetcher" storageUtils "github.com/makerdao/vulcanizedb/libraries/shared/storage/utils" + "github.com/makerdao/vulcanizedb/libraries/shared/streamer" "github.com/makerdao/vulcanizedb/libraries/shared/transformer" "github.com/makerdao/vulcanizedb/libraries/shared/watcher" + "github.com/makerdao/vulcanizedb/pkg/fs" "github.com/makerdao/vulcanizedb/utils" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" ) // executeCmd represents the execute command @@ -66,7 +65,7 @@ Specify config location when executing the command: ./vulcanizedb execute --config=./environments/config_name.toml`, Run: func(cmd *cobra.Command, args []string) { SubCommand = cmd.CalledAs() - LogWithCommand = *log.WithField("SubCommand", SubCommand) + LogWithCommand = *logrus.WithField("SubCommand", SubCommand) execute() }, } @@ -126,7 +125,7 @@ func execute() { if len(ethStorageInitializers) > 0 { switch storageDiffsSource { case "geth": - log.Debug("fetching storage diffs from geth pub sub") + logrus.Debug("fetching storage diffs from geth pub sub") rpcClient, _ := getClients() stateDiffStreamer := streamer.NewStateDiffStreamer(rpcClient) payloadChan := make(chan statediff.Payload) @@ -136,7 +135,7 @@ func execute() { wg.Add(1) go watchEthStorage(&sw, &wg) default: - log.Debug("fetching storage diffs from csv") + logrus.Debug("fetching storage diffs from csv") tailer := fs.FileTailer{Path: storageDiffsPath} storageFetcher := fetcher.NewCsvTailStorageFetcher(tailer) sw := watcher.NewStorageWatcher(storageFetcher, &db) diff --git a/cmd/fullSync.go b/cmd/fullSync.go index df0199ab3..c87200458 100644 --- a/cmd/fullSync.go +++ b/cmd/fullSync.go @@ -19,14 +19,13 @@ package cmd import ( "time" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "github.com/makerdao/vulcanizedb/pkg/core" "github.com/makerdao/vulcanizedb/pkg/datastore" "github.com/makerdao/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/makerdao/vulcanizedb/pkg/history" "github.com/makerdao/vulcanizedb/utils" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" ) // fullSyncCmd represents the fullSync command @@ -50,7 +49,7 @@ Expects ethereum node to be running and requires a .toml config: `, Run: func(cmd *cobra.Command, args []string) { SubCommand = cmd.CalledAs() - LogWithCommand = *log.WithField("SubCommand", SubCommand) + LogWithCommand = *logrus.WithField("SubCommand", SubCommand) fullSync() }, } diff --git a/cmd/headerSync.go b/cmd/headerSync.go index a6085f751..32edad253 100644 --- a/cmd/headerSync.go +++ b/cmd/headerSync.go @@ -19,15 +19,14 @@ package cmd import ( "time" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "github.com/makerdao/vulcanizedb/pkg/core" "github.com/makerdao/vulcanizedb/pkg/datastore" "github.com/makerdao/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/makerdao/vulcanizedb/pkg/eth" "github.com/makerdao/vulcanizedb/pkg/history" "github.com/makerdao/vulcanizedb/utils" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" ) // headerSyncCmd represents the headerSync command @@ -51,7 +50,7 @@ Expects ethereum node to be running and requires a .toml config: `, Run: func(cmd *cobra.Command, args []string) { SubCommand = cmd.CalledAs() - LogWithCommand = *log.WithField("SubCommand", SubCommand) + LogWithCommand = *logrus.WithField("SubCommand", SubCommand) headerSync() }, } diff --git a/cmd/root.go b/cmd/root.go index 2b72cabeb..27bc9f7e6 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -17,21 +17,22 @@ package cmd import ( + "errors" "fmt" + "strconv" "strings" "time" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/rpc" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "github.com/spf13/viper" - "github.com/makerdao/vulcanizedb/pkg/config" "github.com/makerdao/vulcanizedb/pkg/eth" "github.com/makerdao/vulcanizedb/pkg/eth/client" vRpc "github.com/makerdao/vulcanizedb/pkg/eth/converters/rpc" "github.com/makerdao/vulcanizedb/pkg/eth/node" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/spf13/viper" ) var ( @@ -47,7 +48,7 @@ var ( endingBlockNumber int64 recheckHeadersArg bool SubCommand string - LogWithCommand log.Entry + LogWithCommand logrus.Entry storageDiffsSource string ) @@ -62,9 +63,9 @@ var rootCmd = &cobra.Command{ } func Execute() { - log.Info("----- Starting vDB -----") + logrus.Info("----- Starting vDB -----") if err := rootCmd.Execute(); err != nil { - log.Fatal(err) + logrus.Fatal(err) } } @@ -72,7 +73,7 @@ func initFuncs(cmd *cobra.Command, args []string) { setViperConfigs() logLvlErr := logLevel() if logLvlErr != nil { - log.Fatal("Could not set log level: ", logLvlErr) + logrus.Fatal("Could not set log level: ", logLvlErr) } } @@ -93,15 +94,15 @@ func setViperConfigs() { } func logLevel() error { - lvl, err := log.ParseLevel(viper.GetString("log.level")) + lvl, err := logrus.ParseLevel(viper.GetString("log.level")) if err != nil { return err } - log.SetLevel(lvl) - if lvl > log.InfoLevel { - log.SetReportCaller(true) + logrus.SetLevel(lvl) + if lvl > logrus.InfoLevel { + logrus.SetReportCaller(true) } - log.Info("Log level set to ", lvl.String()) + logrus.Info("Log level set to ", lvl.String()) return nil } @@ -122,7 +123,7 @@ func init() { rootCmd.PersistentFlags().String("filesystem-storageDiffsPath", "", "location of storage diffs csv file") rootCmd.PersistentFlags().String("storageDiffs-source", "csv", "where to get the state diffs: csv or geth") rootCmd.PersistentFlags().String("exporter-name", "exporter", "name of exporter plugin") - rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "Log level (trace, debug, info, warn, error, fatal, panic") + rootCmd.PersistentFlags().String("log-level", logrus.InfoLevel.String(), "Log level (trace, debug, info, warn, error, fatal, panic") viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name")) viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port")) @@ -143,15 +144,15 @@ func initConfig() { } else { noConfigError := "No config file passed with --config flag" fmt.Println("Error: ", noConfigError) - log.Fatal(noConfigError) + logrus.Fatal(noConfigError) } if err := viper.ReadInConfig(); err == nil { - log.Printf("Using config file: %s\n\n", viper.ConfigFileUsed()) + logrus.Printf("Using config file: %s\n\n", viper.ConfigFileUsed()) } else { invalidConfigError := "Couldn't read config file" formattedError := fmt.Sprintf("%s: %s", invalidConfigError, err.Error()) - log.Fatal(formattedError) + logrus.Fatal(formattedError) } } @@ -174,3 +175,57 @@ func getClients() (client.RpcClient, *ethclient.Client) { return rpcClient, ethClient } + +func prepConfig() error { + LogWithCommand.Info("configuring plugin") + names := viper.GetStringSlice("exporter.transformerNames") + transformers := make(map[string]config.Transformer) + for _, name := range names { + transformer := viper.GetStringMapString("exporter." + name) + p, pOK := transformer["path"] + if !pOK || p == "" { + return fmt.Errorf("transformer config is missing `path` value: %s", name) + } + r, rOK := transformer["repository"] + if !rOK || r == "" { + return fmt.Errorf("transformer config is missing `repository` value: %s", name) + } + m, mOK := transformer["migrations"] + if !mOK || m == "" { + return fmt.Errorf("transformer config is missing `migrations` value: %s", name) + } + mr, mrOK := transformer["rank"] + if !mrOK || mr == "" { + return fmt.Errorf("transformer config is missing `rank` value: %s", name) + } + rank, err := strconv.ParseUint(mr, 10, 64) + if err != nil { + return fmt.Errorf("migration `rank` can't be converted to an unsigned integer: %s", name) + } + t, tOK := transformer["type"] + if !tOK { + return fmt.Errorf("transformer config is missing `type` value: %s", name) + } + transformerType := config.GetTransformerType(t) + if transformerType == config.UnknownTransformerType { + return errors.New(`unknown transformer type in exporter config accepted types are "eth_event", "eth_storage"`) + } + + transformers[name] = config.Transformer{ + Path: p, + Type: transformerType, + RepositoryPath: r, + MigrationPath: m, + MigrationRank: rank, + } + } + + genConfig = config.Plugin{ + Transformers: transformers, + FilePath: "$GOPATH/src/github.com/makerdao/vulcanizedb/plugins", + FileName: viper.GetString("exporter.name"), + Save: viper.GetBool("exporter.save"), + Home: viper.GetString("exporter.home"), + } + return nil +}