diff --git a/.travis.yml b/.travis.yml index a8ebb0648..9bf1dc7ad 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ dist: trusty language: go go: -- 1.14 +- 1.15 services: - postgresql - docker @@ -20,7 +20,7 @@ deploy: - provider: script script: bash ./.travis/deploy.sh staging on: - branch: staging + branch: beta - provider: script script: bash ./.travis/deploy.sh prod on: diff --git a/.travis/deploy.sh b/.travis/deploy.sh index e54ae76ac..c2c7788e2 100755 --- a/.travis/deploy.sh +++ b/.travis/deploy.sh @@ -29,9 +29,6 @@ fi message BUILDING HEADER-SYNC docker build -f dockerfiles/header_sync/Dockerfile . -t makerdao/vdb-headersync:$TAG -message BUILDING EXTRACT-DIFFS -docker build -f dockerfiles/extract_diffs/Dockerfile . -t makerdao/vdb-extract-diffs:$TAG - message BUILDING RESET-HEADER-CHECK docker build -f dockerfiles/reset_header_check_count/Dockerfile . -t makerdao/vdb-reset-header-check:$TAG @@ -42,9 +39,6 @@ echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USER" --password-stdi message PUSHING HEADER-SYNC docker push makerdao/vdb-headersync:$TAG -message PUSHING EXTRACT-DIFFS -docker push makerdao/vdb-extract-diffs:$TAG - message PUSHING RESET-HEADER-CHECK docker push makerdao/vdb-reset-header-check:$TAG @@ -53,17 +47,9 @@ if [ "$ENVIRONMENT" == "prod" ]; then message DEPLOYING HEADER-SYNC aws ecs update-service --cluster vdb-cluster-$ENVIRONMENT --service vdb-header-sync-$ENVIRONMENT --force-new-deployment --endpoint https://ecs.$PROD_REGION.amazonaws.com --region $PROD_REGION - message DEPLOYING EXTRACT-DIFFS - aws ecs update-service --cluster vdb-cluster-$ENVIRONMENT --service vdb-extract-diffs-$ENVIRONMENT --force-new-deployment --endpoint https://ecs.$PROD_REGION.amazonaws.com --region $PROD_REGION elif [ "$ENVIRONMENT" == "staging" ]; then message DEPLOYING HEADER-SYNC aws ecs update-service --cluster vdb-cluster-$ENVIRONMENT --service vdb-header-sync-$ENVIRONMENT --force-new-deployment --endpoint https://ecs.$STAGING_REGION.amazonaws.com --region $STAGING_REGION - - message DEPLOYING EXTRACT-DIFFS - aws ecs update-service --cluster vdb-cluster-$ENVIRONMENT --service vdb-extract-diffs-$ENVIRONMENT --force-new-deployment --endpoint https://ecs.$STAGING_REGION.amazonaws.com --region $STAGING_REGION - - message DEPLOYING EXTRACT-DIFFS-NEW-GETH - aws ecs update-service --cluster vdb-cluster-$ENVIRONMENT --service vdb-extract-diffs2-$ENVIRONMENT --force-new-deployment --endpoint https://ecs.$STAGING_REGION.amazonaws.com --region $STAGING_REGION else message UNKNOWN ENVIRONMENT fi diff --git a/Makefile b/Makefile index cabc1e4df..bac547076 100644 --- a/Makefile +++ b/Makefile @@ -9,10 +9,7 @@ $(BIN)/ginkgo: go get -u github.com/onsi/ginkgo/ginkgo ## Migration tool -GOOSE = $(BIN)/goose -$(BIN)/goose: - go get -u -d github.com/pressly/goose/cmd/goose - go build -tags='no_mysql no_sqlite' -o $(BIN)/goose github.com/pressly/goose/cmd/goose +GOOSE = go run -tags='no_mysql no_sqlite3 no_mssql no_redshift' github.com/pressly/goose/cmd/goose ## Source linter LINT = $(BIN)/golint @@ -25,9 +22,8 @@ $(BIN)/gometalinter.v2: go get -u gopkg.in/alecthomas/gometalinter.v2 $(METALINT) --install - .PHONY: installtools -installtools: | $(LINT) $(GOOSE) $(GINKGO) +installtools: | $(LINT) $(GINKGO) echo "Installing tools" .PHONY: metalint @@ -65,6 +61,7 @@ test: | $(GINKGO) $(LINT) .PHONY: integrationtest integrationtest: | $(GINKGO) $(LINT) + test -n "$(CLIENT_IPCPATH)" # $$(CLIENT_IPCPATH) go vet ./... go fmt ./... dropdb --if-exists $(TEST_DB) @@ -100,30 +97,30 @@ checkmigname: # Migration operations ## Rollback the last migration .PHONY: rollback -rollback: $(GOOSE) checkdbvars +rollback: checkdbvars $(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" down - pg_dump -O -s $(CONNECT_STRING) > db/schema.sql + pg_dump -n 'public' -O -s $(CONNECT_STRING) > db/schema.sql ## Rollbackt to a select migration (id/timestamp) .PHONY: rollback_to -rollback_to: $(GOOSE) checkmigration checkdbvars +rollback_to: checkmigration checkdbvars $(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" down-to "$(MIGRATION)" ## Apply all migrations not already run .PHONY: migrate -migrate: $(GOOSE) checkdbvars +migrate: checkdbvars $(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" up - pg_dump -O -s $(CONNECT_STRING) > db/schema.sql + pg_dump -n 'public' -O -s $(CONNECT_STRING) > db/schema.sql ## Create a new migration file .PHONY: new_migration -new_migration: $(GOOSE) checkmigname +new_migration: checkmigname $(GOOSE) -dir db/migrations create $(NAME) sql ## Check which migrations are applied at the moment .PHONY: migration_status -migration_status: $(GOOSE) checkdbvars +migration_status: checkdbvars $(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" status # Convert timestamped migrations to versioned (to be run in CI); @@ -138,25 +135,29 @@ import: test -n "$(NAME)" # $$NAME psql $(NAME) < db/schema.sql - # Docker actions -## Rinkeby docker environment -RINKEBY_COMPOSE_FILE=dockerfiles/rinkeby/docker-compose.yml - -.PHONY: rinkeby_env_up -rinkeby_env_up: - docker-compose -f $(RINKEBY_COMPOSE_FILE) up -d geth - docker-compose -f $(RINKEBY_COMPOSE_FILE) up --build migrations - docker-compose -f $(RINKEBY_COMPOSE_FILE) up -d --build vulcanizedb - -.PHONY: rinkeby_env_deploy -rinkeby_env_deploy: - docker-compose -f $(RINKEBY_COMPOSE_FILE) up -d --build vulcanizedb - -.PHONY: dev_env_migrate -rinkeby_env_migrate: - docker-compose -f $(RINKEBY_COMPOSE_FILE) up --build migrations - -.PHONY: rinkeby_env_down -rinkeby_env_down: - docker-compose -f $(RINKEBY_COMPOSE_FILE) down +# Build any docker image in dockerfiles +.PHONY: dockerbuild +dockerbuild: + test -n "$(IMAGE)" # $$IMAGE + docker build -t $(IMAGE) -f dockerfiles/$(IMAGE)/Dockerfile . + +.PHONY: header_sync +header_sync: STARTING_BLOCK_NUMBER ?= 10000000 +header_sync: HOST ?= host.docker.internal +header_sync: DATABASE_PASSWORD ?= postgres +header_sync: IMAGE_WITH_TAG ?= header_sync:latest +header_sync: + test -n "$(NAME)" # $$(NAME) - Database Name + test -n "$(CLIENT_IPCPATH)" # $$(CLIENT_IPCPATH) + docker run \ + -it \ + -p "5432:5432" \ + -e "STARTING_BLOCK_NUMBER=$(STARTING_BLOCK_NUMBER)" \ + -e "DATABASE_NAME=$(NAME)" \ + -e "DATABASE_HOSTNAME=$(HOST)" \ + -e "DATABASE_PORT=$(PORT)" \ + -e "DATABASE_USER=$(USER)" \ + -e "DATABASE_PASSWORD=$(DATABASE_PASSWORD)" \ + -e "CLIENT_IPCPATH=$(CLIENT_IPCPATH)" \ + $(IMAGE_WITH_TAG) diff --git a/cmd/backfillEvents.go b/cmd/backfillEvents.go index 2bd418787..632cc1e83 100644 --- a/cmd/backfillEvents.go +++ b/cmd/backfillEvents.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/makerdao/vulcanizedb/libraries/shared/logs" + "github.com/makerdao/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/makerdao/vulcanizedb/utils" "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -42,10 +43,19 @@ func backFillEvents() error { LogWithCommand.Fatalf("SubCommand %v: exporting transformers failed: %v", SubCommand, exportTransformersErr) } + if len(ethEventInitializers) < 1 { + logrus.Warn("not back-filling events because no transformers configured for back-fill") + return nil + } + blockChain := getBlockChain() db := utils.LoadPostgres(databaseConfig, blockChain.Node()) - extractor := logs.NewLogExtractor(&db, blockChain) + repo, repoErr := repositories.NewCheckedHeadersRepository(&db, genConfig.Schema) + if repoErr != nil { + return fmt.Errorf("error creating checked headers repository %w for schema %s", repoErr, genConfig.Schema) + } + extractor := logs.NewLogExtractor(&db, blockChain, repo) for _, initializer := range ethEventInitializers { transformer := initializer(&db) diff --git a/cmd/backfillStorage.go b/cmd/backfillStorage.go index aa7d2c2e5..dbdc2445a 100644 --- a/cmd/backfillStorage.go +++ b/cmd/backfillStorage.go @@ -64,7 +64,8 @@ func backfillStorage() error { } if len(storageInitializers) == 0 { - return fmt.Errorf("SubCommand %v: no storage transformers found in the given config", SubCommand) + logrus.Warn("not back-filling storage because no contracts configured for back-fill") + return nil } var loader backfill.StorageValueLoader diff --git a/cmd/compose.go b/cmd/compose.go index 0a3daf483..4a29104e4 100644 --- a/cmd/compose.go +++ b/cmd/compose.go @@ -41,6 +41,7 @@ var composeCmd = &cobra.Command{ [exporter] home = "github.com/makerdao/vulcanizedb" name = "exampleTransformerExporter" + schema = "" save = false transformerNames = [ "transformer1", diff --git a/cmd/execute.go b/cmd/execute.go index b77ead1b7..46fc0eb72 100644 --- a/cmd/execute.go +++ b/cmd/execute.go @@ -26,6 +26,7 @@ import ( "github.com/makerdao/vulcanizedb/libraries/shared/logs" "github.com/makerdao/vulcanizedb/libraries/shared/transformer" "github.com/makerdao/vulcanizedb/libraries/shared/watcher" + "github.com/makerdao/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/makerdao/vulcanizedb/pkg/fs" "github.com/makerdao/vulcanizedb/utils" "github.com/sirupsen/logrus" @@ -77,7 +78,8 @@ func init() { executeCmd.Flags().BoolVarP(&recheckHeadersArg, "recheck-headers", "r", false, "whether to re-check headers for watched events") executeCmd.Flags().DurationVarP(&retryInterval, "retry-interval", "i", 7*time.Second, "interval duration between retries on execution error") executeCmd.Flags().IntVarP(&maxUnexpectedErrors, "max-unexpected-errs", "m", 5, "maximum number of unexpected errors to allow (with retries) before exiting") - executeCmd.Flags().Int64VarP(&diffBlockFromHeadOfChain, "diff-blocks-from-head", "d", -1, "number of blocks from head of chain to start reprocessing diffs, defaults to -1 so all diffs are processsed") + executeCmd.Flags().Int64VarP(&newDiffBlockFromHeadOfChain, "new-diff-blocks-from-head", "d", -1, "number of blocks from head of chain to start reprocessing new diffs, defaults to -1 so all diffs are processsed") + executeCmd.Flags().Int64VarP(&unrecognizedDiffBlockFromHeadOfChain, "unrecognized-diff-blocks-from-head", "u", -1, "number of blocks from head of chain to start reprocessing unrecognized diffs, defaults to -1 so all diffs are processsed") } func executeTransformers() { @@ -95,7 +97,11 @@ func executeTransformers() { // Use WaitGroup to wait on both goroutines var wg sync.WaitGroup if len(ethEventInitializers) > 0 { - extractor := logs.NewLogExtractor(&db, blockChain) + repo, repoErr := repositories.NewCheckedHeadersRepository(&db, genConfig.Schema) + if repoErr != nil { + LogWithCommand.Fatalf("failed to create checked headers repository %s for schema %s", repoErr.Error(), genConfig.Schema) + } + extractor := logs.NewLogExtractor(&db, blockChain, repo) delegator := logs.NewLogDelegator(&db) eventHealthCheckMessage := []byte("event watcher starting\n") statusWriter := fs.NewStatusWriter(healthCheckFile, eventHealthCheckMessage) @@ -109,12 +115,19 @@ func executeTransformers() { } if len(ethStorageInitializers) > 0 { - storageHealthCheckMessage := []byte("storage watcher starting\n") - statusWriter := fs.NewStatusWriter(healthCheckFile, storageHealthCheckMessage) - sw := watcher.NewStorageWatcher(&db, diffBlockFromHeadOfChain, statusWriter) - sw.AddTransformers(ethStorageInitializers) + newDiffStorageHealthCheckMessage := []byte("storage watcher for new diffs starting\n") + newDiffStatusWriter := fs.NewStatusWriter(healthCheckFile, newDiffStorageHealthCheckMessage) + newDiffStorageWatcher := watcher.NewStorageWatcher(&db, newDiffBlockFromHeadOfChain, newDiffStatusWriter, watcher.New) + newDiffStorageWatcher.AddTransformers(ethStorageInitializers) + wg.Add(1) + go watchEthStorage(&newDiffStorageWatcher, &wg) + + unrecognizedDiffStorageHealthCheckMessage := []byte("storage watcher for unrecognized diffs starting\n") + unrecognizedDiffStatusWriter := fs.NewStatusWriter(healthCheckFile, unrecognizedDiffStorageHealthCheckMessage) + unrecognizedDiffStorageWatcher := watcher.NewStorageWatcher(&db, unrecognizedDiffBlockFromHeadOfChain, unrecognizedDiffStatusWriter, watcher.Unrecognized) + unrecognizedDiffStorageWatcher.AddTransformers(ethStorageInitializers) wg.Add(1) - go watchEthStorage(&sw, &wg) + go watchEthStorage(&unrecognizedDiffStorageWatcher, &wg) } if len(ethContractInitializers) > 0 { diff --git a/cmd/extractDiffs.go b/cmd/extractDiffs.go index 844811aa6..02959e141 100644 --- a/cmd/extractDiffs.go +++ b/cmd/extractDiffs.go @@ -1,7 +1,11 @@ package cmd import ( - "github.com/ethereum/go-ethereum/statediff" + "strings" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/makerdao/vulcanizedb/libraries/shared/storage" "github.com/makerdao/vulcanizedb/libraries/shared/storage/fetcher" "github.com/makerdao/vulcanizedb/libraries/shared/streamer" @@ -9,6 +13,7 @@ import ( "github.com/makerdao/vulcanizedb/utils" "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "github.com/spf13/viper" ) // extractDiffsCmd represents the extractDiffs command @@ -29,10 +34,22 @@ func init() { rootCmd.AddCommand(extractDiffsCmd) } +func getContractAddresses() []string { + LogWithCommand.Info("Getting contract addresses from config file") + contracts := viper.GetStringMap("contract") + var addresses []string + for contractName := range contracts { + address := viper.GetStringMapString("contract." + contractName)["address"] + addresses = append(addresses, address) + } + return addresses +} + func extractDiffs() { // Setup bc and db objects blockChain := getBlockChain() db := utils.LoadPostgres(databaseConfig, blockChain.Node()) + addressesToWatch := getContractAddresses() healthCheckFile := "/tmp/connection" msg := []byte("geth storage fetcher connection established\n") @@ -40,23 +57,15 @@ func extractDiffs() { // initialize fetcher var storageFetcher fetcher.IStorageFetcher + logrus.Debug("fetching storage diffs from geth") switch storageDiffsSource { case "geth": - logrus.Info("Using original geth patch") - logrus.Debug("fetching storage diffs from geth pub sub") - rpcClient, _ := getClients() - stateDiffStreamer := streamer.NewStateDiffStreamer(rpcClient) - payloadChan := make(chan statediff.Payload) - - storageFetcher = fetcher.NewGethRpcStorageFetcher(&stateDiffStreamer, payloadChan, fetcher.OldGethPatch, gethStatusWriter) - case "new-geth": - logrus.Info("Using new geth patch") - logrus.Debug("fetching storage diffs from geth pub sub") - rpcClient, _ := getClients() - stateDiffStreamer := streamer.NewStateDiffStreamer(rpcClient) - payloadChan := make(chan statediff.Payload) - - storageFetcher = fetcher.NewGethRpcStorageFetcher(&stateDiffStreamer, payloadChan, fetcher.NewGethPatch, gethStatusWriter) + logrus.Info("Using new geth patch with filters event system") + _, ethClient := getClients() + filterQuery := createFilterQuery(addressesToWatch) + stateDiffStreamer := streamer.NewEthStateChangeStreamer(ethClient, filterQuery) + payloadChan := make(chan filters.Payload) + storageFetcher = fetcher.NewGethRpcStorageFetcher(&stateDiffStreamer, payloadChan, gethStatusWriter) default: logrus.Debug("fetching storage diffs from csv") tailer := fs.FileTailer{Path: storageDiffsPath} @@ -73,3 +82,18 @@ func extractDiffs() { LogWithCommand.Fatalf("extracting diffs failed: %s", err.Error()) } } + +func createFilterQuery(watchedAddresses []string) ethereum.FilterQuery { + logrus.Infof("Creating a filter query for %d watched addresses", len(watchedAddresses)) + addressesToLog := strings.Join(watchedAddresses[:], ", ") + logrus.Infof("Watched addresses: %s", addressesToLog) + + var addresses []common.Address + for _, addressString := range watchedAddresses { + addresses = append(addresses, common.HexToAddress(addressString)) + } + + return ethereum.FilterQuery{ + Addresses: addresses, + } +} diff --git a/cmd/resetHeaderCheckCount.go b/cmd/resetHeaderCheckCount.go index 86df7e03a..c0438df89 100644 --- a/cmd/resetHeaderCheckCount.go +++ b/cmd/resetHeaderCheckCount.go @@ -64,6 +64,11 @@ func init() { func resetHeaderCount(blockNumber int64) error { blockChain := getBlockChain() db := utils.LoadPostgres(databaseConfig, blockChain.Node()) - repo := repositories.NewCheckedHeadersRepository(&db) + repo, repoErr := repositories.NewCheckedHeadersRepository(&db, genConfig.Schema) + + if repoErr != nil { + return fmt.Errorf("error creating checked headers repository %w", repoErr) + } + return repo.MarkSingleHeaderUnchecked(blockNumber) } diff --git a/cmd/root.go b/cmd/root.go index 085e7236a..2f5892aa4 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -40,19 +40,20 @@ import ( ) var ( - LogWithCommand logrus.Entry - SubCommand string - cfgFile string - databaseConfig config.Database - diffBlockFromHeadOfChain int64 - genConfig config.Plugin - ipc string - maxUnexpectedErrors int - recheckHeadersArg bool - retryInterval time.Duration - startingBlockNumber int64 - storageDiffsPath string - storageDiffsSource string + LogWithCommand logrus.Entry + SubCommand string + cfgFile string + databaseConfig config.Database + newDiffBlockFromHeadOfChain int64 + unrecognizedDiffBlockFromHeadOfChain int64 + genConfig config.Plugin + ipc string + maxUnexpectedErrors int + recheckHeadersArg bool + retryInterval time.Duration + startingBlockNumber int64 + storageDiffsPath string + storageDiffsSource string ) const ( @@ -219,6 +220,7 @@ func prepConfig() error { genConfig = config.Plugin{ Transformers: transformers, FilePath: "$GOPATH/src/github.com/makerdao/vulcanizedb/plugins", + Schema: viper.GetString("exporter.schema"), FileName: viper.GetString("exporter.name"), Save: viper.GetBool("exporter.save"), Home: viper.GetString("exporter.home"), diff --git a/db/migrations/00002_add_address_table.sql b/db/migrations/00002_add_address_table.sql index 6866f6e1d..d644b95e7 100644 --- a/db/migrations/00002_add_address_table.sql +++ b/db/migrations/00002_add_address_table.sql @@ -1,11 +1,10 @@ -- +goose Up CREATE TABLE public.addresses ( - id SERIAL PRIMARY KEY, + id BIGSERIAL PRIMARY KEY, address character varying(42), - hashed_address character varying(66), UNIQUE (address) ); -- +goose Down -DROP TABLE public.addresses; \ No newline at end of file +DROP TABLE public.addresses; diff --git a/db/migrations/00003_create_headers_table.sql b/db/migrations/00003_create_headers_table.sql index 76aec7436..f3dc45144 100644 --- a/db/migrations/00003_create_headers_table.sql +++ b/db/migrations/00003_create_headers_table.sql @@ -6,7 +6,6 @@ CREATE TABLE public.headers block_number BIGINT NOT NULL, raw JSONB, block_timestamp NUMERIC, - check_count INTEGER NOT NULL DEFAULT 0, eth_node_id INTEGER NOT NULL REFERENCES eth_nodes (id) ON DELETE CASCADE, created TIMESTAMP NOT NULL DEFAULT NOW(), updated TIMESTAMP NOT NULL DEFAULT NOW(), @@ -29,17 +28,15 @@ CREATE TRIGGER header_updated FOR EACH ROW EXECUTE PROCEDURE set_header_updated(); --- Index is removed when table is -CREATE INDEX headers_block_number ON public.headers (block_number); -CREATE INDEX headers_check_count ON public.headers (check_count); -CREATE INDEX headers_eth_node ON public.headers (eth_node_id); +CREATE INDEX headers_block_number + ON public.headers (block_number); +CREATE INDEX headers_block_timestamp_index + ON public.headers (block_timestamp); +CREATE INDEX headers_eth_node + ON public.headers (eth_node_id); -- +goose Down -DROP INDEX headers_block_number; -DROP INDEX headers_check_count; -DROP INDEX headers_eth_node; - DROP TRIGGER header_updated ON public.headers; DROP FUNCTION set_header_updated(); diff --git a/db/migrations/00005_create_storage_diffs_table.sql b/db/migrations/00005_create_storage_diffs_table.sql index 43806a3ea..4e7109473 100644 --- a/db/migrations/00005_create_storage_diffs_table.sql +++ b/db/migrations/00005_create_storage_diffs_table.sql @@ -1,16 +1,54 @@ -- +goose Up +CREATE TYPE public.diff_status AS ENUM ( + 'new', + 'transformed', + 'unrecognized', + 'noncanonical', + 'unwatched' + ); + CREATE TABLE public.storage_diff ( id BIGSERIAL PRIMARY KEY, + address BYTEA, block_height BIGINT, block_hash BYTEA, - hashed_address BYTEA, storage_key BYTEA, storage_value BYTEA, - checked BOOLEAN NOT NULL DEFAULT FALSE, - from_backfill BOOLEAN NOT NULL DEFAULT FALSE, - UNIQUE (block_height, block_hash, hashed_address, storage_key, storage_value) + eth_node_id INTEGER NOT NULL REFERENCES public.eth_nodes (id) ON DELETE CASCADE, + status diff_status NOT NULL DEFAULT 'new', + from_backfill BOOLEAN NOT NULL DEFAULT FALSE, + created TIMESTAMP NOT NULL DEFAULT NOW(), + updated TIMESTAMP NOT NULL DEFAULT NOW(), + UNIQUE (block_height, block_hash, address, storage_key, storage_value) ); +CREATE INDEX storage_diff_new_status_index + ON public.storage_diff (status) WHERE status = 'new'; +CREATE INDEX storage_diff_unrecognized_status_index + ON public.storage_diff (status) WHERE status = 'unrecognized'; +CREATE INDEX storage_diff_eth_node + ON public.storage_diff (eth_node_id); + +-- +goose StatementBegin +CREATE FUNCTION set_storage_updated() RETURNS TRIGGER AS +$$ +BEGIN + NEW.updated = NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; +-- +goose StatementEnd + +CREATE TRIGGER storage_updated + BEFORE UPDATE + ON public.storage_diff + FOR EACH ROW +EXECUTE PROCEDURE set_storage_updated(); + -- +goose Down -DROP TABLE public.storage_diff; \ No newline at end of file +DROP TYPE public.diff_status CASCADE; +DROP TRIGGER storage_updated ON public.storage_diff; +DROP FUNCTION set_storage_updated(); + +DROP TABLE public.storage_diff; diff --git a/db/migrations/00006_create_transactions_table.sql b/db/migrations/00006_create_transactions_table.sql index 0c2831506..45c31b31a 100644 --- a/db/migrations/00006_create_transactions_table.sql +++ b/db/migrations/00006_create_transactions_table.sql @@ -12,12 +12,32 @@ CREATE TABLE public.transactions tx_from VARCHAR(44), tx_index INTEGER, tx_to VARCHAR(44), - "value" NUMERIC + "value" NUMERIC, + created TIMESTAMP NOT NULL DEFAULT NOW(), + updated TIMESTAMP NOT NULL DEFAULT NOW() ); +-- +goose StatementBegin +CREATE FUNCTION set_transaction_updated() RETURNS TRIGGER AS +$$ +BEGIN + NEW.updated = NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; +-- +goose StatementEnd + +CREATE TRIGGER transaction_updated + BEFORE UPDATE + ON public.transactions + FOR EACH ROW +EXECUTE PROCEDURE set_transaction_updated(); + CREATE INDEX transactions_header ON transactions (header_id); -- +goose Down -DROP INDEX transactions_header; +DROP TRIGGER transaction_updated ON public.transactions; +DROP FUNCTION set_transaction_updated(); + DROP TABLE transactions; diff --git a/db/migrations/00007_create_receipts_table.sql b/db/migrations/00007_create_receipts_table.sql index 2686f3ea6..a6e47fa36 100644 --- a/db/migrations/00007_create_receipts_table.sql +++ b/db/migrations/00007_create_receipts_table.sql @@ -4,15 +4,32 @@ CREATE TABLE public.receipts id SERIAL PRIMARY KEY, transaction_id INTEGER NOT NULL REFERENCES public.transactions (id) ON DELETE CASCADE, header_id INTEGER NOT NULL REFERENCES public.headers (id) ON DELETE CASCADE, - contract_address_id INTEGER NOT NULL REFERENCES public.addresses (id) ON DELETE CASCADE, + contract_address_id BIGINT NOT NULL REFERENCES public.addresses (id) ON DELETE CASCADE, cumulative_gas_used NUMERIC, gas_used NUMERIC, state_root VARCHAR(66), status INTEGER, tx_hash VARCHAR(66), rlp BYTEA, + created TIMESTAMP NOT NULL DEFAULT NOW(), + updated TIMESTAMP NOT NULL DEFAULT NOW(), UNIQUE (header_id, transaction_id) ); +-- +goose StatementBegin +CREATE FUNCTION set_receipt_updated() RETURNS TRIGGER AS +$$ +BEGIN + NEW.updated = NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; +-- +goose StatementEnd + +CREATE TRIGGER receipt_updated + BEFORE UPDATE + ON public.receipts + FOR EACH ROW +EXECUTE PROCEDURE set_receipt_updated(); CREATE INDEX receipts_contract_address ON public.receipts (contract_address_id); @@ -20,6 +37,6 @@ CREATE INDEX receipts_transaction ON public.receipts (transaction_id); -- +goose Down -DROP INDEX receipts_transaction; -DROP INDEX receipts_contract_address; +DROP TRIGGER receipt_updated ON public.receipts; +DROP FUNCTION set_receipt_updated(); DROP TABLE receipts; diff --git a/db/migrations/00008_create_event_logs_table.sql b/db/migrations/00008_create_event_logs_table.sql index 0e867529c..84aa3527d 100644 --- a/db/migrations/00008_create_event_logs_table.sql +++ b/db/migrations/00008_create_event_logs_table.sql @@ -1,10 +1,9 @@ -- +goose Up --- SQL in this section is executed when the migration is applied. CREATE TABLE public.event_logs ( - id SERIAL PRIMARY KEY, + id BIGSERIAL PRIMARY KEY, header_id INTEGER NOT NULL REFERENCES public.headers (id) ON DELETE CASCADE, - address INTEGER NOT NULL REFERENCES public.addresses (id) ON DELETE CASCADE, + address BIGINT NOT NULL REFERENCES public.addresses (id) ON DELETE CASCADE, topics BYTEA[], data BYTEA, block_number BIGINT, @@ -13,9 +12,26 @@ CREATE TABLE public.event_logs tx_index INTEGER, log_index INTEGER, raw JSONB, - transformed BOOL NOT NULL DEFAULT FALSE, + transformed BOOL NOT NULL DEFAULT FALSE, + created TIMESTAMP NOT NULL DEFAULT NOW(), + updated TIMESTAMP NOT NULL DEFAULT NOW(), UNIQUE (header_id, tx_index, log_index) ); +-- +goose StatementBegin +CREATE FUNCTION set_event_log_updated() RETURNS TRIGGER AS +$$ +BEGIN + NEW.updated = NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; +-- +goose StatementEnd + +CREATE TRIGGER event_log_updated + BEFORE UPDATE + ON public.event_logs + FOR EACH ROW +EXECUTE PROCEDURE set_event_log_updated(); CREATE INDEX event_logs_address ON event_logs (address); @@ -23,11 +39,10 @@ CREATE INDEX event_logs_transaction ON event_logs (tx_hash); CREATE INDEX event_logs_untransformed ON event_logs (transformed) - WHERE transformed is false; + WHERE transformed = false; -- +goose Down --- SQL in this section is executed when the migration is rolled back. -DROP INDEX event_logs_transaction; -DROP INDEX event_logs_address; -DROP INDEX event_logs_untransformed; +DROP TRIGGER event_log_updated ON public.event_logs; +DROP FUNCTION set_event_log_updated(); + DROP TABLE event_logs; \ No newline at end of file diff --git a/db/migrations/00009_create_watched_logs_table.sql b/db/migrations/00009_create_watched_logs_table.sql index 4268a68a1..0e463e27f 100644 --- a/db/migrations/00009_create_watched_logs_table.sql +++ b/db/migrations/00009_create_watched_logs_table.sql @@ -1,5 +1,4 @@ -- +goose Up --- SQL in this section is executed when the migration is applied. CREATE TABLE public.watched_logs ( id SERIAL PRIMARY KEY, @@ -8,5 +7,4 @@ CREATE TABLE public.watched_logs ); -- +goose Down --- SQL in this section is executed when the migration is rolled back. DROP TABLE public.watched_logs; diff --git a/db/migrations/20200305202405_create_get_or_create_header_function.sql b/db/migrations/00010_create_get_or_create_header_function.sql similarity index 91% rename from db/migrations/20200305202405_create_get_or_create_header_function.sql rename to db/migrations/00010_create_get_or_create_header_function.sql index 76a35eb60..b995cfd87 100644 --- a/db/migrations/20200305202405_create_get_or_create_header_function.sql +++ b/db/migrations/00010_create_get_or_create_header_function.sql @@ -1,6 +1,4 @@ -- +goose Up --- SQL in this section is executed when the migration is applied. - -- +goose StatementBegin CREATE OR REPLACE FUNCTION public.get_or_create_header(block_number BIGINT, hash VARCHAR(66), raw JSONB, block_timestamp NUMERIC, eth_node_id INTEGER) RETURNS INTEGER AS @@ -47,7 +45,8 @@ $$ LANGUAGE plpgsql; -- +goose StatementEnd +COMMENT ON FUNCTION public.get_or_create_header(block_number BIGINT, hash VARCHAR, raw JSONB, block_timestamp NUMERIC, eth_node_id INTEGER) + IS E'@omit'; -- +goose Down --- SQL in this section is executed when the migration is rolled back. DROP FUNCTION public.get_or_create_header(block_number BIGINT, hash VARCHAR, raw JSONB, block_timestamp NUMERIC, eth_node_id INTEGER); diff --git a/db/migrations/20200406181508_create_create_back_filled_diff_function.sql b/db/migrations/00011_create_create_back_filled_diff_function.sql similarity index 56% rename from db/migrations/20200406181508_create_create_back_filled_diff_function.sql rename to db/migrations/00011_create_create_back_filled_diff_function.sql index aadb6a238..1acfe2706 100644 --- a/db/migrations/20200406181508_create_create_back_filled_diff_function.sql +++ b/db/migrations/00011_create_create_back_filled_diff_function.sql @@ -1,14 +1,15 @@ -- +goose Up --- SQL in this section is executed when the migration is applied. -- +goose StatementBegin -CREATE OR REPLACE FUNCTION public.create_back_filled_diff(block_height BIGINT, block_hash BYTEA, hashed_address BYTEA, - storage_key BYTEA, storage_value BYTEA) RETURNS VOID AS +CREATE OR REPLACE FUNCTION public.create_back_filled_diff(block_height BIGINT, block_hash BYTEA, address BYTEA, + storage_key BYTEA, storage_value BYTEA, + eth_node_id INTEGER) RETURNS VOID AS $$ DECLARE last_storage_value BYTEA := ( SELECT storage_diff.storage_value FROM public.storage_diff - WHERE storage_diff.hashed_address = create_back_filled_diff.hashed_address + WHERE storage_diff.block_height <= create_back_filled_diff.block_height + AND storage_diff.address = create_back_filled_diff.address AND storage_diff.storage_key = create_back_filled_diff.storage_key ORDER BY storage_diff.block_height DESC LIMIT 1 @@ -25,11 +26,11 @@ BEGIN RETURN; END IF; - INSERT INTO public.storage_diff (block_height, block_hash, hashed_address, storage_key, storage_value, - from_backfill) + INSERT INTO public.storage_diff (block_height, block_hash, address, storage_key, storage_value, + eth_node_id, from_backfill) VALUES (create_back_filled_diff.block_height, create_back_filled_diff.block_hash, - create_back_filled_diff.hashed_address, create_back_filled_diff.storage_key, - create_back_filled_diff.storage_value, true) + create_back_filled_diff.address, create_back_filled_diff.storage_key, + create_back_filled_diff.storage_value, create_back_filled_diff.eth_node_id, true) ON CONFLICT DO NOTHING; RETURN; @@ -38,9 +39,8 @@ $$ LANGUAGE plpgsql; -- +goose StatementEnd -COMMENT ON FUNCTION public.create_back_filled_diff(block_height BIGINT, block_hash BYTEA, hashed_address BYTEA, storage_key BYTEA, storage_value BYTEA) +COMMENT ON FUNCTION public.create_back_filled_diff(block_height BIGINT, block_hash BYTEA, address BYTEA, storage_key BYTEA, storage_value BYTEA, eth_node_id INTEGER) IS E'@omit'; -- +goose Down --- SQL in this section is executed when the migration is rolled back. -DROP FUNCTION public.create_back_filled_diff(block_height BIGINT, block_hash BYTEA, hashed_address BYTEA, storage_key BYTEA, storage_value BYTEA); +DROP FUNCTION public.create_back_filled_diff(block_height BIGINT, block_hash BYTEA, address BYTEA, storage_key BYTEA, storage_value BYTEA, eth_node_id INTEGER); diff --git a/db/migrations/20200414084022_add_storage_diff_checked_index.sql b/db/migrations/20200414084022_add_storage_diff_checked_index.sql deleted file mode 100644 index 615382de0..000000000 --- a/db/migrations/20200414084022_add_storage_diff_checked_index.sql +++ /dev/null @@ -1,6 +0,0 @@ --- +goose Up -CREATE INDEX storage_diff_checked_index ON public.storage_diff (checked) WHERE checked is false; - - --- +goose Down -DROP INDEX storage_diff_checked_index; \ No newline at end of file diff --git a/db/migrations/20200415113349_update_create_back_filled_diff_function.sql b/db/migrations/20200415113349_update_create_back_filled_diff_function.sql deleted file mode 100644 index 2d613707c..000000000 --- a/db/migrations/20200415113349_update_create_back_filled_diff_function.sql +++ /dev/null @@ -1,78 +0,0 @@ --- +goose Up --- +goose StatementBegin -CREATE OR REPLACE FUNCTION public.create_back_filled_diff(block_height BIGINT, block_hash BYTEA, hashed_address BYTEA, - storage_key BYTEA, storage_value BYTEA) RETURNS VOID AS -$$ -DECLARE - last_storage_value BYTEA := ( - SELECT storage_diff.storage_value - FROM public.storage_diff - WHERE storage_diff.block_height <= create_back_filled_diff.block_height - AND storage_diff.hashed_address = create_back_filled_diff.hashed_address - AND storage_diff.storage_key = create_back_filled_diff.storage_key - ORDER BY storage_diff.block_height DESC - LIMIT 1 - ); - empty_storage_value BYTEA := ( - SELECT '\x0000000000000000000000000000000000000000000000000000000000000000'::BYTEA - ); -BEGIN - IF last_storage_value = create_back_filled_diff.storage_value THEN - RETURN; - END IF; - - IF last_storage_value is null and create_back_filled_diff.storage_value = empty_storage_value THEN - RETURN; - END IF; - - INSERT INTO public.storage_diff (block_height, block_hash, hashed_address, storage_key, storage_value, - from_backfill) - VALUES (create_back_filled_diff.block_height, create_back_filled_diff.block_hash, - create_back_filled_diff.hashed_address, create_back_filled_diff.storage_key, - create_back_filled_diff.storage_value, true) - ON CONFLICT DO NOTHING; - - RETURN; -END -$$ - LANGUAGE plpgsql; --- +goose StatementEnd - --- +goose Down --- +goose StatementBegin -CREATE OR REPLACE FUNCTION public.create_back_filled_diff(block_height BIGINT, block_hash BYTEA, hashed_address BYTEA, - storage_key BYTEA, storage_value BYTEA) RETURNS VOID AS -$$ -DECLARE - last_storage_value BYTEA := ( - SELECT storage_diff.storage_value - FROM public.storage_diff - WHERE storage_diff.hashed_address = create_back_filled_diff.hashed_address - AND storage_diff.storage_key = create_back_filled_diff.storage_key - ORDER BY storage_diff.block_height DESC - LIMIT 1 - ); - empty_storage_value BYTEA := ( - SELECT '\x0000000000000000000000000000000000000000000000000000000000000000'::BYTEA - ); -BEGIN - IF last_storage_value = create_back_filled_diff.storage_value THEN - RETURN; - END IF; - - IF last_storage_value is null and create_back_filled_diff.storage_value = empty_storage_value THEN - RETURN; - END IF; - - INSERT INTO public.storage_diff (block_height, block_hash, hashed_address, storage_key, storage_value, - from_backfill) - VALUES (create_back_filled_diff.block_height, create_back_filled_diff.block_hash, - create_back_filled_diff.hashed_address, create_back_filled_diff.storage_key, - create_back_filled_diff.storage_value, true) - ON CONFLICT DO NOTHING; - - RETURN; -END -$$ - LANGUAGE plpgsql; --- +goose StatementEnd \ No newline at end of file diff --git a/db/migrations/20200603213514_add_headers_block_timestamp_index.sql b/db/migrations/20200603213514_add_headers_block_timestamp_index.sql deleted file mode 100644 index afa9d60a8..000000000 --- a/db/migrations/20200603213514_add_headers_block_timestamp_index.sql +++ /dev/null @@ -1,6 +0,0 @@ --- +goose Up -CREATE INDEX headers_block_timestamp_index ON public.headers (block_timestamp); - - --- +goose Down -DROP INDEX headers_block_timestamp_index; \ No newline at end of file diff --git a/db/schema.sql b/db/schema.sql index b42b2a761..9ec16edd3 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -17,10 +17,37 @@ SET client_min_messages = warning; SET row_security = off; -- --- Name: create_back_filled_diff(bigint, bytea, bytea, bytea, bytea); Type: FUNCTION; Schema: public; Owner: - +-- Name: public; Type: SCHEMA; Schema: -; Owner: - -- -CREATE FUNCTION public.create_back_filled_diff(block_height bigint, block_hash bytea, hashed_address bytea, storage_key bytea, storage_value bytea) RETURNS void +CREATE SCHEMA public; + + +-- +-- Name: SCHEMA public; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON SCHEMA public IS 'standard public schema'; + + +-- +-- Name: diff_status; Type: TYPE; Schema: public; Owner: - +-- + +CREATE TYPE public.diff_status AS ENUM ( + 'new', + 'transformed', + 'unrecognized', + 'noncanonical', + 'unwatched' +); + + +-- +-- Name: create_back_filled_diff(bigint, bytea, bytea, bytea, bytea, integer); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.create_back_filled_diff(block_height bigint, block_hash bytea, address bytea, storage_key bytea, storage_value bytea, eth_node_id integer) RETURNS void LANGUAGE plpgsql AS $$ DECLARE @@ -28,7 +55,7 @@ DECLARE SELECT storage_diff.storage_value FROM public.storage_diff WHERE storage_diff.block_height <= create_back_filled_diff.block_height - AND storage_diff.hashed_address = create_back_filled_diff.hashed_address + AND storage_diff.address = create_back_filled_diff.address AND storage_diff.storage_key = create_back_filled_diff.storage_key ORDER BY storage_diff.block_height DESC LIMIT 1 @@ -40,28 +67,25 @@ BEGIN IF last_storage_value = create_back_filled_diff.storage_value THEN RETURN; END IF; - IF last_storage_value is null and create_back_filled_diff.storage_value = empty_storage_value THEN RETURN; END IF; - - INSERT INTO public.storage_diff (block_height, block_hash, hashed_address, storage_key, storage_value, - from_backfill) + INSERT INTO public.storage_diff (block_height, block_hash, address, storage_key, storage_value, + eth_node_id, from_backfill) VALUES (create_back_filled_diff.block_height, create_back_filled_diff.block_hash, - create_back_filled_diff.hashed_address, create_back_filled_diff.storage_key, - create_back_filled_diff.storage_value, true) + create_back_filled_diff.address, create_back_filled_diff.storage_key, + create_back_filled_diff.storage_value, create_back_filled_diff.eth_node_id, true) ON CONFLICT DO NOTHING; - RETURN; END $$; -- --- Name: FUNCTION create_back_filled_diff(block_height bigint, block_hash bytea, hashed_address bytea, storage_key bytea, storage_value bytea); Type: COMMENT; Schema: public; Owner: - +-- Name: FUNCTION create_back_filled_diff(block_height bigint, block_hash bytea, address bytea, storage_key bytea, storage_value bytea, eth_node_id integer); Type: COMMENT; Schema: public; Owner: - -- -COMMENT ON FUNCTION public.create_back_filled_diff(block_height bigint, block_hash bytea, hashed_address bytea, storage_key bytea, storage_value bytea) IS '@omit'; +COMMENT ON FUNCTION public.create_back_filled_diff(block_height bigint, block_hash bytea, address bytea, storage_key bytea, storage_value bytea, eth_node_id integer) IS '@omit'; -- @@ -93,25 +117,42 @@ BEGIN IF matching_header_id != 0 THEN RETURN matching_header_id; END IF; - IF nonmatching_header_id != 0 AND block_number <= max_block_number - 15 THEN RETURN nonmatching_header_id; END IF; - IF nonmatching_header_id != 0 AND block_number > max_block_number - 15 THEN DELETE FROM public.headers WHERE id = nonmatching_header_id; END IF; - INSERT INTO public.headers (hash, block_number, raw, block_timestamp, eth_node_id) VALUES (get_or_create_header.hash, get_or_create_header.block_number, get_or_create_header.raw, get_or_create_header.block_timestamp, get_or_create_header.eth_node_id) RETURNING id INTO inserted_header_id; - RETURN inserted_header_id; END $$; +-- +-- Name: FUNCTION get_or_create_header(block_number bigint, hash character varying, raw jsonb, block_timestamp numeric, eth_node_id integer); Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON FUNCTION public.get_or_create_header(block_number bigint, hash character varying, raw jsonb, block_timestamp numeric, eth_node_id integer) IS '@omit'; + + +-- +-- Name: set_event_log_updated(); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.set_event_log_updated() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN + NEW.updated = NOW(); + RETURN NEW; +END; +$$; + + -- -- Name: set_header_updated(); Type: FUNCTION; Schema: public; Owner: - -- @@ -126,6 +167,48 @@ END; $$; +-- +-- Name: set_receipt_updated(); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.set_receipt_updated() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN + NEW.updated = NOW(); + RETURN NEW; +END; +$$; + + +-- +-- Name: set_storage_updated(); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.set_storage_updated() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN + NEW.updated = NOW(); + RETURN NEW; +END; +$$; + + +-- +-- Name: set_transaction_updated(); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.set_transaction_updated() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN + NEW.updated = NOW(); + RETURN NEW; +END; +$$; + + SET default_tablespace = ''; SET default_with_oids = false; @@ -135,9 +218,8 @@ SET default_with_oids = false; -- CREATE TABLE public.addresses ( - id integer NOT NULL, - address character varying(42), - hashed_address character varying(66) + id bigint NOT NULL, + address character varying(42) ); @@ -146,7 +228,6 @@ CREATE TABLE public.addresses ( -- CREATE SEQUENCE public.addresses_id_seq - AS integer START WITH 1 INCREMENT BY 1 NO MINVALUE @@ -229,9 +310,9 @@ ALTER SEQUENCE public.eth_nodes_id_seq OWNED BY public.eth_nodes.id; -- CREATE TABLE public.event_logs ( - id integer NOT NULL, + id bigint NOT NULL, header_id integer NOT NULL, - address integer NOT NULL, + address bigint NOT NULL, topics bytea[], data bytea, block_number bigint, @@ -240,7 +321,9 @@ CREATE TABLE public.event_logs ( tx_index integer, log_index integer, raw jsonb, - transformed boolean DEFAULT false NOT NULL + transformed boolean DEFAULT false NOT NULL, + created timestamp without time zone DEFAULT now() NOT NULL, + updated timestamp without time zone DEFAULT now() NOT NULL ); @@ -249,7 +332,6 @@ CREATE TABLE public.event_logs ( -- CREATE SEQUENCE public.event_logs_id_seq - AS integer START WITH 1 INCREMENT BY 1 NO MINVALUE @@ -306,7 +388,6 @@ CREATE TABLE public.headers ( block_number bigint NOT NULL, raw jsonb, block_timestamp numeric, - check_count integer DEFAULT 0 NOT NULL, eth_node_id integer NOT NULL, created timestamp without time zone DEFAULT now() NOT NULL, updated timestamp without time zone DEFAULT now() NOT NULL @@ -341,13 +422,15 @@ CREATE TABLE public.receipts ( id integer NOT NULL, transaction_id integer NOT NULL, header_id integer NOT NULL, - contract_address_id integer NOT NULL, + contract_address_id bigint NOT NULL, cumulative_gas_used numeric, gas_used numeric, state_root character varying(66), status integer, tx_hash character varying(66), - rlp bytea + rlp bytea, + created timestamp without time zone DEFAULT now() NOT NULL, + updated timestamp without time zone DEFAULT now() NOT NULL ); @@ -377,13 +460,16 @@ ALTER SEQUENCE public.receipts_id_seq OWNED BY public.receipts.id; CREATE TABLE public.storage_diff ( id bigint NOT NULL, + address bytea, block_height bigint, block_hash bytea, - hashed_address bytea, storage_key bytea, storage_value bytea, - checked boolean DEFAULT false NOT NULL, - from_backfill boolean DEFAULT false NOT NULL + eth_node_id integer NOT NULL, + status public.diff_status DEFAULT 'new'::public.diff_status NOT NULL, + from_backfill boolean DEFAULT false NOT NULL, + created timestamp without time zone DEFAULT now() NOT NULL, + updated timestamp without time zone DEFAULT now() NOT NULL ); @@ -422,7 +508,9 @@ CREATE TABLE public.transactions ( tx_from character varying(44), tx_index integer, tx_to character varying(44), - value numeric + value numeric, + created timestamp without time zone DEFAULT now() NOT NULL, + updated timestamp without time zone DEFAULT now() NOT NULL ); @@ -652,11 +740,11 @@ ALTER TABLE ONLY public.receipts -- --- Name: storage_diff storage_diff_block_height_block_hash_hashed_address_storage_key; Type: CONSTRAINT; Schema: public; Owner: - +-- Name: storage_diff storage_diff_block_height_block_hash_address_storage_key_st_key; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY public.storage_diff - ADD CONSTRAINT storage_diff_block_height_block_hash_hashed_address_storage_key UNIQUE (block_height, block_hash, hashed_address, storage_key, storage_value); + ADD CONSTRAINT storage_diff_block_height_block_hash_address_storage_key_st_key UNIQUE (block_height, block_hash, address, storage_key, storage_value); -- @@ -709,7 +797,7 @@ CREATE INDEX event_logs_transaction ON public.event_logs USING btree (tx_hash); -- Name: event_logs_untransformed; Type: INDEX; Schema: public; Owner: - -- -CREATE INDEX event_logs_untransformed ON public.event_logs USING btree (transformed) WHERE (transformed IS FALSE); +CREATE INDEX event_logs_untransformed ON public.event_logs USING btree (transformed) WHERE (transformed = false); -- @@ -726,13 +814,6 @@ CREATE INDEX headers_block_number ON public.headers USING btree (block_number); CREATE INDEX headers_block_timestamp_index ON public.headers USING btree (block_timestamp); --- --- Name: headers_check_count; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX headers_check_count ON public.headers USING btree (check_count); - - -- -- Name: headers_eth_node; Type: INDEX; Schema: public; Owner: - -- @@ -755,10 +836,24 @@ CREATE INDEX receipts_transaction ON public.receipts USING btree (transaction_id -- --- Name: storage_diff_checked_index; Type: INDEX; Schema: public; Owner: - +-- Name: storage_diff_eth_node; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX storage_diff_eth_node ON public.storage_diff USING btree (eth_node_id); + + +-- +-- Name: storage_diff_new_status_index; Type: INDEX; Schema: public; Owner: - -- -CREATE INDEX storage_diff_checked_index ON public.storage_diff USING btree (checked) WHERE (checked IS FALSE); +CREATE INDEX storage_diff_new_status_index ON public.storage_diff USING btree (status) WHERE (status = 'new'::public.diff_status); + + +-- +-- Name: storage_diff_unrecognized_status_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX storage_diff_unrecognized_status_index ON public.storage_diff USING btree (status) WHERE (status = 'unrecognized'::public.diff_status); -- @@ -768,6 +863,13 @@ CREATE INDEX storage_diff_checked_index ON public.storage_diff USING btree (chec CREATE INDEX transactions_header ON public.transactions USING btree (header_id); +-- +-- Name: event_logs event_log_updated; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER event_log_updated BEFORE UPDATE ON public.event_logs FOR EACH ROW EXECUTE PROCEDURE public.set_event_log_updated(); + + -- -- Name: headers header_updated; Type: TRIGGER; Schema: public; Owner: - -- @@ -775,6 +877,27 @@ CREATE INDEX transactions_header ON public.transactions USING btree (header_id); CREATE TRIGGER header_updated BEFORE UPDATE ON public.headers FOR EACH ROW EXECUTE PROCEDURE public.set_header_updated(); +-- +-- Name: receipts receipt_updated; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER receipt_updated BEFORE UPDATE ON public.receipts FOR EACH ROW EXECUTE PROCEDURE public.set_receipt_updated(); + + +-- +-- Name: storage_diff storage_updated; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER storage_updated BEFORE UPDATE ON public.storage_diff FOR EACH ROW EXECUTE PROCEDURE public.set_storage_updated(); + + +-- +-- Name: transactions transaction_updated; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER transaction_updated BEFORE UPDATE ON public.transactions FOR EACH ROW EXECUTE PROCEDURE public.set_transaction_updated(); + + -- -- Name: checked_headers checked_headers_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- @@ -839,6 +962,14 @@ ALTER TABLE ONLY public.receipts ADD CONSTRAINT receipts_transaction_id_fkey FOREIGN KEY (transaction_id) REFERENCES public.transactions(id) ON DELETE CASCADE; +-- +-- Name: storage_diff storage_diff_eth_node_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.storage_diff + ADD CONSTRAINT storage_diff_eth_node_id_fkey FOREIGN KEY (eth_node_id) REFERENCES public.eth_nodes(id) ON DELETE CASCADE; + + -- -- Name: transactions transactions_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- diff --git a/dockerfiles/README.md b/dockerfiles/README.md index 4e9ea585d..8f97eecfb 100644 --- a/dockerfiles/README.md +++ b/dockerfiles/README.md @@ -36,11 +36,16 @@ From project root directory: docker build -f dockerfiles/header_sync/Dockerfile . -t header_sync:latest ``` +You can also use the `dockerbuild` make task for any image in dockerfiles with `make dockerbuild IMAGE=header_sync` + + ### Run ``` docker run -e DATABASE_USER=user -e DATABASE_PASSWORD=password -e DATABASE_HOSTNAME=host -e DATABASE_PORT=port -e DATABASE_NAME=name -e STARTING_BLOCK_NUMBER=0 -e CLIENT_IPCPATH=path -it header_sync:latest ``` +`header_sync` had a make task to run it `make header_sync CLIENT_IPCPATH=path NAME=database_name`. It's meant for development, not production. + ## resetHeaderCheckCount Dockerfile for resetting the `headers.check_count` to zero in the database for the given header, so that the execute command will transform the associated events. This is useful in case an event log is missing. diff --git a/dockerfiles/extract_diffs/Dockerfile b/dockerfiles/extract_diffs/Dockerfile deleted file mode 100644 index c9ea7d2cc..000000000 --- a/dockerfiles/extract_diffs/Dockerfile +++ /dev/null @@ -1,48 +0,0 @@ -FROM golang:alpine as builder - -RUN apk --update --no-cache add g++ git linux-headers - -# Build migration tool -WORKDIR /go -RUN GO111MODULE=auto go get -u -d github.com/pressly/goose/cmd/goose -WORKDIR /go/src/github.com/pressly/goose/cmd/goose -RUN GO111MODULE=auto go build -a -ldflags '-s' -tags='no_mysql no_sqlite' -o goose - -ENV GO111MODULE on - -WORKDIR /vulcanizedb -COPY . . -RUN go build - - -FROM golang:alpine - -WORKDIR /app - -# add certificates for node requests via https -# bash for wait-for-it.sh -RUN apk update \ - && apk upgrade \ - && apk add --no-cache \ - ca-certificates \ - bash \ - && update-ca-certificates 2>/dev/null || true - -# Direct logs to stdout for docker log driver -RUN ln -sf /dev/stdout /app/vulcanizedb.log - -# setup environment -ENV GO111MODULE on - -# add required files -COPY --from=builder /vulcanizedb/vulcanizedb . -COPY --from=builder /vulcanizedb/dockerfiles/extract_diffs/startup_script.sh . -COPY --from=builder /vulcanizedb/db/migrations/* db/migrations/ -COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose -# needed for waiting until postgres is ready before starting from docker-compose -COPY --from=builder /vulcanizedb/dockerfiles/wait-for-it.sh . - -HEALTHCHECK CMD test -f /tmp/connection - -# need to execute with a shell to access env variables -CMD ["./startup_script.sh"] \ No newline at end of file diff --git a/dockerfiles/extract_diffs/startup_script.sh b/dockerfiles/extract_diffs/startup_script.sh deleted file mode 100755 index 87b52adf7..000000000 --- a/dockerfiles/extract_diffs/startup_script.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/sh -# Starts the extractDiffs command - -# Verify required args present -MISSING_VAR_MESSAGE=" is required and no value was given" - -function testDatabaseVariables() { - for a in DATABASE_NAME DATABASE_HOSTNAME DATABASE_PORT DATABASE_USER DATABASE_PASSWORD - do - eval arg="$"$a - test $arg - if [ $? -ne 0 ]; then - echo $a $MISSING_VAR_MESSAGE - exit 1 - fi - done -} - -if test -z "$VDB_PG_CONNECT"; then - # Exits if the variable tests fail - testDatabaseVariables - if [ $? -ne 0 ]; then - exit 1 - fi - - # Construct the connection string for postgres - VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable -fi - -# Run the DB migrations -echo "Connecting with: $VDB_PG_CONNECT" -./goose -dir db/migrations postgres "$VDB_PG_CONNECT" up - -if [ $? -ne 0 ]; then - echo "Could not run migrations. Are the database details correct?" - exit 1 -fi - -# Run extractDiffs -echo "Running extractDiffs..." -./vulcanizedb extractDiffs \ No newline at end of file diff --git a/dockerfiles/header_sync/Dockerfile b/dockerfiles/header_sync/Dockerfile index 2095028b6..77b9b5fe2 100644 --- a/dockerfiles/header_sync/Dockerfile +++ b/dockerfiles/header_sync/Dockerfile @@ -2,18 +2,12 @@ FROM golang:alpine as builder RUN apk --update --no-cache add g++ git linux-headers -# Build migration tool -WORKDIR /go -RUN GO111MODULE=auto go get -u -d github.com/pressly/goose/cmd/goose -WORKDIR /go/src/github.com/pressly/goose/cmd/goose -RUN GO111MODULE=auto go build -a -ldflags '-s' -tags='no_mysql no_sqlite' -o goose - ENV GO111MODULE on WORKDIR /vulcanizedb COPY . . RUN go build - +RUN go get -tags='no_mysql no_sqlite3 no_mssql no_redshift' github.com/pressly/goose/cmd/goose@v2.7.0-rc5 FROM golang:alpine @@ -25,20 +19,18 @@ RUN apk update \ && apk upgrade \ && apk add --no-cache \ ca-certificates \ - bash \ + bash \ && update-ca-certificates 2>/dev/null || true -# setup environment -ENV GO111MODULE on - # Direct logs to stdout for docker log driver RUN ln -sf /dev/stdout /app/vulcanizedb.log # add required files COPY --from=builder /vulcanizedb/vulcanizedb . +COPY --from=builder /vulcanizedb/Makefile . COPY --from=builder /vulcanizedb/dockerfiles/header_sync/startup_script.sh . COPY --from=builder /vulcanizedb/db/migrations/* db/migrations/ -COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose +COPY --from=builder /go/bin/goose goose # needed for waiting until postgres is ready before starting from docker-compose COPY --from=builder /vulcanizedb/dockerfiles/wait-for-it.sh . diff --git a/environments/composeAndExecuteAccountTransformer.toml b/environments/composeAndExecuteAccountTransformer.toml index 5b51262a7..9bb644a8c 100644 --- a/environments/composeAndExecuteAccountTransformer.toml +++ b/environments/composeAndExecuteAccountTransformer.toml @@ -8,6 +8,7 @@ [exporter] home = "github.com/makerdao/vulcanizedb" + schema = "public" name = "accountTransformerExporter" save = false transformerNames = [ diff --git a/go.mod b/go.mod index fa9c9b253..e66571eac 100644 --- a/go.mod +++ b/go.mod @@ -1,55 +1,27 @@ module github.com/makerdao/vulcanizedb -go 1.12 +go 1.15 require ( - github.com/allegro/bigcache v1.2.1 // indirect - github.com/apilayer/freegeoip v3.5.0+incompatible // indirect - github.com/aristanetworks/goarista v0.0.0-20191106175434-873d404c7f40 // indirect - github.com/btcsuite/btcd v0.20.1-beta // indirect - github.com/cespare/cp v1.1.1 // indirect github.com/dave/jennifer v1.3.0 - github.com/deckarep/golang-set v1.7.1 // indirect - github.com/docker/docker v1.13.1 // indirect - github.com/edsrzf/mmap-go v1.0.0 // indirect - github.com/elastic/gosigar v0.10.5 // indirect - github.com/ethereum/go-ethereum v1.9.6 - github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect - github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 // indirect - github.com/gorilla/websocket v1.4.1 // indirect - github.com/hashicorp/golang-lru v0.5.3 + github.com/ethereum/go-ethereum v1.9.21 + github.com/hashicorp/golang-lru v0.5.4 github.com/hpcloud/tail v1.0.0 - github.com/huin/goupnp v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/influxdata/influxdb v1.7.9 // indirect - github.com/jackpal/go-nat-pmp v1.0.2 // indirect - github.com/jmoiron/sqlx v0.0.0-20181024163419-82935fac6c1a - github.com/karalabe/usb v0.0.0-20191104083709-911d15fe12a9 // indirect + github.com/jmoiron/sqlx v1.2.0 github.com/lib/pq v1.0.0 - github.com/mattn/go-colorable v0.1.4 // indirect - github.com/mattn/go-isatty v0.0.10 // indirect github.com/mitchellh/go-homedir v1.1.0 - github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect - github.com/olekukonko/tablewriter v0.0.3 // indirect - github.com/onsi/ginkgo v1.10.1 - github.com/onsi/gomega v1.10.0 - github.com/oschwald/maxminddb-golang v1.5.0 // indirect - github.com/pborman/uuid v1.2.0 // indirect - github.com/pressly/goose v2.6.0+incompatible - github.com/prometheus/tsdb v0.10.0 // indirect - github.com/rjeczalik/notify v0.9.2 // indirect - github.com/rs/cors v1.7.0 // indirect + github.com/onsi/ginkgo v1.14.0 + github.com/onsi/gomega v1.10.1 + github.com/pressly/goose v2.7.0-rc5+incompatible github.com/sirupsen/logrus v1.2.0 github.com/spf13/cobra v0.0.3 github.com/spf13/viper v1.3.2 - github.com/status-im/keycard-go v0.0.0-20191119114148-6dd40a46baa0 // indirect - github.com/tyler-smith/go-bip39 v1.0.2 // indirect - golang.org/x/net v0.0.0-20200301022130-244492dfa37a - golang.org/x/sync v0.0.0-20190423024810-112230192c58 - google.golang.org/appengine v1.6.5 // indirect + golang.org/x/net v0.0.0-20200822124328-c89045814202 + google.golang.org/appengine v1.6.6 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 ) -replace github.com/ethereum/go-ethereum => github.com/makerdao/go-ethereum v1.9.11-rc2 +replace github.com/ethereum/go-ethereum => github.com/makerdao/go-ethereum v1.9.21-rc1 replace gopkg.in/urfave/cli.v1 => gopkg.in/urfave/cli.v1 v1.20.0 diff --git a/go.sum b/go.sum index bfe995883..36dd7794c 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,3 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= @@ -14,64 +13,28 @@ github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6L github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= -github.com/Shopify/sarama v1.23.1/go.mod h1:XLH1GYJnLVE0XCr6KdJGVJRTwY30moWNJ4sERjXX6fs= -github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/VictoriaMetrics/fastcache v1.5.3 h1:2odJnXLbFZcoV9KYtQ+7TH1UOq3dn3AssMgieaezkR4= -github.com/VictoriaMetrics/fastcache v1.5.3/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v49daAVERr0H3CuscE= github.com/VictoriaMetrics/fastcache v1.5.7 h1:4y6y0G8PRzszQUYIQHHssv/jgPHAb5qQuuDNdCbyAgw= github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8= -github.com/aead/siphash v1.0.1 h1:FwHfE/T45KPKYuuSAKyyvE+oPWcaQ+CUmFW0bPlM+kg= -github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= -github.com/allegro/bigcache v1.2.1 h1:hg1sY1raCwic3Vnsvje6TT7/pnZba83LeFck5NrFKSc= -github.com/allegro/bigcache v1.2.1/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= -github.com/apilayer/freegeoip v3.5.0+incompatible h1:z1u2gv0/rsSi/HqMDB436AiUROXXim7st5DOg4Ikl4A= -github.com/apilayer/freegeoip v3.5.0+incompatible/go.mod h1:CUfFqErhFhXneJendyQ/rRcuA8kH8JxHvYnbOozmlCU= -github.com/aristanetworks/fsnotify v1.4.2/go.mod h1:D/rtu7LpjYM8tRJphJ0hUBYpjai8SfX+aSNsWDTq/Ks= -github.com/aristanetworks/glog v0.0.0-20180419172825-c15b03b3054f/go.mod h1:KASm+qXFKs/xjSoWn30NrWBBvdTTQq+UjkhjEJHfSFA= +github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847 h1:rtI0fD4oG/8eVokGVPYJEW1F88p1ZNgXiEIs9thEE4A= github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= -github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015 h1:7ABPr1+uJdqESAdlVevnc/2FJGiC/K3uMg1JiELeF+0= -github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= -github.com/aristanetworks/goarista v0.0.0-20191106175434-873d404c7f40 h1:ZdRuixFqR3mfx4FHzclG3COrRgWrYq0VhNgIoYoObcM= -github.com/aristanetworks/goarista v0.0.0-20191106175434-873d404c7f40/go.mod h1:Z4RTxGAuYhPzcq8+EdRM+R8M48Ssle2TsWtwRKa+vns= -github.com/aristanetworks/splunk-hec-go v0.3.3/go.mod h1:1VHO9r17b0K7WmOlLb9nTk/2YanvOEnLMUgsFrxBROc= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6 h1:Eey/GGQ/E5Xp1P2Lyx1qj007hLZfbi0+CoVeJruGCtI= github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= -github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= -github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= -github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo= -github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= -github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d h1:yJzD/yFppdVCf6ApMkVy8cUxV0XrxdP9rVf6D87/Mng= -github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= -github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd h1:R/opQEbFEy9JGkIguV40SvRY1uliPX8ifOvi6ICsFCw= -github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= -github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd h1:qdGvebPBDuYDPGi1WCPjy1tGyMpmDK8IEapSsszn7HE= -github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= -github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723 h1:ZA/jbKoGcVAnER6pCHPEkGdZOV7U1oLUedErBHCUMs0= -github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= -github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc= -github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= -github.com/btcsuite/winsvc v1.0.0 h1:J9B4L7e3oqhXOcm+2IuNApwzQec85lE+QaikUcCs+dk= -github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= -github.com/cespare/cp v1.1.1 h1:nCb6ZLdB7NRaqsm91JtQTAme2SKJzXVsdPIPkyJr1MU= -github.com/cespare/cp v1.1.1/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18/go.mod h1:HD5P3vAIAh+Y2GAxg0PrPN1P8WkepXGpjbUPDHJqqKM= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= @@ -79,163 +42,108 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/dave/jennifer v1.3.0 h1:p3tl41zjjCZTNBytMwrUuiAnherNUZktlhPTKoF/sEk= github.com/dave/jennifer v1.3.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= -github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea h1:j4317fAZh7X6GqbFowYdYdI0L9bwxL07jyPZIdepyZ0= github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= -github.com/deckarep/golang-set v1.7.1 h1:SCQV0S6gTtp6itiFrTqI+pfmJ4LN85S1YzhDf9rTHJQ= -github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo= -github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/dop251/goja v0.0.0-20200106141417-aaec0e7bde29/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= -github.com/dop251/goja v0.0.0-20200219165308-d1232e640a87/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= -github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= -github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= +github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c h1:JHHhtb9XWJrGNMcrVP6vyzO4dusgi/HnceHTgxSejUM= github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= -github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/elastic/gosigar v0.8.1-0.20180330100440-37f05ff46ffa/go.mod h1:cdorVVzy1fhmEqmtgqkoE3bYtCfSCkVyjTyCIo22xvs= -github.com/elastic/gosigar v0.10.4 h1:6jfw75dsoflhBMRdO6QPzQUgLqUYTsQQQRkkcsHsuPo= -github.com/elastic/gosigar v0.10.4/go.mod h1:cdorVVzy1fhmEqmtgqkoE3bYtCfSCkVyjTyCIo22xvs= -github.com/elastic/gosigar v0.10.5 h1:GzPQ+78RaAb4J63unidA/JavQRKrB6s8IOzN6Ib59jo= -github.com/elastic/gosigar v0.10.5/go.mod h1:cdorVVzy1fhmEqmtgqkoE3bYtCfSCkVyjTyCIo22xvs= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc h1:jtW8jbpkO4YirRSyepBOH8E+2HEw6/hKkBvFPwhUN8c= github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= -github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= -github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/garyburd/redigo v1.6.0/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= -github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 h1:f6D9Hr8xV8uYKlyuj8XIruxlh9WjVjdh1gIicAS7ays= -github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0 h1:8HUsc87TaSWLKwrnumgC8/YconD2fJQsRJAsWaPg2ic= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= -github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2-0.20190517061210-b285ee9cfc6c/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/golang/snappy v0.0.2-0.20200707131729-196ae77b8a26 h1:lMm2hD9Fy0ynom5+85/pbdkiYcBqM1JWmhpAXLmy0fw= +github.com/golang/snappy v0.0.2-0.20200707131729-196ae77b8a26/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989 h1:giknQ4mEuDFmmHSrGcbargOuLHQGtywqo4mheITex54= github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= -github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/graph-gophers/graphql-go v0.0.0-20191024035216-0a9cfbec35a1 h1:jV0CRazQbnsAGKT1z8BjMvouE2pypynEjx/o7eHbkFM= -github.com/graph-gophers/graphql-go v0.0.0-20191024035216-0a9cfbec35a1/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/golang-lru v0.0.0-20160813221303-0a025b7e63ad/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk= -github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/howeyc/fsnotify v0.9.0 h1:0gtV5JmOKH4A8SsFxG2BczSeXWWPvcMT0euZt5gDAxY= -github.com/howeyc/fsnotify v0.9.0/go.mod h1:41HzSPxBGeFRQKEEwgh49TRw/nKBsYZ2cF1OzPjSJsA= +github.com/holiman/uint256 v1.1.1 h1:4JywC80b+/hSfljFlEBLHrrh+CIONLDz9NuFl0af4Mw= +github.com/holiman/uint256 v1.1.1/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huin/goupnp v0.0.0-20161224104101-679507af18f3/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag= github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= -github.com/influxdata/influxdb v1.7.9 h1:uSeBTNO4rBkbp1Be5FKRsAmglM9nlx25TzVQRQt1An4= -github.com/influxdata/influxdb v1.7.9/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= -github.com/influxdata/influxdb1-client v0.0.0-20190809212627-fc22c7df067e/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= -github.com/jackpal/go-nat-pmp v1.0.1 h1:i0LektDkO1QlrTm/cSuP+PyBCDnYvjPLGl4LdWEMiaA= -github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= -github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= -github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= -github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= -github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89 h1:12K8AlpT0/6QUXSfV0yi4Q0jkbq8NDtIKFtF61AoqV0= -github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmoiron/sqlx v0.0.0-20181024163419-82935fac6c1a h1:Jyg5PpIc1nLGrNDM5blVkiSySmRhaD/IiXkvaHzBYnw= -github.com/jmoiron/sqlx v0.0.0-20181024163419-82935fac6c1a/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= -github.com/jrick/logrotate v1.0.0 h1:lQ1bL/n9mBNeIXoTUoYRlK4dHuNJVofX9oWqBtPnSzI= -github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/karalabe/usb v0.0.0-20190819132248-550797b1cad8 h1:VhnqxaTIudc9IWKx8uXRLnpdSb9noCEj+vHacjmhp68= -github.com/karalabe/usb v0.0.0-20190819132248-550797b1cad8/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356 h1:I/yrLt2WilKxlQKCM52clh5rGzTKpVctGT1lH4Dc8Jw= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= -github.com/karalabe/usb v0.0.0-20191104083709-911d15fe12a9 h1:ZHuwnjpP8LsVsUYqTqeVAI+GfDfJ6UNPrExZF+vX/DQ= -github.com/karalabe/usb v0.0.0-20191104083709-911d15fe12a9/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= -github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23 h1:FOOIBWrEkLgmlgGfMuZT83xIwfPDxEI2OHu6xUmJMFE= -github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/reedsolomon v1.9.2/go.mod h1:CwCi+NUr9pqSVktrkN+Ondf06rkhYZ/pcNv7fu+8Un4= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/makerdao/go-ethereum v1.5.10-0.20200508151415-a332a1649dff h1:oH6AkDyruMAXlE1WELwaAc5iEnRBsQfQALS+gGzDHIM= -github.com/makerdao/go-ethereum v1.5.10-0.20200508151415-a332a1649dff/go.mod h1:qeDFnA9rW6VM6KC91PIlcpXMr8V+KY4udYi/RSZtPT4= -github.com/makerdao/go-ethereum v1.9.11-rc2 h1:ZjxmFdkar8P43EkK17ni8y3vXxHHFZjOd12HqNalf8M= -github.com/makerdao/go-ethereum v1.9.11-rc2/go.mod h1:7oC0Ni6dosMv5pxMigm6s0hN8g4haJMBnqmmo0D9YfQ= +github.com/makerdao/go-ethereum v1.9.21-rc1 h1:izuLZxMHei6UX89Q2Ld1v5PYcuYEhw34/Ncf/e/WPW8= +github.com/makerdao/go-ethereum v1.9.21-rc1/go.mod h1:RXAVzbGrSGmDkDnHymruTAIEjUR3E4TX0EOpaj702sI= +github.com/mattn/go-colorable v0.1.0 h1:v2XXALHHh6zHfYTJ+cSkwtyffnaOyR1MXaA91mTrb8o= github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035 h1:USWjF42jDCSEeikX/G1g40ZWnsPXN5WkZ4jMHZWyBK4= github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= -github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= -github.com/mattn/go-isatty v0.0.10 h1:qxFzApOv4WsAL965uUPIsXzAKCZxN2p9UqdhFS4ZW10= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.6 h1:V2iyH+aX9C5fsYCpK60U8BYIvmhqxuOL3JZcqc1NB7k= -github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -243,84 +151,58 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= -github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c h1:1RHs3tNxjXGHeul8z2t6H2N2TlAqpKe5yryJztRx4Jk= github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/olekukonko/tablewriter v0.0.3 h1:i0LBnzgiChAWHJYTQAZJDOgf8MNxAVYZJ2m63SIDimI= -github.com/olekukonko/tablewriter v0.0.3/go.mod h1:YZeBtGzYYEsCHp2LST/u/0NDwGkRoBtmn1cIWCJiS6M= github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= -github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.10.0 h1:Gwkk+PTu/nfOwNMtUB/mRUv0X7ewW5dO4AERT1ThVKo= -github.com/onsi/gomega v1.10.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= -github.com/openconfig/gnmi v0.0.0-20190823184014-89b2bf29312c/go.mod h1:t+O9It+LKzfOAhKTT5O0ehDix+MTqbtT0T9t+7zzOvc= -github.com/openconfig/reference v0.0.0-20190727015836-8dfd928c9696/go.mod h1:ym2A+zigScwkSEb/cVQB0/ZMpU3rqiH6X7WRRsxgOGw= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/oschwald/maxminddb-golang v1.5.0 h1:rmyoIV6z2/s9TCJedUuDiKht2RN12LWJ1L7iRGtWY64= -github.com/oschwald/maxminddb-golang v1.5.0/go.mod h1:3jhIUymTJ5VREKyIhWm66LJiQt04F0UCDdodShpjWsY= +github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222 h1:goeTyGkArOZIVOMA0dQbyuPWGNQJZGPwPu/QS9GlpnA= github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= -github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= -github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pressly/goose v2.6.0+incompatible h1:3f8zIQ8rfgP9tyI0Hmcs2YNAqUCL1c+diLe3iU8Qd/k= -github.com/pressly/goose v2.6.0+incompatible/go.mod h1:m+QHWCqxR3k8D9l7qfzuC/djtlfzxr34mozWDYEu1z8= +github.com/pressly/goose v2.7.0-rc5+incompatible h1:txvo810iG1P/rafOx31LYDlOyikBK8A/8prKP4j066w= +github.com/pressly/goose v2.7.0-rc5+incompatible/go.mod h1:m+QHWCqxR3k8D9l7qfzuC/djtlfzxr34mozWDYEu1z8= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150 h1:ZeU+auZj1iNzN8iVhff6M38Mfu73FQiJve/GEXYJBjE= github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/prometheus/tsdb v0.10.0 h1:If5rVCMTp6W2SiRAQFlbpJNgVlgMEd+U2GZckwK38ic= -github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4= -github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= -github.com/rjeczalik/notify v0.9.2 h1:MiTWrPj55mNDHEiIX5YUSKefw/+lCQVoAFmD6oQm5w8= -github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa4QEjJeqM= +github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00 h1:8DPul/X0IT/1TNMIxoKLwdemEOBBHDC/K4EB16Cw5WE= github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= -github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= -github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521 h1:3hxavr+IHMsQBrYUPQM5v0CgENFktkkbg1sfpgM3h20= github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubrpZHtQLnOf6EwhN2hEMusxZOhcW9H3UQ= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/shirou/gopsutil v2.20.5+incompatible h1:tYH07UPoQt0OCQdgWWMgYHy3/a9bcxNpBIysykNIP7I= +github.com/shirou/gopsutil v2.20.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spaolacci/murmur3 v1.0.1-0.20190317074736-539464a789e9/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= @@ -333,11 +215,8 @@ github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= -github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48 h1:ju5UTwk5Odtm4trrY+4Ca4RMj5OyXbmVeDAVad2T0Jw= -github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= -github.com/status-im/keycard-go v0.0.0-20191119114148-6dd40a46baa0 h1:5UdlDkkBoPrJfh7zkfoR3X5utJhNs/MCQysK3x0ycgg= -github.com/status-im/keycard-go v0.0.0-20191119114148-6dd40a46baa0/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 h1:gIlAHnH1vJb5vwEjIp5kBj/eu99p/bl0Ay2goiPe5xE= github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw= github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 h1:njlZPzLwU639dk2kqnCPPv+wNjq7Xb6EfUxe/oX0/NM= @@ -349,137 +228,87 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= -github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= -github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d h1:gZZadD8H+fF+n9CmNhYL1Y0dJB+kLOmKd7FbPJLeGHs= -github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= -github.com/templexxx/cpufeat v0.0.0-20180724012125-cef66df7f161/go.mod h1:wM7WEvslTq+iOEAMDLSzhVuOt5BRZ05WirO+b09GHQU= -github.com/templexxx/xor v0.0.0-20181023030647-4e92f724b73b/go.mod h1:5XA7W9S6mni3h5uvOC75dA3m9CCCaS83lltmc0ukdi4= -github.com/tjfoc/gmsm v1.0.1/go.mod h1:XxO4hdhhrzAd+G4CjDqaOkd0hUzmtPR/d3EiBBMn/wc= +github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca h1:Ld/zXl5t4+D69SiV4JoN7kkfvJdOWlPpfxrzxpLMoUk= +github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= +github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZW24CsNJDfeh9Ex6Pm0Rcpc7qrgKBiL44vF4= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= -github.com/tyler-smith/go-bip39 v1.0.2 h1:+t3w+KwLXO6154GNJY+qUtIxLTmFjfUmpguQT1OlOT8= -github.com/tyler-smith/go-bip39 v1.0.2/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/vulcanize/go-ethereum v0.0.0-20190731183759-8e20673bd101 h1:fsHhBzscAwi4u7/F033SFJwTIz+46D8uDWMu2/ZdvzA= -github.com/vulcanize/go-ethereum v0.0.0-20190731183759-8e20673bd101 h1:fsHhBzscAwi4u7/F033SFJwTIz+46D8uDWMu2/ZdvzA= -github.com/vulcanize/go-ethereum v0.0.0-20190731183759-8e20673bd101/go.mod h1:9i0pGnKDUFFr8yC/n8xyrNBVfhYlpwE8J3Ge6ThKvug= -github.com/vulcanize/go-ethereum v0.0.0-20190731183759-8e20673bd101/go.mod h1:9i0pGnKDUFFr8yC/n8xyrNBVfhYlpwE8J3Ge6ThKvug= -github.com/vulcanize/vulcanizedb v0.0.9 h1:ozV2t/MG4/WLgP89blPOw4r1KwM5ji9+DW9CXPGFX08= -github.com/vulcanize/vulcanizedb v0.0.9/go.mod h1:9zfi6VIiCrDVHXe9Z0xmW4CDmNPFwdLuHLcRjjSrdLQ= github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 h1:1cngl9mPEoITZG8s8cVcUy5CeIBYhEESkOB7m6Gmkrk= github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees= -github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= -github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/xtaci/kcp-go v5.4.5+incompatible/go.mod h1:bN6vIwHQbfHaHtFpEssmWsN45a+AZwO7eyRCmEIbtvE= -github.com/xtaci/lossyconn v0.0.0-20190602105132-8df528c0c9ae/go.mod h1:gXtu8J62kEgmN++bm9BVICuT/e8yiLI2KFobd/TRFsE= -golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5 h1:bselrhR0Or1vomJZC8ZIjWtbDmn9OYFLX5Ik9alpJpE= -golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= -golang.org/x/crypto v0.0.0-20190926114937-fa1a29108794/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= -golang.org/x/crypto v0.0.0-20191119213627-4f8c1d86b1ba h1:9bFeDpN3gTqNanMVqNcoR/pJQuP5uroC3t1D7eXozTE= -golang.org/x/crypto v0.0.0-20191119213627-4f8c1d86b1ba/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4 h1:QmwruyY+bKbDDL0BaglrbZABEali68eoMFhTZpCjYVA= -golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65 h1:+rhAzEzT3f4JtomfC371qB+0Ola2caSKcY69NUBZrRQ= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190912160710-24e19bdeb0f2 h1:4dVFTC832rPn4pomLSz1vA+are2+dU19w1H8OngV7nc= -golang.org/x/net v0.0.0-20190912160710-24e19bdeb0f2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5 h1:mzjBh+S5frKOsOBobWIMAbXavqjmgO17k/2puhcFR94= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190912141932-bc967efca4b8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 h1:rOhMmluY6kLMhdnrivzec6lLgaVbMHMn2ISQXJeJ5EM= -golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8 h1:AvbQYmiaaaza3cW3QXRyPo5kYgpFIzOAfeAAN7m3qQ4= +golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190912185636-87d9f09c5d89/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/bsm/ratelimit.v1 v1.0.0-20160220154919-db14e161995a/go.mod h1:KF9sEfUPAXdG8Oev9e99iLGnl2uJMjc5B+4y3O7x610= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= -gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= -gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= -gopkg.in/jcmturner/gokrb5.v7 v7.2.3/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= -gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= -gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190213234257-ec84240a7772 h1:hhsSf/5z74Ck/DJYc+R8zpq8KGm7uJvpdLRQED/IedA= -gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190213234257-ec84240a7772/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= -gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190709231704-1e4459ed25ff h1:uuol9OUzSvZntY1v963NAbVd7A+PHLMz1FlCe3Lorcs= -gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190709231704-1e4459ed25ff/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= -gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200316214253-d7b0ff38cac9/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= -gopkg.in/redis.v4 v4.2.4/go.mod h1:8KREHdypkCEojGKQcjMqAODMICIVwZAONWq8RowTITA= -gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= +gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6 h1:a6cXbcDDUkSBlpnkWV1bJ+vv3mOgQEltEJ2rPxroVu0= +gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= -gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/integration_test/contract_watcher_transformer_test.go b/integration_test/contract_watcher_transformer_test.go index 5fd15df00..4e40b5b63 100644 --- a/integration_test/contract_watcher_transformer_test.go +++ b/integration_test/contract_watcher_transformer_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -package integration +package integration_test import ( "fmt" @@ -27,15 +27,15 @@ import ( "github.com/makerdao/vulcanizedb/pkg/contract_watcher/transformer" "github.com/makerdao/vulcanizedb/pkg/core" "github.com/makerdao/vulcanizedb/pkg/datastore" - "github.com/makerdao/vulcanizedb/pkg/datastore/postgres" "github.com/makerdao/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/makerdao/vulcanizedb/test_config" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) var _ = Describe("contractWatcher transformer", func() { var ( - db *postgres.DB + db = test_config.NewTestDB(test_config.NewTestNode()) err error blockChain core.BlockChain headerRepository datastore.HeaderRepository @@ -45,7 +45,7 @@ var _ = Describe("contractWatcher transformer", func() { ) BeforeEach(func() { - db, blockChain = test_helpers.SetupDBandBC() + blockChain = SetupBC() headerRepository = repositories.NewHeaderRepository(db) }) diff --git a/integration_test/geth_blockchain_test.go b/integration_test/geth_blockchain_test.go index 1a05778d5..0c57cd78e 100644 --- a/integration_test/geth_blockchain_test.go +++ b/integration_test/geth_blockchain_test.go @@ -18,30 +18,16 @@ package integration_test import ( "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/ethclient" - "github.com/ethereum/go-ethereum/rpc" "github.com/makerdao/vulcanizedb/pkg/core" - "github.com/makerdao/vulcanizedb/pkg/eth" - "github.com/makerdao/vulcanizedb/pkg/eth/client" - "github.com/makerdao/vulcanizedb/pkg/eth/converters" - "github.com/makerdao/vulcanizedb/pkg/eth/node" - "github.com/makerdao/vulcanizedb/test_config" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) var _ = Describe("Reading from the Geth blockchain", func() { - var blockChain *eth.BlockChain + var blockChain core.BlockChain BeforeEach(func() { - rawRpcClient, err := rpc.Dial(test_config.TestClient.IPCPath) - Expect(err).NotTo(HaveOccurred()) - rpcClient := client.NewRpcClient(rawRpcClient, test_config.TestClient.IPCPath) - ethClient := ethclient.NewClient(rawRpcClient) - blockChainClient := client.NewEthClient(ethClient) - node := node.MakeNode(rpcClient) - transactionConverter := converters.NewTransactionConverter(ethClient) - blockChain = eth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter) + blockChain = SetupBC() }) It("retrieves the node info", func(done Done) { diff --git a/integration_test/integration_test.go b/integration_test/integration_test.go new file mode 100644 index 000000000..5d1677f94 --- /dev/null +++ b/integration_test/integration_test.go @@ -0,0 +1,51 @@ +package integration_test + +import ( + "errors" + + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/rpc" + "github.com/makerdao/vulcanizedb/pkg/config" + "github.com/makerdao/vulcanizedb/pkg/core" + "github.com/makerdao/vulcanizedb/pkg/eth" + "github.com/makerdao/vulcanizedb/pkg/eth/client" + "github.com/makerdao/vulcanizedb/pkg/eth/converters" + "github.com/makerdao/vulcanizedb/pkg/eth/node" + "github.com/makerdao/vulcanizedb/test_config" + . "github.com/onsi/gomega" + "github.com/sirupsen/logrus" +) + +var TestClient config.Client + +func init() { + ipc := test_config.TestConfig.GetString("client.ipcPath") + + // If we don't have an ipc path in the config file, check the env variable + if ipc == "" { + test_config.TestConfig.BindEnv("url", "CLIENT_IPCPATH") + ipc = test_config.TestConfig.GetString("url") + } + if ipc == "" { + logrus.Fatal(errors.New("testing.toml IPC path or $CLIENT_IPCPATH env variable need to be set")) + } + + TestClient = config.Client{ + IPCPath: ipc, + } +} + +func SetupBC() core.BlockChain { + con := TestClient + testIPC := con.IPCPath + rawRPCClient, err := rpc.Dial(testIPC) + Expect(err).NotTo(HaveOccurred()) + rpcClient := client.NewRpcClient(rawRPCClient, testIPC) + ethClient := ethclient.NewClient(rawRPCClient) + blockChainClient := client.NewEthClient(ethClient) + madeNode := node.MakeNode(rpcClient) + transactionConverter := converters.NewTransactionConverter(ethClient) + blockChain := eth.NewBlockChain(blockChainClient, rpcClient, madeNode, transactionConverter) + + return blockChain +} diff --git a/libraries/shared/factories/storage/README.md b/libraries/shared/factories/storage/README.md index 9fcc824d7..295c7c3e1 100644 --- a/libraries/shared/factories/storage/README.md +++ b/libraries/shared/factories/storage/README.md @@ -71,7 +71,6 @@ func (lookup *keysLookup) refreshMappings() error { if err != nil { return err } - lookup.mappings = utils.AddHashedKeys(lookup.mappings) return nil } ``` diff --git a/libraries/shared/factories/storage/keys_lookup.go b/libraries/shared/factories/storage/keys_lookup.go index ca90ec8e6..13d3ad513 100644 --- a/libraries/shared/factories/storage/keys_lookup.go +++ b/libraries/shared/factories/storage/keys_lookup.go @@ -20,7 +20,6 @@ import ( "fmt" "github.com/ethereum/go-ethereum/common" - "github.com/makerdao/vulcanizedb/libraries/shared/storage" "github.com/makerdao/vulcanizedb/libraries/shared/storage/types" "github.com/makerdao/vulcanizedb/pkg/datastore/postgres" ) @@ -59,7 +58,6 @@ func (lookup *keysLookup) Lookup(key common.Hash) (types.ValueMetadata, error) { if refreshErr != nil { return metadata, fmt.Errorf("error refreshing mappings in keys lookup: %w", refreshErr) } - lookup.mappings = storage.AddHashedKeys(lookup.mappings) metadata, ok = lookup.mappings[key] if !ok { return metadata, fmt.Errorf("%w: %s", types.ErrKeyNotFound, key.Hex()) diff --git a/libraries/shared/factories/storage/keys_lookup_test.go b/libraries/shared/factories/storage/keys_lookup_test.go index 5da469df6..f60990214 100644 --- a/libraries/shared/factories/storage/keys_lookup_test.go +++ b/libraries/shared/factories/storage/keys_lookup_test.go @@ -18,7 +18,6 @@ package storage_test import ( "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" "github.com/makerdao/vulcanizedb/libraries/shared/factories/storage" "github.com/makerdao/vulcanizedb/libraries/shared/mocks" "github.com/makerdao/vulcanizedb/libraries/shared/storage/types" @@ -91,17 +90,6 @@ var _ = Describe("Storage keys lookup", func() { Expect(metadata).To(Equal(fakeMetadata)) }) - It("returns metadata for hashed version of key (accommodates keys emitted from Geth)", func() { - fakeKey := test_data.FakeHash() - loader.StorageKeyMappings = map[common.Hash]types.ValueMetadata{fakeKey: fakeMetadata} - - hashedKey := common.BytesToHash(crypto.Keccak256(fakeKey.Bytes())) - metadata, err := lookup.Lookup(hashedKey) - - Expect(err).NotTo(HaveOccurred()) - Expect(metadata).To(Equal(fakeMetadata)) - }) - It("returns key not found error if key not found", func() { fakeKey := test_data.FakeHash() _, err := lookup.Lookup(fakeKey) diff --git a/libraries/shared/factories/storage/transformer.go b/libraries/shared/factories/storage/transformer.go index 59088e424..40d65d512 100644 --- a/libraries/shared/factories/storage/transformer.go +++ b/libraries/shared/factories/storage/transformer.go @@ -27,7 +27,6 @@ import ( type ITransformer interface { Execute(diff types.PersistedDiff) error - KeccakContractAddress() common.Hash GetStorageKeysLookup() KeysLookup GetContractAddress() common.Address } @@ -38,7 +37,6 @@ type Transformer struct { Address common.Address StorageKeysLookup KeysLookup Repository Repository - hashedAddress common.Hash } func (transformer Transformer) GetStorageKeysLookup() KeysLookup { @@ -55,16 +53,6 @@ func (transformer Transformer) NewTransformer(db *postgres.DB) ITransformer { return &transformer } -func (transformer *Transformer) KeccakContractAddress() common.Hash { - emptyHash := common.Hash{} - if transformer.hashedAddress == emptyHash { - transformer.hashedAddress = types.HexToKeccak256Hash(transformer.Address.Hex()) - return transformer.hashedAddress - } - - return transformer.hashedAddress -} - func (transformer Transformer) Execute(diff types.PersistedDiff) error { metadata, lookupErr := transformer.StorageKeysLookup.Lookup(diff.StorageKey) if lookupErr != nil { diff --git a/libraries/shared/factories/storage/transformer_test.go b/libraries/shared/factories/storage/transformer_test.go index 8b857e263..e6e53d970 100644 --- a/libraries/shared/factories/storage/transformer_test.go +++ b/libraries/shared/factories/storage/transformer_test.go @@ -23,6 +23,7 @@ import ( "github.com/makerdao/vulcanizedb/libraries/shared/factories/storage" "github.com/makerdao/vulcanizedb/libraries/shared/mocks" "github.com/makerdao/vulcanizedb/libraries/shared/storage/types" + "github.com/makerdao/vulcanizedb/libraries/shared/test_data" "github.com/makerdao/vulcanizedb/pkg/fakes" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -45,14 +46,6 @@ var _ = Describe("Storage transformer", func() { } }) - It("returns the keccaked contract address being watched", func() { - fakeAddress := fakes.FakeAddress - keccakOfAddress := types.HexToKeccak256Hash(fakeAddress.Hex()) - t.Address = fakeAddress - - Expect(t.KeccakContractAddress()).To(Equal(keccakOfAddress)) - }) - It("returns the contract address being watched", func() { fakeAddress := fakes.FakeAddress t.Address = fakeAddress @@ -86,11 +79,11 @@ var _ = Describe("Storage transformer", func() { ID: rand.Int63(), HeaderID: fakeHeaderID, RawDiff: types.RawDiff{ - HashedAddress: common.Hash{}, - BlockHash: common.HexToHash(fakeBlockHash), - BlockHeight: fakeBlockNumber, - StorageKey: common.Hash{}, - StorageValue: rawValue.Hash(), + Address: fakes.FakeAddress, + BlockHash: common.HexToHash(fakeBlockHash), + BlockHeight: fakeBlockNumber, + StorageKey: common.Hash{}, + StorageValue: rawValue.Hash(), }, } @@ -140,11 +133,11 @@ var _ = Describe("Storage transformer", func() { ID: rand.Int63(), HeaderID: fakeHeaderID, RawDiff: types.RawDiff{ - HashedAddress: common.Hash{}, - BlockHash: common.HexToHash(fakeBlockHash), - BlockHeight: fakeBlockNumber, - StorageKey: common.Hash{}, - StorageValue: rawValue.Hash(), + Address: test_data.FakeAddress(), + BlockHash: common.HexToHash(fakeBlockHash), + BlockHeight: fakeBlockNumber, + StorageKey: common.Hash{}, + StorageValue: rawValue.Hash(), }, } diff --git a/libraries/shared/logs/extractor.go b/libraries/shared/logs/extractor.go index 9c8e75ca9..3e1c6bbf3 100644 --- a/libraries/shared/logs/extractor.go +++ b/libraries/shared/logs/extractor.go @@ -32,9 +32,14 @@ import ( "github.com/sirupsen/logrus" ) +type BlockIdentifier string + var ( - ErrNoUncheckedHeaders = errors.New("no unchecked headers available for log fetching") - ErrNoWatchedAddresses = errors.New("no watched addresses configured in the log extractor") + StartInterval BlockIdentifier = "start" + EndInterval BlockIdentifier = "end" + ErrNoUncheckedHeaders = errors.New("no unchecked headers available for log fetching") + ErrNoWatchedAddresses = errors.New("no watched addresses configured in the log extractor") + HeaderChunkSize int64 = 1000 ) type ILogExtractor interface { @@ -57,11 +62,12 @@ type LogExtractor struct { RecheckHeaderCap int64 } -func NewLogExtractor(db *postgres.DB, bc core.BlockChain) *LogExtractor { +func NewLogExtractor(db *postgres.DB, bc core.BlockChain, chr datastore.CheckedHeadersRepository) *LogExtractor { return &LogExtractor{ - CheckedHeadersRepository: repositories.NewCheckedHeadersRepository(db), + CheckedHeadersRepository: chr, CheckedLogsRepository: repositories.NewCheckedLogsRepository(db), Fetcher: fetcher.NewLogFetcher(bc), + HeaderRepository: repositories.NewHeaderRepository(db), LogRepository: repositories.NewEventLogRepository(db), Syncer: transactions.NewTransactionsSyncer(db, bc), RecheckHeaderCap: constants.RecheckHeaderCap, @@ -152,21 +158,56 @@ func (extractor LogExtractor) BackFillLogs(endingBlock int64) error { return fmt.Errorf("error extracting logs: %w", ErrNoWatchedAddresses) } - headers, headersErr := extractor.HeaderRepository.GetHeadersInRange(*extractor.StartingBlock, endingBlock) - if headersErr != nil { - logrus.Errorf("error fetching missing headers: %s", headersErr) - return fmt.Errorf("error getting unchecked headers to check for logs: %w", headersErr) + ranges, chunkErr := ChunkRanges(*extractor.StartingBlock, endingBlock, HeaderChunkSize) + if chunkErr != nil { + return fmt.Errorf("error chunking headers to lookup in logs backfill: %w", chunkErr) } - for _, header := range headers { - err := extractor.fetchAndPersistLogsForHeader(header) - if err != nil { - return fmt.Errorf("error fetching and persisting logs for header with id %d: %w", header.Id, err) + for _, r := range ranges { + headers, headersErr := extractor.HeaderRepository.GetHeadersInRange(r[StartInterval], r[EndInterval]) + if headersErr != nil { + logrus.Errorf("error fetching missing headers: %s", headersErr) + return fmt.Errorf("error getting unchecked headers to check for logs: %w", headersErr) + } + + for _, header := range headers { + err := extractor.fetchAndPersistLogsForHeader(header) + if err != nil { + return fmt.Errorf("error fetching and persisting logs for header with id %d: %w", header.Id, err) + } } } + return nil } +func ChunkRanges(startingBlock, endingBlock, interval int64) ([]map[BlockIdentifier]int64, error) { + if endingBlock <= startingBlock { + return nil, errors.New("ending block for backfill not > starting block") + } + + totalLength := endingBlock - startingBlock + numIntervals := totalLength / interval + if totalLength%interval != 0 { + numIntervals++ + } + + results := make([]map[BlockIdentifier]int64, numIntervals) + for i := int64(0); i < numIntervals; i++ { + nextStartBlock := startingBlock + i*interval + nextEndingBlock := nextStartBlock + interval - 1 + if nextEndingBlock > endingBlock { + nextEndingBlock = endingBlock + } + nextInterval := map[BlockIdentifier]int64{ + StartInterval: nextStartBlock, + EndInterval: nextEndingBlock, + } + results[i] = nextInterval + } + return results, nil +} + func logError(description string, err error, header core.Header) { logrus.WithFields(logrus.Fields{ "headerId": header.Id, diff --git a/libraries/shared/logs/extractor_test.go b/libraries/shared/logs/extractor_test.go index 9bfb6ad5d..75895ffb8 100644 --- a/libraries/shared/logs/extractor_test.go +++ b/libraries/shared/logs/extractor_test.go @@ -170,7 +170,8 @@ var _ = Describe("Log extractor", func() { mockCheckedHeadersRepository.UncheckedHeadersReturnHeaders = []core.Header{{}} extractor.CheckedHeadersRepository = mockCheckedHeadersRepository startingBlockNumber := rand.Int63() - extractor.AddTransformerConfig(getTransformerConfig(startingBlockNumber, defaultEndingBlockNumber)) + addErr := extractor.AddTransformerConfig(getTransformerConfig(startingBlockNumber, defaultEndingBlockNumber)) + Expect(addErr).NotTo(HaveOccurred()) err := extractor.ExtractLogs(constants.HeaderUnchecked) @@ -187,7 +188,8 @@ var _ = Describe("Log extractor", func() { mockCheckedHeadersRepository.UncheckedHeadersReturnHeaders = []core.Header{{}} extractor.CheckedHeadersRepository = mockCheckedHeadersRepository startingBlockNumber := rand.Int63() - extractor.AddTransformerConfig(getTransformerConfig(startingBlockNumber, defaultEndingBlockNumber)) + addErr := extractor.AddTransformerConfig(getTransformerConfig(startingBlockNumber, defaultEndingBlockNumber)) + Expect(addErr).NotTo(HaveOccurred()) err := extractor.ExtractLogs(constants.HeaderRecheck) @@ -393,29 +395,43 @@ var _ = Describe("Log extractor", func() { }) It("gets headers from transformer's starting block through passed ending block", func() { - fakeConfig := event.TransformerConfig{ - ContractAddresses: []string{fakes.FakeAddress.Hex()}, - Topic: fakes.FakeHash.Hex(), - StartingBlockNumber: rand.Int63(), - } - extractor.AddTransformerConfig(fakeConfig) + startingBlock := addTransformerConfig(extractor) mockHeaderRepository := &fakes.MockHeaderRepository{} extractor.HeaderRepository = mockHeaderRepository - endingBlock := rand.Int63() + endingBlock := startingBlock + 1 _ = extractor.BackFillLogs(endingBlock) - Expect(mockHeaderRepository.GetHeadersInRangeStartingBlock).To(Equal(fakeConfig.StartingBlockNumber)) - Expect(mockHeaderRepository.GetHeadersInRangeEndingBlock).To(Equal(endingBlock)) + Expect(mockHeaderRepository.GetHeadersInRangeStartingBlocks).To(ContainElement(startingBlock)) + Expect(mockHeaderRepository.GetHeadersInRangeEndingBlocks).To(ContainElement(endingBlock)) + }) + + It("chunks headers if range greater than interval defined by HeaderChunkSize", func() { + startingBlock := addTransformerConfig(extractor) + mockHeaderRepository := &fakes.MockHeaderRepository{} + extractor.HeaderRepository = mockHeaderRepository + endingBlock := startingBlock + logs.HeaderChunkSize*2 + + err := extractor.BackFillLogs(endingBlock) + + Expect(err).NotTo(HaveOccurred()) + Expect(mockHeaderRepository.GetHeadersInRangeStartingBlocks).To(ConsistOf([]int64{ + startingBlock, + startingBlock + logs.HeaderChunkSize, + })) + Expect(mockHeaderRepository.GetHeadersInRangeEndingBlocks).To(ConsistOf([]int64{ + startingBlock + logs.HeaderChunkSize - 1, + startingBlock + logs.HeaderChunkSize*2 - 1, + })) }) It("returns error if getting headers in range returns error", func() { mockHeaderRepository := &fakes.MockHeaderRepository{} mockHeaderRepository.GetHeadersInRangeError = fakes.FakeError extractor.HeaderRepository = mockHeaderRepository - addTransformerConfig(extractor) + startingBlock := addTransformerConfig(extractor) - err := extractor.BackFillLogs(0) + err := extractor.BackFillLogs(startingBlock + 1) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) @@ -424,11 +440,11 @@ var _ = Describe("Log extractor", func() { It("does nothing if no headers found", func() { mockHeaderRepository := &fakes.MockHeaderRepository{} extractor.HeaderRepository = mockHeaderRepository - addTransformerConfig(extractor) + startingBlock := addTransformerConfig(extractor) mockLogFetcher := &mocks.MockLogFetcher{} extractor.Fetcher = mockLogFetcher - _ = extractor.BackFillLogs(0) + _ = extractor.BackFillLogs(startingBlock + 1) Expect(mockLogFetcher.FetchCalled).To(BeFalse()) }) @@ -445,7 +461,7 @@ var _ = Describe("Log extractor", func() { mockLogFetcher := &mocks.MockLogFetcher{} extractor.Fetcher = mockLogFetcher - err := extractor.BackFillLogs(0) + err := extractor.BackFillLogs(config.StartingBlockNumber + 1) Expect(err).NotTo(HaveOccurred()) Expect(mockLogFetcher.FetchCalled).To(BeTrue()) @@ -457,12 +473,12 @@ var _ = Describe("Log extractor", func() { It("returns error if fetching logs fails", func() { addHeaderInRange(extractor) - addTransformerConfig(extractor) + startingBlock := addTransformerConfig(extractor) mockLogFetcher := &mocks.MockLogFetcher{} mockLogFetcher.ReturnError = fakes.FakeError extractor.Fetcher = mockLogFetcher - err := extractor.BackFillLogs(0) + err := extractor.BackFillLogs(startingBlock + 1) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) @@ -470,11 +486,11 @@ var _ = Describe("Log extractor", func() { It("does not sync transactions when no logs", func() { addHeaderInRange(extractor) - addTransformerConfig(extractor) + startingBlock := addTransformerConfig(extractor) mockTransactionSyncer := &fakes.MockTransactionSyncer{} extractor.Syncer = mockTransactionSyncer - err := extractor.BackFillLogs(0) + err := extractor.BackFillLogs(startingBlock + 1) Expect(err).NotTo(HaveOccurred()) Expect(mockTransactionSyncer.SyncTransactionsCalled).To(BeFalse()) @@ -484,11 +500,11 @@ var _ = Describe("Log extractor", func() { It("syncs transactions", func() { addHeaderInRange(extractor) addFetchedLog(extractor) - addTransformerConfig(extractor) + startingBlock := addTransformerConfig(extractor) mockTransactionSyncer := &fakes.MockTransactionSyncer{} extractor.Syncer = mockTransactionSyncer - err := extractor.BackFillLogs(0) + err := extractor.BackFillLogs(startingBlock + 1) Expect(err).NotTo(HaveOccurred()) Expect(mockTransactionSyncer.SyncTransactionsCalled).To(BeTrue()) @@ -497,12 +513,12 @@ var _ = Describe("Log extractor", func() { It("returns error if syncing transactions fails", func() { addHeaderInRange(extractor) addFetchedLog(extractor) - addTransformerConfig(extractor) + startingBlock := addTransformerConfig(extractor) mockTransactionSyncer := &fakes.MockTransactionSyncer{} mockTransactionSyncer.SyncTransactionsError = fakes.FakeError extractor.Syncer = mockTransactionSyncer - err := extractor.BackFillLogs(0) + err := extractor.BackFillLogs(startingBlock + 1) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) @@ -510,7 +526,7 @@ var _ = Describe("Log extractor", func() { It("persists fetched logs", func() { addHeaderInRange(extractor) - addTransformerConfig(extractor) + startingBlock := addTransformerConfig(extractor) fakeLogs := []types.Log{{ Address: common.HexToAddress("0xA"), Topics: []common.Hash{common.HexToHash("0xA")}, @@ -522,7 +538,7 @@ var _ = Describe("Log extractor", func() { mockLogRepository := &fakes.MockEventLogRepository{} extractor.LogRepository = mockLogRepository - err := extractor.BackFillLogs(0) + err := extractor.BackFillLogs(startingBlock + 1) Expect(err).NotTo(HaveOccurred()) Expect(mockLogRepository.PassedLogs).To(Equal(fakeLogs)) @@ -531,27 +547,58 @@ var _ = Describe("Log extractor", func() { It("returns error if persisting logs fails", func() { addHeaderInRange(extractor) addFetchedLog(extractor) - addTransformerConfig(extractor) + startingBlock := addTransformerConfig(extractor) mockLogRepository := &fakes.MockEventLogRepository{} mockLogRepository.CreateError = fakes.FakeError extractor.LogRepository = mockLogRepository - err := extractor.BackFillLogs(0) + err := extractor.BackFillLogs(startingBlock + 1) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) }) }) }) + + Describe("ChunkRanges", func() { + It("returns error if upper bound <= lower bound", func() { + _, err := logs.ChunkRanges(10, 10, 1) + + Expect(err).To(HaveOccurred()) + }) + + It("chunks items in range by interval", func() { + res, err := logs.ChunkRanges(1, 20, 10) + + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(ConsistOf([]map[logs.BlockIdentifier]int64{ + {logs.StartInterval: 1, logs.EndInterval: 10}, + {logs.StartInterval: 11, logs.EndInterval: 20}, + })) + }) + + It("truncates final chunk to end at upper bound", func() { + res, err := logs.ChunkRanges(1, 25, 10) + + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(ConsistOf([]map[logs.BlockIdentifier]int64{ + {logs.StartInterval: 1, logs.EndInterval: 10}, + {logs.StartInterval: 11, logs.EndInterval: 20}, + {logs.StartInterval: 21, logs.EndInterval: 25}, + })) + }) + }) }) -func addTransformerConfig(extractor *logs.LogExtractor) { +func addTransformerConfig(extractor *logs.LogExtractor) int64 { fakeConfig := event.TransformerConfig{ ContractAddresses: []string{fakes.FakeAddress.Hex()}, Topic: fakes.FakeHash.Hex(), StartingBlockNumber: rand.Int63(), } - extractor.AddTransformerConfig(fakeConfig) + addErr := extractor.AddTransformerConfig(fakeConfig) + Expect(addErr).NotTo(HaveOccurred()) + return fakeConfig.StartingBlockNumber } func addUncheckedHeader(extractor *logs.LogExtractor) { diff --git a/libraries/shared/mocks/storage_diff_repository.go b/libraries/shared/mocks/storage_diff_repository.go index 2c1f65eb0..df96db43c 100644 --- a/libraries/shared/mocks/storage_diff_repository.go +++ b/libraries/shared/mocks/storage_diff_repository.go @@ -24,11 +24,18 @@ type MockStorageDiffRepository struct { CreateBackFilledStorageValuePassedRawDiffs []types.RawDiff CreateBackFilledStorageValueReturnError error CreatePassedRawDiffs []types.RawDiff - GetNewDiffsDiffs []types.PersistedDiff + GetNewDiffsToReturn []types.PersistedDiff GetNewDiffsErrors []error GetNewDiffsPassedMinIDs []int GetNewDiffsPassedLimits []int - MarkCheckedPassedID int64 + GetUnrecognizedDiffsToReturn []types.PersistedDiff + GetUnrecognizedDiffsErrors []error + GetUnrecognizedDiffsPassedMinIDs []int + GetUnrecognizedDiffsPassedLimits []int + MarkTransformedPassedID int64 + MarkUnrecognizedPassedID int64 + MarkNoncanonicalPassedID int64 + MarkUnwatchedPassedID int64 GetFirstDiffIDToReturn int64 GetFirstDiffIDErr error GetFirstDiffBlockHeightPassed int64 @@ -51,11 +58,36 @@ func (repository *MockStorageDiffRepository) GetNewDiffs(minID, limit int) ([]ty if len(repository.GetNewDiffsErrors) > 1 { repository.GetNewDiffsErrors = repository.GetNewDiffsErrors[1:] } - return repository.GetNewDiffsDiffs, err + return repository.GetNewDiffsToReturn, err } -func (repository *MockStorageDiffRepository) MarkChecked(id int64) error { - repository.MarkCheckedPassedID = id +func (repository *MockStorageDiffRepository) GetUnrecognizedDiffs(minID, limit int) ([]types.PersistedDiff, error) { + repository.GetUnrecognizedDiffsPassedMinIDs = append(repository.GetUnrecognizedDiffsPassedMinIDs, minID) + repository.GetUnrecognizedDiffsPassedLimits = append(repository.GetUnrecognizedDiffsPassedLimits, limit) + err := repository.GetUnrecognizedDiffsErrors[0] + if len(repository.GetUnrecognizedDiffsErrors) > 1 { + repository.GetUnrecognizedDiffsErrors = repository.GetUnrecognizedDiffsErrors[1:] + } + return repository.GetUnrecognizedDiffsToReturn, err +} + +func (repository *MockStorageDiffRepository) MarkTransformed(id int64) error { + repository.MarkTransformedPassedID = id + return nil +} + +func (repository *MockStorageDiffRepository) MarkNoncanonical(id int64) error { + repository.MarkNoncanonicalPassedID = id + return nil +} + +func (repository *MockStorageDiffRepository) MarkUnrecognized(id int64) error { + repository.MarkUnrecognizedPassedID = id + return nil +} + +func (repository *MockStorageDiffRepository) MarkUnwatched(id int64) error { + repository.MarkUnwatchedPassedID = id return nil } diff --git a/libraries/shared/mocks/storage_transformer.go b/libraries/shared/mocks/storage_transformer.go index d2db218a7..fe698bef9 100644 --- a/libraries/shared/mocks/storage_transformer.go +++ b/libraries/shared/mocks/storage_transformer.go @@ -26,7 +26,6 @@ import ( type MockStorageTransformer struct { Address common.Address StorageKeysLookup storage.KeysLookup - KeccakOfAddress common.Hash ExecuteErr error PassedDiff types.PersistedDiff } @@ -36,10 +35,6 @@ func (transformer *MockStorageTransformer) Execute(diff types.PersistedDiff) err return transformer.ExecuteErr } -func (transformer *MockStorageTransformer) KeccakContractAddress() common.Hash { - return transformer.KeccakOfAddress -} - func (transformer *MockStorageTransformer) GetContractAddress() common.Address { return transformer.Address } diff --git a/libraries/shared/mocks/streamer.go b/libraries/shared/mocks/streamer.go index 9c7fd53b8..0b69f5e29 100644 --- a/libraries/shared/mocks/streamer.go +++ b/libraries/shared/mocks/streamer.go @@ -1,7 +1,7 @@ package mocks import ( - "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/makerdao/vulcanizedb/pkg/core" "github.com/makerdao/vulcanizedb/pkg/fakes" ) @@ -9,11 +9,11 @@ import ( type MockStoragediffStreamer struct { subscribeError error ClientSubscription *fakes.MockSubscription - PassedPayloadChan chan statediff.Payload - streamPayloads []statediff.Payload + PassedPayloadChan chan filters.Payload + streamPayloads []filters.Payload } -func (streamer *MockStoragediffStreamer) Stream(statediffPayloadChan chan statediff.Payload) (core.Subscription, error) { +func (streamer *MockStoragediffStreamer) Stream(statediffPayloadChan chan filters.Payload) (core.Subscription, error) { streamer.PassedPayloadChan = statediffPayloadChan go func() { @@ -29,6 +29,6 @@ func (streamer *MockStoragediffStreamer) SetSubscribeError(err error) { streamer.subscribeError = err } -func (streamer *MockStoragediffStreamer) SetPayloads(payloads []statediff.Payload) { +func (streamer *MockStoragediffStreamer) SetPayloads(payloads []filters.Payload) { streamer.streamPayloads = payloads } diff --git a/libraries/shared/repository/address_repository.go b/libraries/shared/repository/address_repository.go index af630c393..d94cc15dd 100644 --- a/libraries/shared/repository/address_repository.go +++ b/libraries/shared/repository/address_repository.go @@ -32,12 +32,11 @@ package repository import ( "github.com/ethereum/go-ethereum/common" "github.com/jmoiron/sqlx" - "github.com/makerdao/vulcanizedb/libraries/shared/storage/types" "github.com/makerdao/vulcanizedb/pkg/datastore/postgres" ) const getOrCreateAddressQuery = `WITH addressId AS ( - INSERT INTO addresses (address, hashed_address) VALUES ($1, $2) ON CONFLICT DO NOTHING RETURNING id + INSERT INTO addresses (address) VALUES ($1) ON CONFLICT DO NOTHING RETURNING id ) SELECT id FROM addresses WHERE address = $1 UNION @@ -45,20 +44,18 @@ const getOrCreateAddressQuery = `WITH addressId AS ( func GetOrCreateAddress(db *postgres.DB, address string) (int64, error) { checksumAddress := getChecksumAddress(address) - hashedAddress := types.HexToKeccak256Hash(checksumAddress).Hex() var addressId int64 - getOrCreateErr := db.Get(&addressId, getOrCreateAddressQuery, checksumAddress, hashedAddress) + getOrCreateErr := db.Get(&addressId, getOrCreateAddressQuery, checksumAddress) return addressId, getOrCreateErr } func GetOrCreateAddressInTransaction(tx *sqlx.Tx, address string) (int64, error) { checksumAddress := getChecksumAddress(address) - hashedAddress := types.HexToKeccak256Hash(checksumAddress).Hex() var addressId int64 - getOrCreateErr := tx.Get(&addressId, getOrCreateAddressQuery, checksumAddress, hashedAddress) + getOrCreateErr := tx.Get(&addressId, getOrCreateAddressQuery, checksumAddress) return addressId, getOrCreateErr } diff --git a/libraries/shared/repository/address_repository_test.go b/libraries/shared/repository/address_repository_test.go index 91d0426ba..ca68b105f 100644 --- a/libraries/shared/repository/address_repository_test.go +++ b/libraries/shared/repository/address_repository_test.go @@ -21,7 +21,6 @@ import ( "github.com/jmoiron/sqlx" "github.com/makerdao/vulcanizedb/libraries/shared/repository" - "github.com/makerdao/vulcanizedb/libraries/shared/storage/types" "github.com/makerdao/vulcanizedb/pkg/datastore/postgres" "github.com/makerdao/vulcanizedb/pkg/fakes" "github.com/makerdao/vulcanizedb/test_config" @@ -44,9 +43,8 @@ var _ = Describe("address lookup", func() { }) type dbAddress struct { - Id int64 - Address string - HashedAddress string `db:"hashed_address"` + Id int64 + Address string } Describe("GetOrCreateAddress", func() { @@ -55,10 +53,9 @@ var _ = Describe("address lookup", func() { Expect(createErr).NotTo(HaveOccurred()) var actualAddress dbAddress - getErr := db.Get(&actualAddress, `SELECT id, address, hashed_address FROM public.addresses LIMIT 1`) + getErr := db.Get(&actualAddress, `SELECT id, address FROM public.addresses LIMIT 1`) Expect(getErr).NotTo(HaveOccurred()) - hashedAddress := types.HexToKeccak256Hash(address).Hex() - expectedAddress := dbAddress{Id: addressId, Address: address, HashedAddress: hashedAddress} + expectedAddress := dbAddress{Id: addressId, Address: address} Expect(actualAddress).To(Equal(expectedAddress)) }) @@ -118,10 +115,9 @@ var _ = Describe("address lookup", func() { Expect(commitErr).NotTo(HaveOccurred()) var actualAddress dbAddress - getErr := db.Get(&actualAddress, `SELECT id, address, hashed_address FROM public.addresses LIMIT 1`) + getErr := db.Get(&actualAddress, `SELECT id, address FROM public.addresses LIMIT 1`) Expect(getErr).NotTo(HaveOccurred()) - hashedAddress := types.HexToKeccak256Hash(address).Hex() - expectedAddress := dbAddress{Id: addressId, Address: address, HashedAddress: hashedAddress} + expectedAddress := dbAddress{Id: addressId, Address: address} Expect(actualAddress).To(Equal(expectedAddress)) }) diff --git a/libraries/shared/repository/event_trigger_test.go b/libraries/shared/repository/event_trigger_test.go new file mode 100644 index 000000000..f5a78341c --- /dev/null +++ b/libraries/shared/repository/event_trigger_test.go @@ -0,0 +1,44 @@ +package repository_test + +import ( + "github.com/makerdao/vulcanizedb/libraries/shared/test_data" + "github.com/makerdao/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/makerdao/vulcanizedb/pkg/fakes" + "github.com/makerdao/vulcanizedb/test_config" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("event updated trigger", func() { + var ( + db = test_config.NewTestDB(test_config.NewTestNode()) + headerID int64 + ) + + BeforeEach(func() { + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + var insertHeaderErr error + headerID, insertHeaderErr = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) + Expect(insertHeaderErr).NotTo(HaveOccurred()) + }) + + type dbEvent struct { + Created string + Updated string + } + + It("indicates when a record was created or updated", func() { + var eventLogRes dbEvent + test_data.CreateTestLog(headerID, db) + initialEventLogErr := db.Get(&eventLogRes, `SELECT created, updated FROM public.event_logs`) + Expect(initialEventLogErr).NotTo(HaveOccurred()) + Expect(eventLogRes.Created).To(Equal(eventLogRes.Updated)) + + _, updateErr := db.Exec(`UPDATE public.event_logs SET block_hash = '{"new_block_hash"}' where header_id = $1`, headerID) + Expect(updateErr).NotTo(HaveOccurred()) + updatedEventErr := db.Get(&eventLogRes, `SELECT created, updated FROM public.event_logs`) + Expect(updatedEventErr).NotTo(HaveOccurred()) + Expect(eventLogRes.Created).NotTo(Equal(eventLogRes.Updated)) + }) +}) diff --git a/libraries/shared/repository/receipt_trigger_test.go b/libraries/shared/repository/receipt_trigger_test.go new file mode 100644 index 000000000..efcad9688 --- /dev/null +++ b/libraries/shared/repository/receipt_trigger_test.go @@ -0,0 +1,92 @@ +package repository_test + +import ( + "math/big" + "math/rand" + + "github.com/makerdao/vulcanizedb/libraries/shared/test_data" + "github.com/makerdao/vulcanizedb/pkg/core" + "github.com/makerdao/vulcanizedb/pkg/datastore" + "github.com/makerdao/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/makerdao/vulcanizedb/pkg/fakes" + "github.com/makerdao/vulcanizedb/test_config" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("receipt updated trigger", func() { + var ( + db = test_config.NewTestDB(test_config.NewTestNode()) + receiptRepo repositories.ReceiptRepository + headerRepo datastore.HeaderRepository + headerID int64 + ) + + BeforeEach(func() { + test_config.CleanTestDB(db) + receiptRepo = repositories.ReceiptRepository{} + headerRepo = repositories.NewHeaderRepository(db) + var insertHeaderErr error + headerID, insertHeaderErr = headerRepo.CreateOrUpdateHeader(fakes.FakeHeader) + Expect(insertHeaderErr).NotTo(HaveOccurred()) + }) + + type dbEvent struct { + Created string + Updated string + } + + It("indicates when a receipt record was created or updated", func() { + var receiptRes dbEvent + fromAddress := test_data.FakeAddress() + toAddress := test_data.FakeAddress() + txHash := test_data.FakeHash() + txIndex := big.NewInt(123) + transaction := core.TransactionModel{ + Data: []byte{}, + From: fromAddress.Hex(), + GasLimit: 0, + GasPrice: 0, + Hash: txHash.Hex(), + Nonce: 0, + Raw: []byte{}, + To: toAddress.Hex(), + TxIndex: txIndex.Int64(), + Value: "0", + } + tx, err := db.Beginx() + Expect(err).ToNot(HaveOccurred()) + txId, txErr := headerRepo.CreateTransactionInTx(tx, headerID, transaction) + Expect(txErr).ToNot(HaveOccurred()) + receipt := core.Receipt{ + ContractAddress: fromAddress.Hex(), + TxHash: txHash.Hex(), + GasUsed: uint64(rand.Int31()), + CumulativeGasUsed: uint64(rand.Int31()), + StateRoot: test_data.FakeHash().Hex(), + Rlp: test_data.FakeHash().Bytes(), + } + + _, receiptErr := receiptRepo.CreateReceiptInTx(headerID, txId, receipt, tx) + Expect(receiptErr).ToNot(HaveOccurred()) + if receiptErr == nil { + // this hangs if called when receiptsErr != nil + commitErr := tx.Commit() + Expect(commitErr).ToNot(HaveOccurred()) + } else { + // lookup on public.receipts below hangs if tx is still open + rollbackErr := tx.Rollback() + Expect(rollbackErr).NotTo(HaveOccurred()) + } + + createdReceiptErr := db.Get(&receiptRes, `SELECT created, updated FROM receipts`) + Expect(createdReceiptErr).NotTo(HaveOccurred()) + Expect(receiptRes.Created).To(Equal(receiptRes.Updated)) + + _, updateErr := db.Exec(`UPDATE public.receipts SET tx_hash = '{"new_hash"}' WHERE header_id = $1`, headerID) + Expect(updateErr).NotTo(HaveOccurred()) + updatedReceiptErr := db.Get(&receiptRes, `SELECT created, updated FROM receipts`) + Expect(updatedReceiptErr).NotTo(HaveOccurred()) + Expect(receiptRes.Created).NotTo(Equal(receiptRes.Updated)) + }) +}) diff --git a/libraries/shared/repository/transaction_trigger_test.go b/libraries/shared/repository/transaction_trigger_test.go new file mode 100644 index 000000000..68ca36b1e --- /dev/null +++ b/libraries/shared/repository/transaction_trigger_test.go @@ -0,0 +1,66 @@ +package repository_test + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/makerdao/vulcanizedb/pkg/core" + "github.com/makerdao/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/makerdao/vulcanizedb/pkg/fakes" + "github.com/makerdao/vulcanizedb/test_config" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "math/big" +) + +var _ = Describe("transaction updated trigger", func() { + var ( + db = test_config.NewTestDB(test_config.NewTestNode()) + headerID int64 + ) + + BeforeEach(func() { + test_config.CleanTestDB(db) + }) + + type dbTrans struct { + Created string + Updated string + } + + It("indicates when a record was created or updated", func() { + headerRepository := repositories.NewHeaderRepository(db) + var insertHeaderErr error + headerID, insertHeaderErr = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) + Expect(insertHeaderErr).NotTo(HaveOccurred()) + fromAddress := common.HexToAddress("0x1234") + toAddress := common.HexToAddress("0x5678") + txHash := common.HexToHash("0x9876") + txIndex := big.NewInt(123) + transaction := []core.TransactionModel{{ + Data: []byte{}, + From: fromAddress.Hex(), + GasLimit: 0, + GasPrice: 0, + Hash: txHash.Hex(), + Nonce: 0, + Raw: []byte{}, + To: toAddress.Hex(), + TxIndex: txIndex.Int64(), + Value: "0", + Receipt: core.Receipt{}, + }} + + txError := headerRepository.CreateTransactions(headerID, transaction) + Expect(txError).NotTo(HaveOccurred()) + + var transUpdateRes dbTrans + initialTxErr := db.Get(&transUpdateRes, `SELECT created, updated FROM public.transactions`) + Expect(initialTxErr).NotTo(HaveOccurred()) + Expect(transUpdateRes.Created).To(Equal(transUpdateRes.Updated)) + + _, updateErr := db.Exec(`UPDATE public.transactions SET hash = '{"new_hash"}' WHERE header_id = $1`, headerID) + Expect(updateErr).NotTo(HaveOccurred()) + updatedTransErr := db.Get(&transUpdateRes, `SELECT created, updated FROM public.transactions`) + Expect(updatedTransErr).NotTo(HaveOccurred()) + Expect(transUpdateRes.Created).NotTo(Equal(transUpdateRes.Updated)) + }) +}) diff --git a/libraries/shared/storage/backfill/storage_value_loader.go b/libraries/shared/storage/backfill/storage_value_loader.go index 14cd09365..94832fe39 100644 --- a/libraries/shared/storage/backfill/storage_value_loader.go +++ b/libraries/shared/storage/backfill/storage_value_loader.go @@ -5,7 +5,6 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" "github.com/makerdao/vulcanizedb/libraries/shared/factories/storage" storage2 "github.com/makerdao/vulcanizedb/libraries/shared/storage" "github.com/makerdao/vulcanizedb/libraries/shared/storage/types" @@ -104,16 +103,14 @@ func (r *StorageValueLoader) getAndPersistStorageValues(blockNumber int64, heade blockHash := common.HexToHash(headerHashStr) for address, chunkedKeysToValues := range r.storageByAddress { - keccakOfAddress := crypto.Keccak256Hash(address[:]) for chunkIndex, currentKeysToValues := range chunkedKeysToValues { var keys []storageKey for key, _ := range currentKeysToValues { keys = append(keys, key) } logrus.WithFields(logrus.Fields{ - "Address": address.Hex(), - "HashedAddress": keccakOfAddress.Hex(), - "BlockNumber": blockNumber, + "Address": address.Hex(), + "BlockNumber": blockNumber, }).Infof("Getting and persisting %v storage values", len(keys)) newKeysToValues, getStorageValuesErr := r.bc.BatchGetStorageAt(address, keys, blockNumberBigInt) if getStorageValuesErr != nil { @@ -125,11 +122,11 @@ func (r *StorageValueLoader) getAndPersistStorageValues(blockNumber int64, heade if newValueHash != currentKeysToValues[key] { // update last known value to new value if changed diff := types.RawDiff{ - HashedAddress: keccakOfAddress, - BlockHash: blockHash, - BlockHeight: int(blockNumber), - StorageKey: crypto.Keccak256Hash(key.Bytes()), - StorageValue: newValueHash, + Address: address, + BlockHash: blockHash, + BlockHeight: int(blockNumber), + StorageKey: key, + StorageValue: newValueHash, } createDiffErr := r.StorageDiffRepo.CreateBackFilledStorageValue(diff) if createDiffErr != nil { diff --git a/libraries/shared/storage/backfill/storage_value_loader_test.go b/libraries/shared/storage/backfill/storage_value_loader_test.go index c9692f240..2d14f2214 100644 --- a/libraries/shared/storage/backfill/storage_value_loader_test.go +++ b/libraries/shared/storage/backfill/storage_value_loader_test.go @@ -5,7 +5,6 @@ import ( "math/rand" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" "github.com/makerdao/vulcanizedb/libraries/shared/factories/storage" "github.com/makerdao/vulcanizedb/libraries/shared/mocks" "github.com/makerdao/vulcanizedb/libraries/shared/storage/backfill" @@ -123,8 +122,8 @@ var _ = Describe("StorageValueLoader", func() { It("fetches headers in the given block range", func() { runnerErr := runner.Run() Expect(runnerErr).NotTo(HaveOccurred()) - Expect(headerRepo.GetHeadersInRangeStartingBlock).To(Equal(blockOne)) - Expect(headerRepo.GetHeadersInRangeEndingBlock).To(Equal(blockTwo)) + Expect(headerRepo.GetHeadersInRangeStartingBlocks).To(ConsistOf(blockOne)) + Expect(headerRepo.GetHeadersInRangeEndingBlocks).To(ConsistOf(blockTwo)) }) It("returns an error if a header for the given block cannot be retrieved", func() { @@ -220,18 +219,18 @@ var _ = Describe("StorageValueLoader", func() { headerHashBytes := common.HexToHash(blockOneHeader.Hash) expectedDiffOne := types.RawDiff{ - BlockHeight: int(blockOne), - BlockHash: headerHashBytes, - HashedAddress: crypto.Keccak256Hash(addressOne[:]), - StorageKey: crypto.Keccak256Hash(keyOne.Bytes()), - StorageValue: valueOne, + Address: addressOne, + BlockHeight: int(blockOne), + BlockHash: headerHashBytes, + StorageKey: keyOne, + StorageValue: valueOne, } expectedDiffTwo := types.RawDiff{ - BlockHeight: int(blockOne), - BlockHash: headerHashBytes, - HashedAddress: crypto.Keccak256Hash(addressTwo[:]), - StorageKey: crypto.Keccak256Hash(keyTwo.Bytes()), - StorageValue: valueTwo, + Address: addressTwo, + BlockHeight: int(blockOne), + BlockHash: headerHashBytes, + StorageKey: keyTwo, + StorageValue: valueTwo, } Expect(diffRepo.CreateBackFilledStorageValuePassedRawDiffs).To(ConsistOf(expectedDiffOne, expectedDiffTwo)) @@ -253,25 +252,25 @@ var _ = Describe("StorageValueLoader", func() { headerHashBytes := common.HexToHash(blockOneHeader.Hash) expectedDiffOne := types.RawDiff{ - BlockHeight: int(blockOne), - BlockHash: headerHashBytes, - HashedAddress: crypto.Keccak256Hash(addressOne[:]), - StorageKey: crypto.Keccak256Hash(keyOne.Bytes()), - StorageValue: valueOne, + Address: addressOne, + BlockHeight: int(blockOne), + BlockHash: headerHashBytes, + StorageKey: keyOne, + StorageValue: valueOne, } expectedDiffTwo := types.RawDiff{ - BlockHeight: int(blockOne), - BlockHash: headerHashBytes, - HashedAddress: crypto.Keccak256Hash(addressTwo[:]), - StorageKey: crypto.Keccak256Hash(keyTwo.Bytes()), - StorageValue: valueTwo, + Address: addressTwo, + BlockHeight: int(blockOne), + BlockHash: headerHashBytes, + StorageKey: keyTwo, + StorageValue: valueTwo, } expectedDiffThree := types.RawDiff{ - BlockHeight: int(blockTwo), - BlockHash: common.HexToHash(blockTwoHeader.Hash), - HashedAddress: crypto.Keccak256Hash(addressOne[:]), - StorageKey: crypto.Keccak256Hash(keyOne.Bytes()), - StorageValue: valueTwo, + Address: addressOne, + BlockHeight: int(blockTwo), + BlockHash: common.HexToHash(blockTwoHeader.Hash), + StorageKey: keyOne, + StorageValue: valueTwo, } Expect(diffRepo.CreateBackFilledStorageValuePassedRawDiffs).To(ConsistOf(expectedDiffOne, expectedDiffTwo, expectedDiffThree)) diff --git a/libraries/shared/storage/diff_repository.go b/libraries/shared/storage/diff_repository.go index 0e43c38e5..79c47e64e 100644 --- a/libraries/shared/storage/diff_repository.go +++ b/libraries/shared/storage/diff_repository.go @@ -27,10 +27,22 @@ type DiffRepository interface { CreateStorageDiff(rawDiff types.RawDiff) (int64, error) CreateBackFilledStorageValue(rawDiff types.RawDiff) error GetNewDiffs(minID, limit int) ([]types.PersistedDiff, error) - MarkChecked(id int64) error + GetUnrecognizedDiffs(minID, limit int) ([]types.PersistedDiff, error) + MarkTransformed(id int64) error + MarkNoncanonical(id int64) error + MarkUnrecognized(id int64) error + MarkUnwatched(id int64) error GetFirstDiffIDForBlockHeight(blockHeight int64) (int64, error) } +var ( + New = `new` + Noncanonical = `noncanonical` + Transformed = `transformed` + Unrecognized = `unrecognized` + Unwatched = `unwatched` +) + type diffRepository struct { db *postgres.DB } @@ -43,9 +55,9 @@ func NewDiffRepository(db *postgres.DB) diffRepository { func (repository diffRepository) CreateStorageDiff(rawDiff types.RawDiff) (int64, error) { var storageDiffID int64 row := repository.db.QueryRowx(`INSERT INTO public.storage_diff - (hashed_address, block_height, block_hash, storage_key, storage_value) VALUES ($1, $2, $3, $4, $5) - ON CONFLICT DO NOTHING RETURNING id`, rawDiff.HashedAddress.Bytes(), rawDiff.BlockHeight, rawDiff.BlockHash.Bytes(), - rawDiff.StorageKey.Bytes(), rawDiff.StorageValue.Bytes()) + (address, block_height, block_hash, storage_key, storage_value, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT DO NOTHING RETURNING id`, rawDiff.Address.Bytes(), rawDiff.BlockHeight, rawDiff.BlockHash.Bytes(), + rawDiff.StorageKey.Bytes(), rawDiff.StorageValue.Bytes(), repository.db.NodeID) err := row.Scan(&storageDiffID) if err != nil { return 0, fmt.Errorf("error creating storage diff: %w", err) @@ -54,9 +66,9 @@ func (repository diffRepository) CreateStorageDiff(rawDiff types.RawDiff) (int64 } func (repository diffRepository) CreateBackFilledStorageValue(rawDiff types.RawDiff) error { - _, err := repository.db.Exec(`SELECT * FROM public.create_back_filled_diff($1, $2, $3, $4, $5)`, - rawDiff.BlockHeight, rawDiff.BlockHash.Bytes(), rawDiff.HashedAddress.Bytes(), - rawDiff.StorageKey.Bytes(), rawDiff.StorageValue.Bytes()) + _, err := repository.db.Exec(`SELECT * FROM public.create_back_filled_diff($1, $2, $3, $4, $5, $6)`, + rawDiff.BlockHeight, rawDiff.BlockHash.Bytes(), rawDiff.Address.Bytes(), + rawDiff.StorageKey.Bytes(), rawDiff.StorageValue.Bytes(), repository.db.NodeID) if err != nil { return fmt.Errorf("error creating back filled storage value: %w", err) } @@ -65,16 +77,60 @@ func (repository diffRepository) CreateBackFilledStorageValue(rawDiff types.RawD func (repository diffRepository) GetNewDiffs(minID, limit int) ([]types.PersistedDiff, error) { var result []types.PersistedDiff - query := fmt.Sprintf("SELECT * FROM public.storage_diff WHERE checked IS false and id > %d ORDER BY id ASC LIMIT %d", minID, limit) - err := repository.db.Select(&result, query) + err := repository.db.Select( + &result, + `SELECT id, address, block_height, block_hash, storage_key, storage_value, eth_node_id, status, from_backfill + FROM public.storage_diff + WHERE status = $1 AND id > $2 ORDER BY id ASC LIMIT $3`, + New, minID, limit, + ) if err != nil { - return nil, fmt.Errorf("error getting unchecked storage diffs with id greater than %d: %w", minID, err) + return nil, fmt.Errorf("error getting new storage diffs with id greater than %d: %w", minID, err) } return result, nil } -func (repository diffRepository) MarkChecked(id int64) error { - _, err := repository.db.Exec(`UPDATE public.storage_diff SET checked = true WHERE id = $1`, id) +func (repository diffRepository) GetUnrecognizedDiffs(minID, limit int) ([]types.PersistedDiff, error) { + var result []types.PersistedDiff + err := repository.db.Select( + &result, + `SELECT id, address, block_height, block_hash, storage_key, storage_value, eth_node_id, status, from_backfill + FROM public.storage_diff + WHERE status = $1 AND id > $2 ORDER BY id ASC LIMIT $3`, + Unrecognized, minID, limit, + ) + if err != nil { + return nil, fmt.Errorf("error getting unrecognized storage diffs with id greater than %d: %w", minID, err) + } + return result, nil +} + +func (repository diffRepository) MarkTransformed(id int64) error { + _, err := repository.db.Exec(`UPDATE public.storage_diff SET status = $1 WHERE id = $2`, Transformed, id) + if err != nil { + return fmt.Errorf("error marking diff %d transformed: %w", id, err) + } + return nil +} + +func (repository diffRepository) MarkUnrecognized(id int64) error { + _, err := repository.db.Exec(`UPDATE public.storage_diff SET status = $1 WHERE id = $2`, Unrecognized, id) + if err != nil { + return fmt.Errorf("error marking diff %d checked: %w", id, err) + } + return nil +} + +func (repository diffRepository) MarkNoncanonical(id int64) error { + _, err := repository.db.Exec(`UPDATE public.storage_diff SET status = $1 WHERE id = $2`, Noncanonical, id) + if err != nil { + return fmt.Errorf("error marking diff %d checked: %w", id, err) + } + return nil +} + +func (repository diffRepository) MarkUnwatched(id int64) error { + _, err := repository.db.Exec(`UPDATE public.storage_diff SET status = $1 WHERE id = $2`, Unwatched, id) if err != nil { return fmt.Errorf("error marking diff %d checked: %w", id, err) } diff --git a/libraries/shared/storage/diff_repository_test.go b/libraries/shared/storage/diff_repository_test.go index 570db99f4..497259a54 100644 --- a/libraries/shared/storage/diff_repository_test.go +++ b/libraries/shared/storage/diff_repository_test.go @@ -24,6 +24,7 @@ import ( "github.com/makerdao/vulcanizedb/libraries/shared/storage" "github.com/makerdao/vulcanizedb/libraries/shared/storage/types" "github.com/makerdao/vulcanizedb/libraries/shared/test_data" + "github.com/makerdao/vulcanizedb/pkg/datastore/postgres" "github.com/makerdao/vulcanizedb/test_config" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -40,14 +41,19 @@ var _ = Describe("Storage diffs repository", func() { test_config.CleanTestDB(db) repo = storage.NewDiffRepository(db) fakeStorageDiff = types.RawDiff{ - HashedAddress: test_data.FakeHash(), - BlockHash: test_data.FakeHash(), - BlockHeight: rand.Int(), - StorageKey: test_data.FakeHash(), - StorageValue: test_data.FakeHash(), + Address: test_data.FakeAddress(), + BlockHash: test_data.FakeHash(), + BlockHeight: rand.Int(), + StorageKey: test_data.FakeHash(), + StorageValue: test_data.FakeHash(), } }) + type dbStorageDiff struct { + Created string + Updated string + } + Describe("CreateStorageDiff", func() { It("adds a storage diff to the db, returning id", func() { id, createErr := repo.CreateStorageDiff(fakeStorageDiff) @@ -55,14 +61,15 @@ var _ = Describe("Storage diffs repository", func() { Expect(createErr).NotTo(HaveOccurred()) Expect(id).NotTo(BeZero()) var persisted types.PersistedDiff - getErr := db.Get(&persisted, `SELECT * FROM public.storage_diff`) + getErr := db.Get(&persisted, `SELECT id, address, block_hash, block_height, storage_key, storage_value, status FROM public.storage_diff`) Expect(getErr).NotTo(HaveOccurred()) Expect(persisted.ID).To(Equal(id)) - Expect(persisted.HashedAddress).To(Equal(fakeStorageDiff.HashedAddress)) + Expect(persisted.Address).To(Equal(fakeStorageDiff.Address)) Expect(persisted.BlockHash).To(Equal(fakeStorageDiff.BlockHash)) Expect(persisted.BlockHeight).To(Equal(fakeStorageDiff.BlockHeight)) Expect(persisted.StorageKey).To(Equal(fakeStorageDiff.StorageKey)) Expect(persisted.StorageValue).To(Equal(fakeStorageDiff.StorageValue)) + Expect(persisted.Status).To(Equal(storage.New)) }) It("does not duplicate storage diffs", func() { @@ -78,6 +85,22 @@ var _ = Describe("Storage diffs repository", func() { Expect(getErr).NotTo(HaveOccurred()) Expect(count).To(Equal(1)) }) + + It("indicates when a record was created or updated", func() { + id, createErr := repo.CreateStorageDiff(fakeStorageDiff) + Expect(createErr).NotTo(HaveOccurred()) + + var storageDiffUpdatedRes dbStorageDiff + initialStorageErr := db.Get(&storageDiffUpdatedRes, `SELECT created, updated FROM public.storage_diff`) + Expect(initialStorageErr).NotTo(HaveOccurred()) + Expect(storageDiffUpdatedRes.Created).To(Equal(storageDiffUpdatedRes.Updated)) + + _, updateErr := db.Exec(`UPDATE public.storage_diff SET block_hash = '{"new_block_hash"}' where id = $1`, id) + Expect(updateErr).NotTo(HaveOccurred()) + updatedDiffErr := db.Get(&storageDiffUpdatedRes, `SELECT created, updated FROM public.storage_diff`) + Expect(updatedDiffErr).NotTo(HaveOccurred()) + Expect(storageDiffUpdatedRes.Created).NotTo(Equal(storageDiffUpdatedRes.Updated)) + }) }) Describe("CreateBackFilledStorageValue", func() { @@ -86,13 +109,14 @@ var _ = Describe("Storage diffs repository", func() { Expect(createErr).NotTo(HaveOccurred()) var persisted types.PersistedDiff - getErr := db.Get(&persisted, `SELECT * FROM public.storage_diff`) + getErr := db.Get(&persisted, `SELECT address, block_hash, block_height, storage_key, storage_value, status FROM public.storage_diff`) Expect(getErr).NotTo(HaveOccurred()) - Expect(persisted.HashedAddress).To(Equal(fakeStorageDiff.HashedAddress)) + Expect(persisted.Address).To(Equal(fakeStorageDiff.Address)) Expect(persisted.BlockHash).To(Equal(fakeStorageDiff.BlockHash)) Expect(persisted.BlockHeight).To(Equal(fakeStorageDiff.BlockHeight)) Expect(persisted.StorageKey).To(Equal(fakeStorageDiff.StorageKey)) Expect(persisted.StorageValue).To(Equal(fakeStorageDiff.StorageValue)) + Expect(persisted.Status).To(Equal("new")) }) It("marks diff as back-filled", func() { @@ -191,23 +215,14 @@ var _ = Describe("Storage diffs repository", func() { }) Describe("GetNewDiffs", func() { - It("sends diffs that are not marked as checked", func() { - fakeRawDiff := types.RawDiff{ - HashedAddress: test_data.FakeHash(), - BlockHash: test_data.FakeHash(), - BlockHeight: rand.Int(), - StorageKey: test_data.FakeHash(), - StorageValue: test_data.FakeHash(), - } + It("sends diffs that are marked as 'new'", func() { fakePersistedDiff := types.PersistedDiff{ - RawDiff: fakeRawDiff, - ID: rand.Int63(), + RawDiff: fakeStorageDiff, + ID: rand.Int63(), + EthNodeID: db.NodeID, + Status: storage.New, } - _, insertErr := db.Exec(`INSERT INTO public.storage_diff (id, block_height, block_hash, - hashed_address, storage_key, storage_value) VALUES ($1, $2, $3, $4, $5, $6)`, fakePersistedDiff.ID, - fakeRawDiff.BlockHeight, fakeRawDiff.BlockHash.Bytes(), fakeRawDiff.HashedAddress.Bytes(), - fakeRawDiff.StorageKey.Bytes(), fakeRawDiff.StorageValue.Bytes()) - Expect(insertErr).NotTo(HaveOccurred()) + insertTestDiff(fakePersistedDiff, db) diffs, err := repo.GetNewDiffs(0, 1) @@ -215,25 +230,44 @@ var _ = Describe("Storage diffs repository", func() { Expect(diffs).To(ConsistOf(fakePersistedDiff)) }) - It("does not send diff that's marked as checked", func() { - fakeRawDiff := types.RawDiff{ - HashedAddress: test_data.FakeHash(), - BlockHash: test_data.FakeHash(), - BlockHeight: rand.Int(), - StorageKey: test_data.FakeHash(), - StorageValue: test_data.FakeHash(), + It("does not sends diffs that are marked as 'unrecognized'", func() { + unrecognizedPersistedDiff := types.PersistedDiff{ + RawDiff: fakeStorageDiff, + ID: rand.Int63(), + Status: storage.Unrecognized, + EthNodeID: db.NodeID, } - fakePersistedDiff := types.PersistedDiff{ - RawDiff: fakeRawDiff, - ID: rand.Int63(), - Checked: true, + insertTestDiff(unrecognizedPersistedDiff, db) + + diffs, err := repo.GetNewDiffs(0, 1) + + Expect(err).NotTo(HaveOccurred()) + Expect(diffs).To(BeEmpty()) + }) + + It("does not send diffs that are marked as 'transformed'", func() { + transformedPersistedDiff := types.PersistedDiff{ + RawDiff: fakeStorageDiff, + ID: rand.Int63(), + Status: storage.Transformed, + EthNodeID: db.NodeID, + } + insertTestDiff(transformedPersistedDiff, db) + + diffs, err := repo.GetNewDiffs(0, 1) + + Expect(err).NotTo(HaveOccurred()) + Expect(diffs).To(BeEmpty()) + }) + + It("does not send diffs that are marked as 'noncanonical'", func() { + noncanonicalPersistedDiff := types.PersistedDiff{ + RawDiff: fakeStorageDiff, + ID: rand.Int63(), + Status: storage.Noncanonical, + EthNodeID: db.NodeID, } - _, insertErr := db.Exec(`INSERT INTO public.storage_diff (id, block_height, block_hash, - hashed_address, storage_key, storage_value, checked) VALUES ($1, $2, $3, $4, $5, $6, $7)`, - fakePersistedDiff.ID, fakeRawDiff.BlockHeight, fakeRawDiff.BlockHash.Bytes(), - fakeRawDiff.HashedAddress.Bytes(), fakeRawDiff.StorageKey.Bytes(), fakeRawDiff.StorageValue.Bytes(), - fakePersistedDiff.Checked) - Expect(insertErr).NotTo(HaveOccurred()) + insertTestDiff(noncanonicalPersistedDiff, db) diffs, err := repo.GetNewDiffs(0, 1) @@ -245,17 +279,19 @@ var _ = Describe("Storage diffs repository", func() { blockZero := rand.Int() for i := 0; i < 2; i++ { fakeRawDiff := types.RawDiff{ - HashedAddress: test_data.FakeHash(), - BlockHash: test_data.FakeHash(), - BlockHeight: blockZero + i, - StorageKey: test_data.FakeHash(), - StorageValue: test_data.FakeHash(), + Address: test_data.FakeAddress(), + BlockHash: test_data.FakeHash(), + BlockHeight: blockZero + i, + StorageKey: test_data.FakeHash(), + StorageValue: test_data.FakeHash(), + } + persistedDiff := types.PersistedDiff{ + RawDiff: fakeRawDiff, + ID: rand.Int63(), + Status: storage.New, + EthNodeID: db.NodeID, } - _, insertErr := db.Exec(`INSERT INTO public.storage_diff (block_height, block_hash, - hashed_address, storage_key, storage_value) VALUES ($1, $2, $3, $4, $5)`, fakeRawDiff.BlockHeight, - fakeRawDiff.BlockHash.Bytes(), fakeRawDiff.HashedAddress.Bytes(), fakeRawDiff.StorageKey.Bytes(), - fakeRawDiff.StorageValue.Bytes()) - Expect(insertErr).NotTo(HaveOccurred()) + insertTestDiff(persistedDiff, db) } minID := 0 @@ -271,34 +307,149 @@ var _ = Describe("Storage diffs repository", func() { }) }) - Describe("MarkChecked", func() { - It("marks a diff as checked", func() { - fakeRawDiff := types.RawDiff{ - HashedAddress: test_data.FakeHash(), - BlockHash: test_data.FakeHash(), - BlockHeight: rand.Int(), - StorageKey: test_data.FakeHash(), - StorageValue: test_data.FakeHash(), - } + Describe("GetUnrecognizedDiffs", func() { + It("sends diffs that are marked as 'unrecognized'", func() { fakePersistedDiff := types.PersistedDiff{ - RawDiff: fakeRawDiff, - ID: rand.Int63(), - Checked: false, + RawDiff: fakeStorageDiff, + ID: rand.Int63(), + EthNodeID: db.NodeID, + Status: "unrecognized", } - _, insertErr := db.Exec(`INSERT INTO public.storage_diff (id, block_height, block_hash, - hashed_address, storage_key, storage_value, checked) VALUES ($1, $2, $3, $4, $5, $6, $7)`, - fakePersistedDiff.ID, fakeRawDiff.BlockHeight, fakeRawDiff.BlockHash.Bytes(), - fakeRawDiff.HashedAddress.Bytes(), fakeRawDiff.StorageKey.Bytes(), fakeRawDiff.StorageValue.Bytes(), - fakePersistedDiff.Checked) - Expect(insertErr).NotTo(HaveOccurred()) + insertTestDiff(fakePersistedDiff, db) - err := repo.MarkChecked(fakePersistedDiff.ID) + diffs, err := repo.GetUnrecognizedDiffs(0, 1) Expect(err).NotTo(HaveOccurred()) - var checked bool - checkedErr := db.Get(&checked, `SELECT checked FROM public.storage_diff WHERE id = $1`, fakePersistedDiff.ID) - Expect(checkedErr).NotTo(HaveOccurred()) - Expect(checked).To(BeTrue()) + Expect(diffs).To(ConsistOf(fakePersistedDiff)) + }) + + It("does not sends diffs that are marked as 'new'", func() { + unrecognizedPersistedDiff := types.PersistedDiff{ + RawDiff: fakeStorageDiff, + ID: rand.Int63(), + Status: "new", + EthNodeID: db.NodeID, + } + insertTestDiff(unrecognizedPersistedDiff, db) + + diffs, err := repo.GetUnrecognizedDiffs(0, 1) + + Expect(err).NotTo(HaveOccurred()) + Expect(diffs).To(BeEmpty()) + }) + + It("does not send diffs that are marked as 'transformed'", func() { + transformedPersistedDiff := types.PersistedDiff{ + RawDiff: fakeStorageDiff, + ID: rand.Int63(), + Status: "transformed", + EthNodeID: db.NodeID, + } + insertTestDiff(transformedPersistedDiff, db) + + diffs, err := repo.GetUnrecognizedDiffs(0, 1) + + Expect(err).NotTo(HaveOccurred()) + Expect(diffs).To(BeEmpty()) + }) + + It("does not send diffs that are marked as 'noncanonical'", func() { + noncanonicalPersistedDiff := types.PersistedDiff{ + RawDiff: fakeStorageDiff, + ID: rand.Int63(), + Status: "noncanonical", + EthNodeID: db.NodeID, + } + insertTestDiff(noncanonicalPersistedDiff, db) + + diffs, err := repo.GetUnrecognizedDiffs(0, 1) + + Expect(err).NotTo(HaveOccurred()) + Expect(diffs).To(BeEmpty()) + }) + + It("enables seeking diffs with greater ID", func() { + blockZero := rand.Int() + for i := 0; i < 2; i++ { + fakeRawDiff := types.RawDiff{ + Address: test_data.FakeAddress(), + BlockHash: test_data.FakeHash(), + BlockHeight: blockZero + i, + StorageKey: test_data.FakeHash(), + StorageValue: test_data.FakeHash(), + } + persistedDiff := types.PersistedDiff{ + RawDiff: fakeRawDiff, + ID: rand.Int63(), + Status: "new", + EthNodeID: db.NodeID, + } + insertTestDiff(persistedDiff, db) + } + + minID := 0 + limit := 1 + diffsOne, errOne := repo.GetNewDiffs(minID, limit) + Expect(errOne).NotTo(HaveOccurred()) + Expect(len(diffsOne)).To(Equal(1)) + nextID := int(diffsOne[0].ID) + diffsTwo, errTwo := repo.GetNewDiffs(nextID, limit) + Expect(errTwo).NotTo(HaveOccurred()) + Expect(len(diffsTwo)).To(Equal(1)) + Expect(int(diffsTwo[0].ID) > nextID).To(BeTrue()) + }) + }) + + Describe("Changing the diff status", func() { + var fakePersistedDiff types.PersistedDiff + BeforeEach(func() { + fakePersistedDiff = types.PersistedDiff{ + RawDiff: fakeStorageDiff, + ID: rand.Int63(), + Status: storage.New, + EthNodeID: db.NodeID, + } + insertTestDiff(fakePersistedDiff, db) + }) + + It("marks a diff as transformed", func() { + err := repo.MarkTransformed(fakePersistedDiff.ID) + + Expect(err).NotTo(HaveOccurred()) + var status string + getStatusErr := db.Get(&status, `SELECT status FROM public.storage_diff WHERE id = $1`, fakePersistedDiff.ID) + Expect(getStatusErr).NotTo(HaveOccurred()) + Expect(status).To(Equal(storage.Transformed)) + }) + + It("marks a diff as unrecognized", func() { + err := repo.MarkUnrecognized(fakePersistedDiff.ID) + + Expect(err).NotTo(HaveOccurred()) + var status string + getStatusErr := db.Get(&status, `SELECT status FROM public.storage_diff WHERE id = $1`, fakePersistedDiff.ID) + Expect(getStatusErr).NotTo(HaveOccurred()) + Expect(status).To(Equal(storage.Unrecognized)) + }) + + It("marks a diff as noncanonical", func() { + err := repo.MarkNoncanonical(fakePersistedDiff.ID) + + Expect(err).NotTo(HaveOccurred()) + var status string + getStatusErr := db.Get(&status, `SELECT status FROM public.storage_diff WHERE id = $1`, fakePersistedDiff.ID) + Expect(getStatusErr).NotTo(HaveOccurred()) + Expect(status).To(Equal(storage.Noncanonical)) + }) + + It("marks a diff as unwatched", func() { + err := repo.MarkUnwatched(fakePersistedDiff.ID) + + Expect(err).NotTo(HaveOccurred()) + var status string + getStatusErr := db.Get(&status, `SELECT status FROM public.storage_diff WHERE id = $1`, fakePersistedDiff.ID) + Expect(getStatusErr).NotTo(HaveOccurred()) + Expect(status).To(Equal(storage.Unwatched)) }) }) @@ -306,11 +457,11 @@ var _ = Describe("Storage diffs repository", func() { It("sends first diff for a given block height", func() { blockHeight := fakeStorageDiff.BlockHeight fakeStorageDiff2 := types.RawDiff{ - HashedAddress: test_data.FakeHash(), - BlockHash: test_data.FakeHash(), - BlockHeight: blockHeight, - StorageKey: test_data.FakeHash(), - StorageValue: test_data.FakeHash(), + Address: test_data.FakeAddress(), + BlockHash: test_data.FakeHash(), + BlockHeight: blockHeight, + StorageKey: test_data.FakeHash(), + StorageValue: test_data.FakeHash(), } id1, create1Err := repo.CreateStorageDiff(fakeStorageDiff) @@ -326,11 +477,11 @@ var _ = Describe("Storage diffs repository", func() { It("sends a diff for the next block height if one doesn't exist for the block passed in", func() { blockHeight := fakeStorageDiff.BlockHeight fakeStorageDiff2 := types.RawDiff{ - HashedAddress: test_data.FakeHash(), - BlockHash: test_data.FakeHash(), - BlockHeight: blockHeight, - StorageKey: test_data.FakeHash(), - StorageValue: test_data.FakeHash(), + Address: test_data.FakeAddress(), + BlockHash: test_data.FakeHash(), + BlockHeight: blockHeight, + StorageKey: test_data.FakeHash(), + StorageValue: test_data.FakeHash(), } id1, create1Err := repo.CreateStorageDiff(fakeStorageDiff) @@ -345,30 +496,20 @@ var _ = Describe("Storage diffs repository", func() { }) It("won't fail if all of the diffs within the id range are already checked", func() { - fakeRawDiff := types.RawDiff{ - HashedAddress: test_data.FakeHash(), - BlockHash: test_data.FakeHash(), - BlockHeight: rand.Int(), - StorageKey: test_data.FakeHash(), - StorageValue: test_data.FakeHash(), - } fakePersistedDiff := types.PersistedDiff{ - RawDiff: fakeRawDiff, - ID: rand.Int63(), - Checked: true, + RawDiff: fakeStorageDiff, + ID: rand.Int63(), + Status: storage.Transformed, + EthNodeID: db.NodeID, } - _, insertErr := db.Exec(`INSERT INTO public.storage_diff (id, block_height, block_hash, - hashed_address, storage_key, storage_value, checked) VALUES ($1, $2, $3, $4, $5, $6, $7)`, - fakePersistedDiff.ID, fakeRawDiff.BlockHeight, fakeRawDiff.BlockHash.Bytes(), - fakeRawDiff.HashedAddress.Bytes(), fakeRawDiff.StorageKey.Bytes(), fakeRawDiff.StorageValue.Bytes(), - fakePersistedDiff.Checked) - Expect(insertErr).NotTo(HaveOccurred()) + + insertTestDiff(fakePersistedDiff, db) var insertedDiffID int64 getInsertedDiffIDErr := db.Get(&insertedDiffID, `SELECT id FROM storage_diff LIMIT 1`) Expect(getInsertedDiffIDErr).NotTo(HaveOccurred()) - blockBeforeDiffBlockHeight := int64(fakeRawDiff.BlockHeight - 1) + blockBeforeDiffBlockHeight := int64(fakeStorageDiff.BlockHeight - 1) diffID, diffErr := repo.GetFirstDiffIDForBlockHeight(blockBeforeDiffBlockHeight) Expect(diffErr).NotTo(HaveOccurred()) Expect(diffID).To(Equal(insertedDiffID)) @@ -381,3 +522,13 @@ var _ = Describe("Storage diffs repository", func() { }) }) }) + +func insertTestDiff(persistedDiff types.PersistedDiff, db *postgres.DB) { + rawDiff := persistedDiff.RawDiff + _, insertErr := db.Exec(`INSERT INTO public.storage_diff (id, block_height, block_hash, + address, storage_key, storage_value, status, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, + persistedDiff.ID, rawDiff.BlockHeight, rawDiff.BlockHash.Bytes(), + rawDiff.Address.Bytes(), rawDiff.StorageKey.Bytes(), rawDiff.StorageValue.Bytes(), + persistedDiff.Status, persistedDiff.EthNodeID) + Expect(insertErr).NotTo(HaveOccurred()) +} diff --git a/libraries/shared/storage/extractor.go b/libraries/shared/storage/extractor.go index e8c708f31..bc9e1d2e9 100644 --- a/libraries/shared/storage/extractor.go +++ b/libraries/shared/storage/extractor.go @@ -48,8 +48,8 @@ func (extractor DiffExtractor) persistDiff(rawDiff types.RawDiff) { _, err := extractor.StorageDiffRepository.CreateStorageDiff(rawDiff) if err != nil { if errors.Is(err, sql.ErrNoRows) { - logrus.Tracef("ignoring duplicate diff. Block number: %v, blockHash: %v, hashedAddress: %v, storageKey: %v, storageValue: %v", - rawDiff.BlockHeight, rawDiff.BlockHash.Hex(), rawDiff.HashedAddress, rawDiff.StorageKey, rawDiff.StorageValue) + logrus.Tracef("ignoring duplicate diff. Block number: %v, blockHash: %v, address: %v, storageKey: %v, storageValue: %v", + rawDiff.BlockHeight, rawDiff.BlockHash.Hex(), rawDiff.Address, rawDiff.StorageKey, rawDiff.StorageValue) return } logrus.Warnf("failed to persist storage diff: %s", err.Error()) diff --git a/libraries/shared/storage/extractor_test.go b/libraries/shared/storage/extractor_test.go index ad78ee5be..76d809860 100644 --- a/libraries/shared/storage/extractor_test.go +++ b/libraries/shared/storage/extractor_test.go @@ -48,11 +48,11 @@ var _ = Describe("Storage diff extractor", func() { It("persists fetched storage diff", func() { fakeDiff := types.RawDiff{ - HashedAddress: test_data.FakeHash(), - BlockHash: test_data.FakeHash(), - BlockHeight: rand.Int(), - StorageKey: test_data.FakeHash(), - StorageValue: test_data.FakeHash(), + Address: test_data.FakeAddress(), + BlockHash: test_data.FakeHash(), + BlockHeight: rand.Int(), + StorageKey: test_data.FakeHash(), + StorageValue: test_data.FakeHash(), } mockFetcher.DiffsToReturn = []types.RawDiff{fakeDiff} mockFetcher.ErrsToReturn = []error{fakes.FakeError} diff --git a/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher.go b/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher.go index 8b09c5b8a..615a0ff10 100644 --- a/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher.go +++ b/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher.go @@ -17,37 +17,34 @@ package fetcher import ( "fmt" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/statediff" "github.com/makerdao/vulcanizedb/libraries/shared/storage/types" "github.com/makerdao/vulcanizedb/libraries/shared/streamer" "github.com/makerdao/vulcanizedb/pkg/fs" "github.com/sirupsen/logrus" ) -type GethPatchVersion int - -const ( - OldGethPatch GethPatchVersion = iota - NewGethPatch -) - type GethRpcStorageFetcher struct { - statediffPayloadChan chan statediff.Payload + statediffPayloadChan chan filters.Payload streamer streamer.Streamer - gethVersion GethPatchVersion statusWriter fs.StatusWriter } -func NewGethRpcStorageFetcher(streamer streamer.Streamer, statediffPayloadChan chan statediff.Payload, gethVersion GethPatchVersion, statusWriter fs.StatusWriter) GethRpcStorageFetcher { +func NewGethRpcStorageFetcher(streamer streamer.Streamer, statediffPayloadChan chan filters.Payload, statusWriter fs.StatusWriter) GethRpcStorageFetcher { return GethRpcStorageFetcher{ statediffPayloadChan: statediffPayloadChan, streamer: streamer, - gethVersion: gethVersion, statusWriter: statusWriter, } } +var ( + processingDiffsLogString = "processing %d storage diffs for account %s" + addingDiffsLogString = "adding storage diff to out channel. keccak of address: %v, block height: %v, storage key: %v, storage value: %v" +) + func (fetcher GethRpcStorageFetcher) FetchStorageDiffs(out chan<- types.RawDiff, errs chan<- error) { ethStatediffPayloadChan := fetcher.statediffPayloadChan clientSubscription, clientSubErr := fetcher.streamer.Stream(ethStatediffPayloadChan) @@ -67,45 +64,32 @@ func (fetcher GethRpcStorageFetcher) FetchStorageDiffs(out chan<- types.RawDiff, case err := <-clientSubscription.Err(): logrus.Errorf("error with client subscription: %s", err.Error()) errs <- err - case diff := <-ethStatediffPayloadChan: - logrus.Trace("received a statediff") - stateDiff := new(statediff.StateDiff) - decodeErr := rlp.DecodeBytes(diff.StateDiffRlp, stateDiff) - logrus.Tracef("received a statediff from block: %v", stateDiff.BlockNumber) - if decodeErr != nil { - logrus.Warn("Error decoding state diff into RLP: ", decodeErr) - errs <- decodeErr - } - - accounts := getAccountsFromDiff(*stateDiff) - logrus.Trace(fmt.Sprintf("iterating through %d accounts on stateDiff for block %d", len(accounts), stateDiff.BlockNumber)) - for _, account := range accounts { - logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account", len(account.Storage))) - for _, accountStorage := range account.Storage { - diff, formatErr := fetcher.formatDiff(account, stateDiff, accountStorage) - logrus.Tracef("adding storage diff to out channel. keccak of address: %v, block height: %v, storage key: %v, storage value: %v", - diff.HashedAddress.Hex(), diff.BlockHeight, diff.StorageKey.Hex(), diff.StorageValue.Hex()) - if formatErr != nil { - errs <- formatErr - } - - out <- diff - } - } + case diffPayload := <-ethStatediffPayloadChan: + logrus.Trace("received a statediff payload") + fetcher.handleDiffPayload(diffPayload, out, errs) } - } } -func (fetcher GethRpcStorageFetcher) formatDiff(account statediff.AccountDiff, stateDiff *statediff.StateDiff, storage statediff.StorageDiff) (types.RawDiff, error) { - if fetcher.gethVersion == OldGethPatch { - return types.FromOldGethStateDiff(account, stateDiff, storage) - } else { - return types.FromNewGethStateDiff(account, stateDiff, storage) +func (fetcher GethRpcStorageFetcher) handleDiffPayload(payload filters.Payload, out chan<- types.RawDiff, errs chan<- error) { + var stateDiff filters.StateDiff + decodeErr := rlp.DecodeBytes(payload.StateDiffRlp, &stateDiff) + if decodeErr != nil { + errs <- fmt.Errorf("error decoding storage diff from geth payload: %w", decodeErr) + return } -} -func getAccountsFromDiff(stateDiff statediff.StateDiff) []statediff.AccountDiff { - accounts := append(stateDiff.CreatedAccounts, stateDiff.UpdatedAccounts...) - return append(accounts, stateDiff.DeletedAccounts...) + for _, account := range stateDiff.UpdatedAccounts { + logrus.Infof(processingDiffsLogString, len(account.Storage), common.Bytes2Hex(account.Key)) + for _, accountStorage := range account.Storage { + rawDiff, formatErr := types.FromGethStateDiff(account, &stateDiff, accountStorage) + if formatErr != nil { + errs <- formatErr + return + } + + logrus.Tracef(addingDiffsLogString, rawDiff.Address.Hex(), rawDiff.BlockHeight, rawDiff.StorageKey.Hex(), rawDiff.StorageValue.Hex()) + out <- rawDiff + } + } } diff --git a/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher_test.go b/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher_test.go index 0364122de..60a097e59 100644 --- a/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher_test.go +++ b/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher_test.go @@ -15,10 +15,12 @@ package fetcher_test import ( + "fmt" + "io" + "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/statediff" "github.com/makerdao/vulcanizedb/libraries/shared/mocks" "github.com/makerdao/vulcanizedb/libraries/shared/storage/fetcher" "github.com/makerdao/vulcanizedb/libraries/shared/storage/types" @@ -31,156 +33,27 @@ import ( var _ = Describe("Geth RPC Storage Fetcher", func() { var ( streamer *mocks.MockStoragediffStreamer - statediffPayloadChan chan statediff.Payload + statediffPayloadChan chan filters.Payload statediffFetcher fetcher.GethRpcStorageFetcher storagediffChan chan types.RawDiff subscription *fakes.MockSubscription errorChan chan error statusWriter fakes.MockStatusWriter + stateDiffPayloads []filters.Payload + badStateDiffPayloads = []filters.Payload{{}} //This empty payload is "bad" because it does not contain the required StateDiffRlp ) - Describe("StorageFetcher for the Old Geth Patch", func() { - BeforeEach(func() { - subscription = &fakes.MockSubscription{Errs: make(chan error)} - streamer = &mocks.MockStoragediffStreamer{ClientSubscription: subscription} - statediffPayloadChan = make(chan statediff.Payload, 1) - statusWriter = fakes.MockStatusWriter{} - statediffFetcher = fetcher.NewGethRpcStorageFetcher(streamer, statediffPayloadChan, fetcher.OldGethPatch, &statusWriter) - storagediffChan = make(chan types.RawDiff) - errorChan = make(chan error) - }) - - It("adds errors to errors channel if the streamer fails to subscribe", func(done Done) { - streamer.SetSubscribeError(fakes.FakeError) - - go func() { - failedSub := func() { - statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan) - } - Expect(failedSub).To(Panic()) - }() - - Expect(<-errorChan).To(MatchError(fakes.FakeError)) - close(done) - }) - - It("streams StatediffPayloads from a Geth RPC subscription", func(done Done) { - streamer.SetPayloads([]statediff.Payload{test_data.MockStatediffPayload}) - - go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan) - - streamedPayload := <-statediffPayloadChan - Expect(streamedPayload).To(Equal(test_data.MockStatediffPayload)) - Expect(streamer.PassedPayloadChan).To(Equal(statediffPayloadChan)) - close(done) - }) - - Describe("when subscription established", func() { - It("creates file for health check when connection established", func(done Done) { - go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan) - - Eventually(func() bool { - return statusWriter.WriteCalled - }).Should(BeTrue()) - close(done) - }) - - It("adds error to errors channel if the subscription fails", func(done Done) { - go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan) - - subscription.Errs <- fakes.FakeError - - Expect(<-errorChan).To(MatchError(fakes.FakeError)) - close(done) - }) - - It("adds errors to error channel if decoding the state diff RLP fails", func(done Done) { - badStatediffPayload := statediff.Payload{} - streamer.SetPayloads([]statediff.Payload{badStatediffPayload}) - - go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan) - - Expect(<-errorChan).To(MatchError("EOF")) - - close(done) - }) - - It("adds parsed statediff payloads to the out channel for the old geth patch", func(done Done) { - streamer.SetPayloads([]statediff.Payload{test_data.MockStatediffPayload}) - - go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan) - - height := test_data.BlockNumber - intHeight := int(height.Int64()) - createdExpectedStorageDiff := types.RawDiff{ - HashedAddress: common.BytesToHash(test_data.ContractLeafKey[:]), - BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), - BlockHeight: intHeight, - StorageKey: common.BytesToHash(test_data.StorageKey), - StorageValue: common.BytesToHash(test_data.SmallStorageValue), - } - updatedExpectedStorageDiff := types.RawDiff{ - HashedAddress: common.BytesToHash(test_data.AnotherContractLeafKey[:]), - BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), - BlockHeight: intHeight, - StorageKey: common.BytesToHash(test_data.StorageKey), - StorageValue: common.BytesToHash(test_data.LargeStorageValue), - } - deletedExpectedStorageDiff := types.RawDiff{ - HashedAddress: common.BytesToHash(test_data.AnotherContractLeafKey[:]), - BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), - BlockHeight: intHeight, - StorageKey: common.BytesToHash(test_data.StorageKey), - StorageValue: common.BytesToHash(test_data.SmallStorageValue), - } - - createdStateDiff := <-storagediffChan - updatedStateDiff := <-storagediffChan - deletedStateDiff := <-storagediffChan - - Expect(createdStateDiff).To(Equal(createdExpectedStorageDiff)) - Expect(updatedStateDiff).To(Equal(updatedExpectedStorageDiff)) - Expect(deletedStateDiff).To(Equal(deletedExpectedStorageDiff)) - - close(done) - }) - - It("adds errors to error channel if formatting the diff as a StateDiff object fails", func(done Done) { - accountDiffs := test_data.CreatedAccountDiffs - accountDiffs[0].Storage = []statediff.StorageDiff{test_data.StorageWithBadValue} - - stateDiff := statediff.StateDiff{ - BlockNumber: test_data.BlockNumber, - BlockHash: common.HexToHash(test_data.BlockHash), - CreatedAccounts: accountDiffs, - } - - stateDiffRlp, err := rlp.EncodeToBytes(stateDiff) - Expect(err).NotTo(HaveOccurred()) - - badStatediffPayload := statediff.Payload{ - StateDiffRlp: stateDiffRlp, - } - streamer.SetPayloads([]statediff.Payload{badStatediffPayload}) - - go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan) - - Expect(<-errorChan).To(MatchError("rlp: input contains more than one value")) - - close(done) - }) - }) - }) - Describe("StorageFetcher for the New Geth Patch", func() { + // This tests fetching diff payloads from the updated simplified geth patch: https://github.com/makerdao/go-ethereum/tree/allow-state-diff-subscription + // - diffs are formatted with the FromGethStateDiff method BeforeEach(func() { subscription = &fakes.MockSubscription{Errs: make(chan error)} streamer = &mocks.MockStoragediffStreamer{ClientSubscription: subscription} - statediffPayloadChan = make(chan statediff.Payload, 1) - statusWriter = fakes.MockStatusWriter{} - statediffFetcher = fetcher.NewGethRpcStorageFetcher(streamer, statediffPayloadChan, fetcher.NewGethPatch, &statusWriter) + statediffPayloadChan = make(chan filters.Payload, 1) + statediffFetcher = fetcher.NewGethRpcStorageFetcher(streamer, statediffPayloadChan, &statusWriter) storagediffChan = make(chan types.RawDiff) errorChan = make(chan error) + stateDiffPayloads = []filters.Payload{test_data.MockStatediffPayload} }) It("adds errors to errors channel if the streamer fails to subscribe", func(done Done) { @@ -198,7 +71,7 @@ var _ = Describe("Geth RPC Storage Fetcher", func() { }) It("streams StatediffPayloads from a Geth RPC subscription", func(done Done) { - streamer.SetPayloads([]statediff.Payload{test_data.MockStatediffPayload}) + streamer.SetPayloads(stateDiffPayloads) go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan) @@ -228,43 +101,43 @@ var _ = Describe("Geth RPC Storage Fetcher", func() { }) It("adds errors to error channel if decoding the state diff RLP fails", func(done Done) { - badStatediffPayload := statediff.Payload{} - streamer.SetPayloads([]statediff.Payload{badStatediffPayload}) + streamer.SetPayloads(badStateDiffPayloads) go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan) - Expect(<-errorChan).To(MatchError("EOF")) - + expectedErr := fmt.Errorf("error decoding storage diff from geth payload: %w", io.EOF) + Expect(<-errorChan).To(MatchError(expectedErr)) close(done) }) - It("adds parsed statediff payloads to the out channel for the new geth patch", func(done Done) { - streamer.SetPayloads([]statediff.Payload{test_data.MockStatediffPayload}) + It("adds parsed statediff payloads to the out channel", func(done Done) { + streamer.SetPayloads(stateDiffPayloads) go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan) height := test_data.BlockNumber intHeight := int(height.Int64()) + expectedDiff1 := types.RawDiff{ - HashedAddress: crypto.Keccak256Hash(test_data.ContractLeafKey[:]), - BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), - BlockHeight: intHeight, - StorageKey: crypto.Keccak256Hash(test_data.StorageKey), - StorageValue: common.BytesToHash(test_data.SmallStorageValue), + Address: common.BytesToAddress(test_data.ContractLeafKey[:]), + BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), + BlockHeight: intHeight, + StorageKey: common.BytesToHash(test_data.StorageKey), + StorageValue: common.BytesToHash(test_data.SmallStorageValue), } expectedDiff2 := types.RawDiff{ - HashedAddress: crypto.Keccak256Hash(test_data.AnotherContractLeafKey[:]), - BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), - BlockHeight: intHeight, - StorageKey: crypto.Keccak256Hash(test_data.StorageKey), - StorageValue: common.BytesToHash(test_data.LargeStorageValue), + Address: common.BytesToAddress(test_data.AnotherContractLeafKey[:]), + BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), + BlockHeight: intHeight, + StorageKey: common.BytesToHash(test_data.StorageKey), + StorageValue: common.BytesToHash(test_data.LargeStorageValue), } expectedDiff3 := types.RawDiff{ - HashedAddress: crypto.Keccak256Hash(test_data.AnotherContractLeafKey[:]), - BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), - BlockHeight: intHeight, - StorageKey: crypto.Keccak256Hash(test_data.StorageKey), - StorageValue: common.BytesToHash(test_data.SmallStorageValue), + Address: common.BytesToAddress(test_data.AnotherContractLeafKey[:]), + BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), + BlockHeight: intHeight, + StorageKey: common.BytesToHash(test_data.StorageKey), + StorageValue: common.BytesToHash(test_data.SmallStorageValue), } diff1 := <-storagediffChan @@ -279,26 +152,16 @@ var _ = Describe("Geth RPC Storage Fetcher", func() { }) It("adds errors to error channel if formatting the diff as a StateDiff object fails", func(done Done) { - accountDiffs := test_data.CreatedAccountDiffs - accountDiffs[0].Storage = []statediff.StorageDiff{test_data.StorageWithBadValue} - - stateDiff := statediff.StateDiff{ - BlockNumber: test_data.BlockNumber, - BlockHash: common.HexToHash(test_data.BlockHash), - CreatedAccounts: accountDiffs, - } - + stateDiff := test_data.StateDiffWithBadStorageValue stateDiffRlp, err := rlp.EncodeToBytes(stateDiff) Expect(err).NotTo(HaveOccurred()) + payloadToReturn := filters.Payload{StateDiffRlp: stateDiffRlp} - badStatediffPayload := statediff.Payload{ - StateDiffRlp: stateDiffRlp, - } - streamer.SetPayloads([]statediff.Payload{badStatediffPayload}) + streamer.SetPayloads([]filters.Payload{payloadToReturn}) go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan) - Expect(<-errorChan).To(MatchError("rlp: input contains more than one value")) + Expect(<-errorChan).To(MatchError(rlp.ErrMoreThanOneValue)) close(done) }) diff --git a/libraries/shared/storage/keys_lookup.go b/libraries/shared/storage/keys_lookup.go deleted file mode 100644 index 1344b94bf..000000000 --- a/libraries/shared/storage/keys_lookup.go +++ /dev/null @@ -1,38 +0,0 @@ -// VulcanizeDB -// Copyright © 2019 Vulcanize - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. - -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package storage - -import ( - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" - "github.com/makerdao/vulcanizedb/libraries/shared/storage/types" -) - -func AddHashedKeys(currentMappings map[common.Hash]types.ValueMetadata) map[common.Hash]types.ValueMetadata { - copyOfCurrentMappings := make(map[common.Hash]types.ValueMetadata) - for k, v := range currentMappings { - copyOfCurrentMappings[k] = v - } - for k, v := range copyOfCurrentMappings { - currentMappings[hashKey(k)] = v - } - return currentMappings -} - -func hashKey(key common.Hash) common.Hash { - return crypto.Keccak256Hash(key.Bytes()) -} diff --git a/libraries/shared/storage/keys_lookup_test.go b/libraries/shared/storage/keys_lookup_test.go deleted file mode 100644 index 76d80768c..000000000 --- a/libraries/shared/storage/keys_lookup_test.go +++ /dev/null @@ -1,48 +0,0 @@ -// VulcanizeDB -// Copyright © 2019 Vulcanize - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. - -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package storage_test - -import ( - "github.com/ethereum/go-ethereum/common" - "github.com/makerdao/vulcanizedb/libraries/shared/storage" - "github.com/makerdao/vulcanizedb/libraries/shared/storage/types" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -var _ = Describe("Storage keys lookup utils", func() { - Describe("AddHashedKeys", func() { - It("returns a copy of the map with an additional slot for the hashed version of every key", func() { - fakeMap := map[common.Hash]types.ValueMetadata{} - fakeStorageKey := common.HexToHash("72c72de6b203d67cb6cd54fc93300109fcc6fd6eac88e390271a3d548794d800") - var fakeMappingKey types.Key = "fakeKey" - fakeMetadata := types.ValueMetadata{ - Name: "fakeName", - Keys: map[types.Key]string{fakeMappingKey: "fakeValue"}, - Type: types.Uint48, - } - fakeMap[fakeStorageKey] = fakeMetadata - - result := storage.AddHashedKeys(fakeMap) - - Expect(len(result)).To(Equal(2)) - expectedHashedStorageKey := common.HexToHash("2165edb4e1c37b99b60fa510d84f939dd35d5cd1d1c8f299d6456ea09df65a76") - Expect(fakeMap[fakeStorageKey]).To(Equal(fakeMetadata)) - Expect(fakeMap[expectedHashedStorageKey]).To(Equal(fakeMetadata)) - }) - }) -}) diff --git a/libraries/shared/storage/types/diff.go b/libraries/shared/storage/types/diff.go index 9353cf641..183875b4d 100644 --- a/libraries/shared/storage/types/diff.go +++ b/libraries/shared/storage/types/diff.go @@ -21,26 +21,27 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/statediff" ) const ExpectedRowLength = 5 type RawDiff struct { - HashedAddress common.Hash `db:"hashed_address"` - BlockHash common.Hash `db:"block_hash"` - BlockHeight int `db:"block_height"` - StorageKey common.Hash `db:"storage_key"` - StorageValue common.Hash `db:"storage_value"` + Address common.Address `db:"address"` + BlockHash common.Hash `db:"block_hash"` + BlockHeight int `db:"block_height"` + StorageKey common.Hash `db:"storage_key"` + StorageValue common.Hash `db:"storage_value"` } type PersistedDiff struct { RawDiff - Checked bool + Status string FromBackfill bool `db:"from_backfill"` ID int64 HeaderID int64 `db:"header_id"` + EthNodeID int64 `db:"eth_node_id"` } func FromParityCsvRow(csvRow []string) (RawDiff, error) { @@ -52,15 +53,15 @@ func FromParityCsvRow(csvRow []string) (RawDiff, error) { return RawDiff{}, err } return RawDiff{ - HashedAddress: HexToKeccak256Hash(csvRow[0]), - BlockHash: common.HexToHash(csvRow[1]), - BlockHeight: height, - StorageKey: common.HexToHash(csvRow[3]), - StorageValue: common.HexToHash(csvRow[4]), + Address: common.HexToAddress(csvRow[0]), + BlockHash: common.HexToHash(csvRow[1]), + BlockHeight: height, + StorageKey: common.HexToHash(csvRow[3]), + StorageValue: common.HexToHash(csvRow[4]), }, nil } -func FromOldGethStateDiff(account statediff.AccountDiff, stateDiff *statediff.StateDiff, storage statediff.StorageDiff) (RawDiff, error) { +func FromGethStateDiff(account filters.AccountDiff, stateDiff *filters.StateDiff, storage filters.StorageDiff) (RawDiff, error) { var decodedRLPStorageValue []byte err := rlp.DecodeBytes(storage.Value, &decodedRLPStorageValue) if err != nil { @@ -68,37 +69,14 @@ func FromOldGethStateDiff(account statediff.AccountDiff, stateDiff *statediff.St } return RawDiff{ - HashedAddress: common.BytesToHash(account.Key), - BlockHash: stateDiff.BlockHash, - BlockHeight: int(stateDiff.BlockNumber.Int64()), - StorageKey: common.BytesToHash(storage.Key), - StorageValue: common.BytesToHash(decodedRLPStorageValue), + Address: common.BytesToAddress(account.Key), + BlockHash: stateDiff.BlockHash, + BlockHeight: int(stateDiff.BlockNumber.Int64()), + StorageKey: common.BytesToHash(storage.Key), + StorageValue: common.BytesToHash(decodedRLPStorageValue), }, nil } -func FromNewGethStateDiff(account statediff.AccountDiff, stateDiff *statediff.StateDiff, storage statediff.StorageDiff) (RawDiff, error) { - var decodedRLPStorageValue []byte - err := rlp.DecodeBytes(storage.Value, &decodedRLPStorageValue) - if err != nil { - return RawDiff{}, err - } - - return RawDiff{ - HashedAddress: crypto.Keccak256Hash(account.Key), - BlockHash: stateDiff.BlockHash, - BlockHeight: int(stateDiff.BlockNumber.Int64()), - StorageKey: crypto.Keccak256Hash(storage.Key), - StorageValue: common.BytesToHash(decodedRLPStorageValue), - }, nil -} - -func ToPersistedDiff(raw RawDiff, id int64) PersistedDiff { - return PersistedDiff{ - RawDiff: raw, - ID: id, - } -} - func HexToKeccak256Hash(hex string) common.Hash { return crypto.Keccak256Hash(common.FromHex(hex)) } diff --git a/libraries/shared/storage/types/diff_test.go b/libraries/shared/storage/types/diff_test.go index 7860045cf..10db6af06 100644 --- a/libraries/shared/storage/types/diff_test.go +++ b/libraries/shared/storage/types/diff_test.go @@ -21,9 +21,8 @@ import ( "math/rand" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/statediff" "github.com/makerdao/vulcanizedb/libraries/shared/storage/types" "github.com/makerdao/vulcanizedb/libraries/shared/test_data" "github.com/makerdao/vulcanizedb/pkg/fakes" @@ -44,8 +43,7 @@ var _ = Describe("Storage row parsing", func() { result, err := types.FromParityCsvRow(data) Expect(err).NotTo(HaveOccurred()) - expectedKeccakOfContractAddress := types.HexToKeccak256Hash(contract) - Expect(result.HashedAddress).To(Equal(expectedKeccakOfContractAddress)) + Expect(result.Address).To(Equal(common.HexToAddress(contract))) Expect(result.BlockHash).To(Equal(common.HexToHash(blockHash))) Expect(result.BlockHeight).To(Equal(789)) Expect(result.StorageKey).To(Equal(common.HexToHash(storageKey))) @@ -66,10 +64,10 @@ var _ = Describe("Storage row parsing", func() { }) }) - Describe("FromOldGethStateDiff", func() { + Describe("FromGethStateDiff", func() { var ( - accountDiff = statediff.AccountDiff{Key: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}} - stateDiff = &statediff.StateDiff{ + accountDiff = filters.AccountDiff{Key: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}} + stateDiff = &filters.StateDiff{ BlockNumber: big.NewInt(rand.Int63()), BlockHash: fakes.FakeHash, } @@ -80,16 +78,16 @@ var _ = Describe("Storage row parsing", func() { storageValueRlp, encodeErr := rlp.EncodeToBytes(storageValueBytes) Expect(encodeErr).NotTo(HaveOccurred()) - storageDiff := statediff.StorageDiff{ + storageDiff := filters.StorageDiff{ Key: []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1}, Value: storageValueRlp, } - result, err := types.FromOldGethStateDiff(accountDiff, stateDiff, storageDiff) + result, err := types.FromGethStateDiff(accountDiff, stateDiff, storageDiff) Expect(err).NotTo(HaveOccurred()) - expectedAddress := common.BytesToHash(accountDiff.Key) - Expect(result.HashedAddress).To(Equal(expectedAddress)) + expectedAddress := common.BytesToAddress(accountDiff.Key) + Expect(result.Address).To(Equal(expectedAddress)) Expect(result.BlockHash).To(Equal(fakes.FakeHash)) expectedBlockHeight := int(stateDiff.BlockNumber.Int64()) Expect(result.BlockHeight).To(Equal(expectedBlockHeight)) @@ -104,73 +102,18 @@ var _ = Describe("Storage row parsing", func() { storageValueRlp, encodeErr := rlp.EncodeToBytes(storageValueBytes) Expect(encodeErr).NotTo(HaveOccurred()) - storageDiff := statediff.StorageDiff{ + storageDiff := filters.StorageDiff{ Key: []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1}, Value: storageValueRlp, } - result, err := types.FromOldGethStateDiff(accountDiff, stateDiff, storageDiff) + result, err := types.FromGethStateDiff(accountDiff, stateDiff, storageDiff) Expect(err).NotTo(HaveOccurred()) Expect(result.StorageValue).To(Equal(common.BytesToHash(storageValueBytes))) }) It("returns an err if decoding the storage value Rlp fails", func() { - _, err := types.FromOldGethStateDiff(accountDiff, stateDiff, test_data.StorageWithBadValue) - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError("rlp: input contains more than one value")) - }) - }) - - Describe("FromNewGethStateDiff", func() { - var ( - accountDiff = statediff.AccountDiff{Key: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}} - stateDiff = &statediff.StateDiff{ - BlockNumber: big.NewInt(rand.Int63()), - BlockHash: fakes.FakeHash, - } - ) - - It("adds relevant fields to diff", func() { - storageValueBytes := []byte{3} - storageValueRlp, encodeErr := rlp.EncodeToBytes(storageValueBytes) - Expect(encodeErr).NotTo(HaveOccurred()) - - storageDiff := statediff.StorageDiff{ - Key: []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1}, - Value: storageValueRlp, - } - - result, err := types.FromNewGethStateDiff(accountDiff, stateDiff, storageDiff) - Expect(err).NotTo(HaveOccurred()) - - expectedHashedAddress := crypto.Keccak256Hash(accountDiff.Key) - Expect(result.HashedAddress).To(Equal(expectedHashedAddress)) - Expect(result.BlockHash).To(Equal(fakes.FakeHash)) - expectedBlockHeight := int(stateDiff.BlockNumber.Int64()) - Expect(result.BlockHeight).To(Equal(expectedBlockHeight)) - expectedStorageKey := crypto.Keccak256Hash(storageDiff.Key) - Expect(result.StorageKey).To(Equal(expectedStorageKey)) - expectedStorageValue := common.BytesToHash(storageValueBytes) - Expect(result.StorageValue).To(Equal(expectedStorageValue)) - }) - - It("handles decoding large storage values from their RLP", func() { - storageValueBytes := []byte{1, 2, 3, 4, 5, 0, 9, 8, 7, 6} - storageValueRlp, encodeErr := rlp.EncodeToBytes(storageValueBytes) - Expect(encodeErr).NotTo(HaveOccurred()) - - storageDiff := statediff.StorageDiff{ - Key: []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1}, - Value: storageValueRlp, - } - - result, err := types.FromNewGethStateDiff(accountDiff, stateDiff, storageDiff) - Expect(err).NotTo(HaveOccurred()) - Expect(result.StorageValue).To(Equal(common.BytesToHash(storageValueBytes))) - }) - - It("returns an err if decoding the storage value Rlp fails", func() { - _, err := types.FromNewGethStateDiff(accountDiff, stateDiff, test_data.StorageWithBadValue) + _, err := types.FromGethStateDiff(accountDiff, stateDiff, test_data.StorageWithBadValue) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError("rlp: input contains more than one value")) }) diff --git a/libraries/shared/streamer/statediff_streamer.go b/libraries/shared/streamer/state_change_streamer.go similarity index 53% rename from libraries/shared/streamer/statediff_streamer.go rename to libraries/shared/streamer/state_change_streamer.go index 130cea22f..7089794f2 100644 --- a/libraries/shared/streamer/statediff_streamer.go +++ b/libraries/shared/streamer/state_change_streamer.go @@ -15,26 +15,31 @@ package streamer import ( - "github.com/ethereum/go-ethereum/statediff" + "context" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/makerdao/vulcanizedb/pkg/core" "github.com/sirupsen/logrus" ) type Streamer interface { - Stream(chan statediff.Payload) (core.Subscription, error) + Stream(chan filters.Payload) (core.Subscription, error) } -type StateDiffStreamer struct { - client core.RpcClient +type EthStateChangeStreamer struct { + ethClient core.EthClient + filterQuery ethereum.FilterQuery } -func (streamer *StateDiffStreamer) Stream(payloadChan chan statediff.Payload) (core.Subscription, error) { - logrus.Info("streaming diffs from geth") - return streamer.client.Subscribe("statediff", payloadChan, "stream") +func NewEthStateChangeStreamer(ethClient core.EthClient, filterQuery ethereum.FilterQuery) EthStateChangeStreamer { + return EthStateChangeStreamer{ + ethClient: ethClient, + filterQuery: filterQuery, + } } -func NewStateDiffStreamer(client core.RpcClient) StateDiffStreamer { - return StateDiffStreamer{ - client: client, - } +func (streamer *EthStateChangeStreamer) Stream(payloadChan chan filters.Payload) (core.Subscription, error) { + logrus.Info("streaming diffs from geth") + return streamer.ethClient.SubscribeNewStateChanges(context.Background(), streamer.filterQuery, payloadChan) } diff --git a/libraries/shared/streamer/statediff_streamer_test.go b/libraries/shared/streamer/state_change_streamer_test.go similarity index 60% rename from libraries/shared/streamer/statediff_streamer_test.go rename to libraries/shared/streamer/state_change_streamer_test.go index 3a6c8e6b3..468791962 100644 --- a/libraries/shared/streamer/statediff_streamer_test.go +++ b/libraries/shared/streamer/state_change_streamer_test.go @@ -15,21 +15,26 @@ package streamer_test import ( - "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/makerdao/vulcanizedb/libraries/shared/streamer" "github.com/makerdao/vulcanizedb/pkg/fakes" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) -var _ = Describe("StateDiff Streamer", func() { - It("subscribes to the geth statediff service", func() { - client := &fakes.MockRpcClient{} - streamer := streamer.NewStateDiffStreamer(client) - payloadChan := make(chan statediff.Payload) +var _ = Describe("State Change Streamer", func() { + It("subscribes to the geth state change subscription", func() { + ethClient := &fakes.MockEthClient{} + filterQuery := ethereum.FilterQuery{ + Addresses: []common.Address{fakes.FakeAddress}, + } + streamer := streamer.NewEthStateChangeStreamer(ethClient, filterQuery) + payloadChan := make(chan filters.Payload) _, err := streamer.Stream(payloadChan) Expect(err).NotTo(HaveOccurred()) - client.AssertSubscribeCalledWith("statediff", payloadChan, []interface{}{"stream"}) + ethClient.AssertSubscribeNewStateChangesCalledWith(filterQuery, payloadChan) }) }) diff --git a/libraries/shared/test_data/statediff.go b/libraries/shared/test_data/statediff.go index e51a51077..256c82a8c 100644 --- a/libraries/shared/test_data/statediff.go +++ b/libraries/shared/test_data/statediff.go @@ -20,10 +20,9 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/statediff" ) var ( @@ -37,22 +36,18 @@ var ( StorageKey = common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes() SmallStorageValue = common.Hex2Bytes("03") SmallStorageValueRlp, _ = rlp.EncodeToBytes(SmallStorageValue) - storageWithSmallValue = []statediff.StorageDiff{{ + storageWithSmallValue = []filters.StorageDiff{{ Key: StorageKey, Value: SmallStorageValueRlp, - Path: StoragePath, - Proof: [][]byte{}, }} LargeStorageValue = common.Hex2Bytes("00191b53778c567b14b50ba0000") LargeStorageValueRlp, rlpErr = rlp.EncodeToBytes(LargeStorageValue) - storageWithLargeValue = []statediff.StorageDiff{{ + storageWithLargeValue = []filters.StorageDiff{{ Key: StorageKey, Value: LargeStorageValueRlp, - Path: StoragePath, - Proof: [][]byte{}, }} - EmptyStorage = make([]statediff.StorageDiff, 0) - StorageWithBadValue = statediff.StorageDiff{ + EmptyStorage = make([]filters.StorageDiff, 0) + StorageWithBadValue = filters.StorageDiff{ Key: StorageKey, Value: []byte{0, 1, 2}, // this storage value will fail to be decoded as an RLP with the following error message: @@ -69,56 +64,49 @@ var ( Root: ContractRoot, CodeHash: CodeHash, } - valueBytes, _ = rlp.EncodeToBytes(testAccount) - CreatedAccountDiffs = []statediff.AccountDiff{ - { - Key: ContractLeafKey.Bytes(), - Value: valueBytes, - Storage: storageWithSmallValue, - }, + AccountValueBytes, _ = rlp.EncodeToBytes(testAccount) + testAccountDiff1 = filters.AccountDiff{ + Key: ContractLeafKey.Bytes(), + Value: AccountValueBytes, + Storage: storageWithSmallValue, } - - UpdatedAccountDiffs = []statediff.AccountDiff{{ + testAccountDiff2 = filters.AccountDiff{ Key: AnotherContractLeafKey.Bytes(), - Value: valueBytes, + Value: AccountValueBytes, Storage: storageWithLargeValue, - }} - - DeletedAccountDiffs = []statediff.AccountDiff{{ + } + testAccountDiff3 = filters.AccountDiff{ Key: AnotherContractLeafKey.Bytes(), - Value: valueBytes, + Value: AccountValueBytes, Storage: storageWithSmallValue, - }} + } + UpdatedAccountDiffs = []filters.AccountDiff{testAccountDiff1, testAccountDiff2, testAccountDiff3} - MockStateDiff = statediff.StateDiff{ + MockStateDiff = filters.StateDiff{ BlockNumber: BlockNumber, BlockHash: common.HexToHash(BlockHash), - CreatedAccounts: CreatedAccountDiffs, - DeletedAccounts: DeletedAccountDiffs, UpdatedAccounts: UpdatedAccountDiffs, } MockStateDiffBytes, _ = rlp.EncodeToBytes(MockStateDiff) - mockTransaction1 = types.NewTransaction(0, common.HexToAddress("0x0"), big.NewInt(1000), 50, big.NewInt(100), nil) - mockTransaction2 = types.NewTransaction(1, common.HexToAddress("0x1"), big.NewInt(2000), 100, big.NewInt(200), nil) - MockTransactions = types.Transactions{mockTransaction1, mockTransaction2} - - mockReceipt1 = types.NewReceipt(common.HexToHash("0x0").Bytes(), false, 50) - mockReceipt2 = types.NewReceipt(common.HexToHash("0x1").Bytes(), false, 100) - MockReceipts = types.Receipts{mockReceipt1, mockReceipt2} - - MockHeader = types.Header{ - Time: 0, - Number: BlockNumber, - Root: common.HexToHash("0x0"), - TxHash: common.HexToHash("0x0"), - ReceiptHash: common.HexToHash("0x0"), + MockStatediffPayload = filters.Payload{ + StateDiffRlp: MockStateDiffBytes, } - MockBlock = types.NewBlock(&MockHeader, MockTransactions, nil, MockReceipts) - MockBlockRlp, _ = rlp.EncodeToBytes(MockBlock) - MockStatediffPayload = statediff.Payload{ - BlockRlp: MockBlockRlp, - StateDiffRlp: MockStateDiffBytes, + storageWithBadValue = filters.StorageDiff{ + Key: StorageKey, + Value: []byte{0, 1, 2}, + // this storage value will fail to be decoded as an RLP with the following error message: + // "rlp: input contains more than one value" + } + testAccountDiffWithBadStorageValue = filters.AccountDiff{ + Key: ContractLeafKey.Bytes(), + Value: AccountValueBytes, + Storage: []filters.StorageDiff{storageWithBadValue}, + } + StateDiffWithBadStorageValue = filters.StateDiff{ + BlockNumber: BlockNumber, + BlockHash: common.HexToHash(BlockHash), + UpdatedAccounts: []filters.AccountDiff{testAccountDiffWithBadStorageValue}, } ) diff --git a/libraries/shared/test_data/test_helpers.go b/libraries/shared/test_data/test_helpers.go index 254f70268..1abde2548 100644 --- a/libraries/shared/test_data/test_helpers.go +++ b/libraries/shared/test_data/test_helpers.go @@ -11,6 +11,7 @@ import ( "github.com/makerdao/vulcanizedb/pkg/datastore" "github.com/makerdao/vulcanizedb/pkg/datastore/postgres" "github.com/makerdao/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/makerdao/vulcanizedb/pkg/fakes" . "github.com/onsi/gomega" ) @@ -88,3 +89,38 @@ func getRandomAddress() string { stringHash := hashToPrefixedString(hash) return stringHash[:42] } + +/// ExpectCheckedHeadersInThisSchema is provided so that +/// plugins can easily validate that they have the necessary checked_headers +/// table in their schema for the execute process. +/// +/// Use like so in your tests: +/// var _ = Describe("Your Schema", func() { +/// BeforeEach(func() { +/// test_config.CleanTestDB(db) +/// }) +/// +/// It("has a proper checked headers setup in the schema", func() { +/// test_data.ExpectCheckedHeadersInThisSchema(db, "yourschemaname") +/// }) +/// }) +func ExpectCheckedHeadersInThisSchema(db *postgres.DB, schema string) { + Expect(db).NotTo(BeNil()) + // insert header + blockNumber := rand.Int63() + fakeHeader := fakes.GetFakeHeader(blockNumber) + headerRepo := repositories.NewHeaderRepository(db) + headerID, headerErr := headerRepo.CreateOrUpdateHeader(fakeHeader) + Expect(headerErr).NotTo(HaveOccurred()) + + checkedHeaderRepo, repoErr := repositories.NewCheckedHeadersRepository(db, schema) + Expect(repoErr).NotTo(HaveOccurred()) + uncheckedHeaders, uncheckedHeaderErr := checkedHeaderRepo.UncheckedHeaders(0, -1, 1) + Expect(uncheckedHeaderErr).NotTo(HaveOccurred()) + Expect(len(uncheckedHeaders)).To(Equal(1)) + Expect(uncheckedHeaders[0].BlockNumber).To(Equal(blockNumber)) + + Expect(checkedHeaderRepo.MarkHeaderChecked(headerID)).To(Succeed()) + + Expect(checkedHeaderRepo.UncheckedHeaders(0, -1, 1)).To(BeEmpty()) +} diff --git a/libraries/shared/watcher/storage_watcher.go b/libraries/shared/watcher/storage_watcher.go index f30406f6b..f2f42e839 100644 --- a/libraries/shared/watcher/storage_watcher.go +++ b/libraries/shared/watcher/storage_watcher.go @@ -46,30 +46,39 @@ type IStorageWatcher interface { type StorageWatcher struct { db *postgres.DB HeaderRepository datastore.HeaderRepository - KeccakAddressTransformers map[common.Hash]storage2.ITransformer // keccak hash of an address => transformer + AddressTransformers map[common.Address]storage2.ITransformer // contract address => transformer StorageDiffRepository storage.DiffRepository DiffBlocksFromHeadOfChain int64 // the number of blocks from the head of the chain where diffs should be processed StatusWriter fs.StatusWriter + DiffStatus DiffStatusToWatch } -func NewStorageWatcher(db *postgres.DB, backFromHeadOfChain int64, statusWriter fs.StatusWriter) StorageWatcher { +type DiffStatusToWatch int + +const ( + New DiffStatusToWatch = iota + Unrecognized +) + +func NewStorageWatcher(db *postgres.DB, backFromHeadOfChain int64, statusWriter fs.StatusWriter, diffStatusToWatch DiffStatusToWatch) StorageWatcher { headerRepository := repositories.NewHeaderRepository(db) storageDiffRepository := storage.NewDiffRepository(db) - transformers := make(map[common.Hash]storage2.ITransformer) + transformers := make(map[common.Address]storage2.ITransformer) return StorageWatcher{ db: db, HeaderRepository: headerRepository, - KeccakAddressTransformers: transformers, + AddressTransformers: transformers, StorageDiffRepository: storageDiffRepository, DiffBlocksFromHeadOfChain: backFromHeadOfChain, StatusWriter: statusWriter, + DiffStatus: diffStatusToWatch, } } func (watcher StorageWatcher) AddTransformers(initializers []storage2.TransformerInitializer) { for _, initializer := range initializers { storageTransformer := initializer(watcher.db) - watcher.KeccakAddressTransformers[storageTransformer.KeccakContractAddress()] = storageTransformer + watcher.AddressTransformers[storageTransformer.GetContractAddress()] = storageTransformer } } @@ -88,26 +97,14 @@ func (watcher StorageWatcher) Execute() error { } } -func (watcher StorageWatcher) getMinDiffID() (int, error) { - var minID = 0 - if watcher.DiffBlocksFromHeadOfChain != -1 { - mostRecentHeaderBlockNumber, getHeaderErr := watcher.HeaderRepository.GetMostRecentHeaderBlockNumber() - if getHeaderErr != nil { - return 0, fmt.Errorf("error getting most recent header block number: %w", getHeaderErr) - } - blockNumber := mostRecentHeaderBlockNumber - watcher.DiffBlocksFromHeadOfChain - diffID, getDiffErr := watcher.StorageDiffRepository.GetFirstDiffIDForBlockHeight(blockNumber) - if getDiffErr != nil { - return 0, fmt.Errorf("error getting first diff id for block height %d: %w", blockNumber, getDiffErr) - } - - // We are subtracting an offset from the diffID because it will be passed to GetNewDiffs which returns diffs with ids - // greater than id passed in (minID), and we want to make sure that this diffID here is included in that collection - diffOffset := int64(1) - minID = int(diffID - diffOffset) +func (watcher StorageWatcher) getDiffs(minID, ResultsLimit int) ([]types.PersistedDiff, error) { + switch watcher.DiffStatus { + case New: + return watcher.StorageDiffRepository.GetNewDiffs(minID, ResultsLimit) + case Unrecognized: + return watcher.StorageDiffRepository.GetUnrecognizedDiffs(minID, ResultsLimit) } - - return minID, nil + return nil, errors.New("Unrecognized diff status") } func (watcher StorageWatcher) transformDiffs() error { @@ -117,18 +114,15 @@ func (watcher StorageWatcher) transformDiffs() error { } for { - diffs, extractErr := watcher.StorageDiffRepository.GetNewDiffs(minID, ResultsLimit) + diffs, extractErr := watcher.getDiffs(minID, ResultsLimit) + if extractErr != nil { - return fmt.Errorf("error getting unchecked diffs: %w", extractErr) + return fmt.Errorf("error getting new diffs: %w", extractErr) } for _, diff := range diffs { transformErr := watcher.transformDiff(diff) - if transformErr != nil { - if isCommonTransformError(transformErr) { - logrus.Tracef("error transforming diff: %s", transformErr.Error()) - } else { - logrus.Infof("error transforming diff: %s", transformErr.Error()) - } + if handleErr := watcher.handleTransformError(transformErr, diff); handleErr != nil { + return fmt.Errorf("error transforming diff: %w", handleErr) } } lenDiffs := len(diffs) @@ -141,12 +135,33 @@ func (watcher StorageWatcher) transformDiffs() error { } } +func (watcher StorageWatcher) getMinDiffID() (int, error) { + var minID = 0 + if watcher.DiffBlocksFromHeadOfChain != -1 { + mostRecentHeaderBlockNumber, getHeaderErr := watcher.HeaderRepository.GetMostRecentHeaderBlockNumber() + if getHeaderErr != nil { + return 0, fmt.Errorf("error getting most recent header block number: %w", getHeaderErr) + } + blockNumber := mostRecentHeaderBlockNumber - watcher.DiffBlocksFromHeadOfChain + diffID, getDiffErr := watcher.StorageDiffRepository.GetFirstDiffIDForBlockHeight(blockNumber) + if getDiffErr != nil { + return 0, fmt.Errorf("error getting first diff id for block height %d: %w", blockNumber, getDiffErr) + } + + // We are subtracting an offset from the diffID because it will be passed to GetNewDiffs which returns diffs with ids + // greater than id passed in (minID), and we want to make sure that this diffID here is included in that collection + diffOffset := int64(1) + minID = int(diffID - diffOffset) + } + + return minID, nil +} func (watcher StorageWatcher) transformDiff(diff types.PersistedDiff) error { t, watching := watcher.getTransformer(diff) if !watching { - markCheckedErr := watcher.StorageDiffRepository.MarkChecked(diff.ID) - if markCheckedErr != nil { - return fmt.Errorf("error marking diff checked: %w", markCheckedErr) + markUnwatchedErr := watcher.StorageDiffRepository.MarkUnwatched(diff.ID) + if markUnwatchedErr != nil { + return fmt.Errorf("error marking diff %s: %w", storage.Unwatched, markUnwatchedErr) } return nil } @@ -165,16 +180,16 @@ func (watcher StorageWatcher) transformDiff(diff types.PersistedDiff) error { return fmt.Errorf("error executing storage transformer: %w", executeErr) } - markCheckedErr := watcher.StorageDiffRepository.MarkChecked(diff.ID) - if markCheckedErr != nil { - return fmt.Errorf("error marking diff checked: %w", markCheckedErr) + markTransformedErr := watcher.StorageDiffRepository.MarkTransformed(diff.ID) + if markTransformedErr != nil { + return fmt.Errorf("error marking diff %s: %w", storage.Transformed, markTransformedErr) } return nil } func (watcher StorageWatcher) getTransformer(diff types.PersistedDiff) (storage2.ITransformer, bool) { - storageTransformer, ok := watcher.KeccakAddressTransformers[diff.HashedAddress] + storageTransformer, ok := watcher.AddressTransformers[diff.Address] return storageTransformer, ok } @@ -198,11 +213,28 @@ func (watcher StorageWatcher) handleDiffWithInvalidHeaderHash(diff types.Persist return fmt.Errorf(msg, diff.ID, maxBlockErr) } if diff.BlockHeight < int(maxBlock)-ReorgWindow { - return watcher.StorageDiffRepository.MarkChecked(diff.ID) + return watcher.StorageDiffRepository.MarkNoncanonical(diff.ID) } return nil } +func (watcher StorageWatcher) handleTransformError(transformErr error, diff types.PersistedDiff) error { + if transformErr != nil { + if errors.Is(transformErr, types.ErrKeyNotFound) { + markUnrecognizedErr := watcher.StorageDiffRepository.MarkUnrecognized(diff.ID) + if markUnrecognizedErr != nil { + return markUnrecognizedErr + } + } + if isCommonTransformError(transformErr) { + logrus.Tracef("error transforming diff: %s", transformErr.Error()) + } else { + logrus.Infof("error transforming diff: %s", transformErr.Error()) + } + } + return transformErr +} + func isCommonTransformError(err error) bool { return errors.Is(err, sql.ErrNoRows) || errors.Is(err, types.ErrKeyNotFound) } diff --git a/libraries/shared/watcher/storage_watcher_test.go b/libraries/shared/watcher/storage_watcher_test.go index 7a6d10929..a174d3bf0 100644 --- a/libraries/shared/watcher/storage_watcher_test.go +++ b/libraries/shared/watcher/storage_watcher_test.go @@ -37,47 +37,83 @@ var _ = Describe("Storage Watcher", func() { var statusWriter fakes.MockStatusWriter Describe("AddTransformer", func() { It("adds transformers", func() { - fakeHashedAddress := types.HexToKeccak256Hash("0x12345") - fakeTransformer := &mocks.MockStorageTransformer{KeccakOfAddress: fakeHashedAddress} - w := watcher.NewStorageWatcher(test_config.NewTestDB(test_config.NewTestNode()), -1, &statusWriter) + fakeAddress := fakes.FakeAddress + fakeTransformer := &mocks.MockStorageTransformer{Address: fakeAddress} + w := watcher.NewStorageWatcher(test_config.NewTestDB(test_config.NewTestNode()), -1, &statusWriter, watcher.New) w.AddTransformers([]storage.TransformerInitializer{fakeTransformer.FakeTransformerInitializer}) - Expect(w.KeccakAddressTransformers[fakeHashedAddress]).To(Equal(fakeTransformer)) + Expect(w.AddressTransformers[fakeAddress]).To(Equal(fakeTransformer)) }) }) Describe("Execute", func() { - var ( - storageWatcher watcher.StorageWatcher - mockDiffsRepository *mocks.MockStorageDiffRepository - mockHeaderRepository *fakes.MockHeaderRepository - ) - - BeforeEach(func() { - mockDiffsRepository = &mocks.MockStorageDiffRepository{} - mockHeaderRepository = &fakes.MockHeaderRepository{} - storageWatcher = watcher.NewStorageWatcher(test_config.NewTestDB(test_config.NewTestNode()), -1, &statusWriter) - storageWatcher.HeaderRepository = mockHeaderRepository - storageWatcher.StorageDiffRepository = mockDiffsRepository + When("a watcher is configured to watches 'new' storage diffs", func() { + statusWriter := fakes.MockStatusWriter{} + storageWatcher := watcher.NewStorageWatcher(test_config.NewTestDB(test_config.NewTestNode()), -1, &statusWriter, watcher.New) + input := ExecuteInput{ + watcher: &storageWatcher, + statusWriter: &statusWriter, + } + SharedExecuteBehavior(&input) + }) + + When("a watcher is configured to watches 'unrecognized' storage diffs", func() { + statusWriter := fakes.MockStatusWriter{} + storageWatcher := watcher.NewStorageWatcher(test_config.NewTestDB(test_config.NewTestNode()), -1, &statusWriter, watcher.Unrecognized) + input := ExecuteInput{ + watcher: &storageWatcher, + statusWriter: &statusWriter, + } + SharedExecuteBehavior(&input) }) + }) +}) + +type ExecuteInput struct { + watcher *watcher.StorageWatcher + statusWriter *fakes.MockStatusWriter +} + +func SharedExecuteBehavior(input *ExecuteInput) { + var ( + mockDiffsRepository *mocks.MockStorageDiffRepository + mockHeaderRepository *fakes.MockHeaderRepository + statusWriter = input.statusWriter + storageWatcher = input.watcher + contractAddress common.Address + mockTransformer *mocks.MockStorageTransformer + ) + + BeforeEach(func() { + mockDiffsRepository = &mocks.MockStorageDiffRepository{} + mockHeaderRepository = &fakes.MockHeaderRepository{} + contractAddress = test_data.FakeAddress() + mockTransformer = &mocks.MockStorageTransformer{Address: contractAddress} + storageWatcher.HeaderRepository = mockHeaderRepository + storageWatcher.StorageDiffRepository = mockDiffsRepository + storageWatcher.AddTransformers([]storage.TransformerInitializer{mockTransformer.FakeTransformerInitializer}) + }) + Describe("Execute", func() { It("creates file for health check", func() { - mockDiffsRepository.GetNewDiffsErrors = []error{fakes.FakeError} + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{fakes.FakeError}) err := storageWatcher.Execute() Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) Expect(statusWriter.WriteCalled).To(BeTrue()) }) It("fetches diffs with results limit", func() { - mockDiffsRepository.GetNewDiffsErrors = []error{fakes.FakeError} + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{fakes.FakeError}) err := storageWatcher.Execute() Expect(err).To(HaveOccurred()) - Expect(mockDiffsRepository.GetNewDiffsPassedLimits).To(ConsistOf(watcher.ResultsLimit)) + Expect(err).To(MatchError(fakes.FakeError)) + assertGetDiffsLimits(storageWatcher.DiffStatus, mockDiffsRepository, watcher.ResultsLimit) }) It("fetches diffs with min ID from subsequent queries when previous query returns max results", func() { @@ -87,19 +123,20 @@ var _ = Describe("Storage Watcher", func() { diffID = diffID + i diff := types.PersistedDiff{ RawDiff: types.RawDiff{ - HashedAddress: test_data.FakeHash(), + Address: test_data.FakeAddress(), }, ID: int64(diffID), } diffs = append(diffs, diff) } - mockDiffsRepository.GetNewDiffsDiffs = diffs - mockDiffsRepository.GetNewDiffsErrors = []error{nil, fakes.FakeError} + setDiffsToReturn(storageWatcher.DiffStatus, mockDiffsRepository, diffs) + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{nil, fakes.FakeError}) err := storageWatcher.Execute() Expect(err).To(HaveOccurred()) - Expect(mockDiffsRepository.GetNewDiffsPassedMinIDs).To(ConsistOf(0, diffID)) + Expect(err).To(MatchError(fakes.FakeError)) + assertGetDiffsMinIDs(storageWatcher.DiffStatus, mockDiffsRepository, []int{0, diffID}) }) It("resets min ID to zero when previous query returns fewer than max results", func() { @@ -109,36 +146,58 @@ var _ = Describe("Storage Watcher", func() { diffID = diffID + i diff := types.PersistedDiff{ RawDiff: types.RawDiff{ - HashedAddress: test_data.FakeHash(), + Address: test_data.FakeAddress(), }, ID: int64(diffID), } diffs = append(diffs, diff) } - mockDiffsRepository.GetNewDiffsDiffs = diffs - mockDiffsRepository.GetNewDiffsErrors = []error{nil, fakes.FakeError} + setDiffsToReturn(storageWatcher.DiffStatus, mockDiffsRepository, diffs) + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{nil, fakes.FakeError}) err := storageWatcher.Execute() Expect(err).To(HaveOccurred()) - Expect(mockDiffsRepository.GetNewDiffsPassedMinIDs).To(ConsistOf(0, 0)) + Expect(err).To(MatchError(fakes.FakeError)) + assertGetDiffsMinIDs(storageWatcher.DiffStatus, mockDiffsRepository, []int{0, 0}) }) - It("marks diff as checked if no transformer is watching its address", func() { + It("marks diff as 'unwatched' if no transformer is watching its address", func() { + unwatchedAddress := test_data.FakeAddress() unwatchedDiff := types.PersistedDiff{ RawDiff: types.RawDiff{ - HashedAddress: test_data.FakeHash(), + Address: unwatchedAddress, }, ID: rand.Int63(), } - mockDiffsRepository.GetNewDiffsDiffs = []types.PersistedDiff{unwatchedDiff} - mockDiffsRepository.GetNewDiffsErrors = []error{nil, fakes.FakeError} + setDiffsToReturn(storageWatcher.DiffStatus, mockDiffsRepository, []types.PersistedDiff{unwatchedDiff}) + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{nil, fakes.FakeError}) err := storageWatcher.Execute() Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) - Expect(mockDiffsRepository.MarkCheckedPassedID).To(Equal(unwatchedDiff.ID)) + Expect(mockDiffsRepository.MarkUnwatchedPassedID).To(Equal(unwatchedDiff.ID)) + }) + + It("does not change a diff's status if there's no header for the given block number", func() { + diffWithoutHeader := types.PersistedDiff{ + RawDiff: types.RawDiff{ + Address: contractAddress, + BlockHash: test_data.FakeHash(), + BlockHeight: rand.Int(), + }, + ID: rand.Int63(), + } + setDiffsToReturn(storageWatcher.DiffStatus, mockDiffsRepository, []types.PersistedDiff{diffWithoutHeader}) + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{nil, fakes.FakeError}) + mockHeaderRepository.GetHeaderByBlockNumberError = sql.ErrNoRows + + err := storageWatcher.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(sql.ErrNoRows)) + Expect(mockDiffsRepository.MarkTransformedPassedID).NotTo(Equal(diffWithoutHeader.ID)) }) Describe("When the watcher is configured to skip old diffs", func() { @@ -146,19 +205,15 @@ var _ = Describe("Storage Watcher", func() { var numberOfBlocksFromHeadOfChain = int64(500) BeforeEach(func() { - storageWatcher = watcher.StorageWatcher{ - HeaderRepository: mockHeaderRepository, - StorageDiffRepository: mockDiffsRepository, - KeccakAddressTransformers: map[common.Hash]storage.ITransformer{}, - DiffBlocksFromHeadOfChain: numberOfBlocksFromHeadOfChain, - StatusWriter: &statusWriter, - } + storageWatcher.AddressTransformers = map[common.Address]storage.ITransformer{} + storageWatcher.DiffBlocksFromHeadOfChain = numberOfBlocksFromHeadOfChain + diffID := rand.Int() for i := 0; i < watcher.ResultsLimit; i++ { diffID = diffID + i diff := types.PersistedDiff{ RawDiff: types.RawDiff{ - HashedAddress: test_data.FakeHash(), + Address: test_data.FakeAddress(), }, ID: int64(diffID), } @@ -171,8 +226,8 @@ var _ = Describe("Storage Watcher", func() { mockHeaderRepository.MostRecentHeaderBlockNumber = headerBlockNumber mockDiffsRepository.GetFirstDiffIDToReturn = diffs[0].ID - mockDiffsRepository.GetNewDiffsDiffs = diffs - mockDiffsRepository.GetNewDiffsErrors = []error{nil, fakes.FakeError} + setDiffsToReturn(storageWatcher.DiffStatus, mockDiffsRepository, diffs) + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{nil, fakes.FakeError}) expectedFirstMinDiffID := int(diffs[0].ID - 1) expectedSecondMinDiffID := int(diffs[len(diffs)-1].ID) @@ -182,7 +237,7 @@ var _ = Describe("Storage Watcher", func() { Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) Expect(mockDiffsRepository.GetFirstDiffBlockHeightPassed).To(Equal(headerBlockNumber - numberOfBlocksFromHeadOfChain)) - Expect(mockDiffsRepository.GetNewDiffsPassedMinIDs).To(ConsistOf(expectedFirstMinDiffID, expectedSecondMinDiffID)) + assertGetDiffsMinIDs(storageWatcher.DiffStatus, mockDiffsRepository, []int{expectedFirstMinDiffID, expectedSecondMinDiffID}) }) It("resets min ID back to new min diff when previous query returns fewer than max results", func() { @@ -192,7 +247,7 @@ var _ = Describe("Storage Watcher", func() { diffID = diffID + i diff := types.PersistedDiff{ RawDiff: types.RawDiff{ - HashedAddress: test_data.FakeHash(), + Address: test_data.FakeAddress(), }, ID: int64(diffID), } @@ -203,8 +258,8 @@ var _ = Describe("Storage Watcher", func() { mockHeaderRepository.MostRecentHeaderBlockNumber = headerBlockNumber mockDiffsRepository.GetFirstDiffIDToReturn = diffs[0].ID - mockDiffsRepository.GetNewDiffsDiffs = diffs - mockDiffsRepository.GetNewDiffsErrors = []error{nil, fakes.FakeError} + setDiffsToReturn(storageWatcher.DiffStatus, mockDiffsRepository, diffs) + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{nil, fakes.FakeError}) expectedFirstMinDiffID := int(diffs[0].ID - 1) @@ -213,13 +268,13 @@ var _ = Describe("Storage Watcher", func() { Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) Expect(mockDiffsRepository.GetFirstDiffBlockHeightPassed).To(Equal(headerBlockNumber - numberOfBlocksFromHeadOfChain)) - Expect(mockDiffsRepository.GetNewDiffsPassedMinIDs).To(ConsistOf(expectedFirstMinDiffID, expectedFirstMinDiffID)) + assertGetDiffsMinIDs(storageWatcher.DiffStatus, mockDiffsRepository, []int{expectedFirstMinDiffID, expectedFirstMinDiffID}) }) It("sets minID to 0 if there are no headers with the given block height", func() { mockHeaderRepository.MostRecentHeaderBlockNumberErr = sql.ErrNoRows - mockDiffsRepository.GetNewDiffsDiffs = diffs - mockDiffsRepository.GetNewDiffsErrors = []error{nil, fakes.FakeError} + setDiffsToReturn(storageWatcher.DiffStatus, mockDiffsRepository, diffs) + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{nil, fakes.FakeError}) err := storageWatcher.Execute() Expect(err).To(HaveOccurred()) @@ -227,13 +282,13 @@ var _ = Describe("Storage Watcher", func() { expectedFirstMinDiffID := 0 expectedSecondMinDiffID := int(diffs[len(diffs)-1].ID) - Expect(mockDiffsRepository.GetNewDiffsPassedMinIDs).To(ConsistOf(expectedFirstMinDiffID, expectedSecondMinDiffID)) + assertGetDiffsMinIDs(storageWatcher.DiffStatus, mockDiffsRepository, []int{expectedFirstMinDiffID, expectedSecondMinDiffID}) }) It("sets minID to 0 if there are no diffs with given block range", func() { mockDiffsRepository.GetFirstDiffIDErr = sql.ErrNoRows - mockDiffsRepository.GetNewDiffsDiffs = diffs - mockDiffsRepository.GetNewDiffsErrors = []error{nil, fakes.FakeError} + setDiffsToReturn(storageWatcher.DiffStatus, mockDiffsRepository, diffs) + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{nil, fakes.FakeError}) err := storageWatcher.Execute() Expect(err).To(HaveOccurred()) @@ -241,143 +296,158 @@ var _ = Describe("Storage Watcher", func() { expectedFirstMinDiffID := 0 expectedSecondMinDiffID := int(diffs[len(diffs)-1].ID) - Expect(mockDiffsRepository.GetNewDiffsPassedMinIDs).To(ConsistOf(expectedFirstMinDiffID, expectedSecondMinDiffID)) + assertGetDiffsMinIDs(storageWatcher.DiffStatus, mockDiffsRepository, []int{expectedFirstMinDiffID, expectedSecondMinDiffID}) }) }) - Describe("when diff's address is watched", func() { + Describe("When a header with a non-matching hash is found", func() { var ( - hashedAddress common.Hash - mockTransformer *mocks.MockStorageTransformer + blockNumber int + fakePersistedDiff types.PersistedDiff ) BeforeEach(func() { - hashedAddress = types.HexToKeccak256Hash("0x" + fakes.RandomString(20)) - mockTransformer = &mocks.MockStorageTransformer{KeccakOfAddress: hashedAddress} - storageWatcher.AddTransformers([]storage.TransformerInitializer{mockTransformer.FakeTransformerInitializer}) - }) + blockNumber = rand.Int() + fakeRawDiff := types.RawDiff{ + Address: contractAddress, + BlockHash: test_data.FakeHash(), + BlockHeight: blockNumber, + StorageKey: test_data.FakeHash(), + StorageValue: test_data.FakeHash(), + } + mockHeaderRepository.GetHeaderByBlockNumberReturnID = int64(blockNumber) + mockHeaderRepository.GetHeaderByBlockNumberReturnHash = test_data.FakeHash().Hex() - It("does not mark diff checked if no matching header", func() { - diffWithoutHeader := types.PersistedDiff{ - RawDiff: types.RawDiff{ - HashedAddress: hashedAddress, - BlockHash: test_data.FakeHash(), - BlockHeight: rand.Int(), - }, - ID: rand.Int63(), + fakePersistedDiff = types.PersistedDiff{ + RawDiff: fakeRawDiff, + ID: rand.Int63(), } - mockDiffsRepository.GetNewDiffsDiffs = []types.PersistedDiff{diffWithoutHeader} - mockDiffsRepository.GetNewDiffsErrors = []error{nil, fakes.FakeError} - mockHeaderRepository.GetHeaderByBlockNumberError = errors.New("no matching header") + setDiffsToReturn(storageWatcher.DiffStatus, mockDiffsRepository, []types.PersistedDiff{fakePersistedDiff}) + }) + + It("does not change a diff's status if getting max known block height fails", func() { + maxHeaderErr := errors.New("getting max header failed") + mockHeaderRepository.MostRecentHeaderBlockNumberErr = maxHeaderErr + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{nil}) err := storageWatcher.Execute() Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) - Expect(mockDiffsRepository.MarkCheckedPassedID).NotTo(Equal(diffWithoutHeader.ID)) + Expect(err).To(MatchError(maxHeaderErr)) + Expect(mockDiffsRepository.MarkTransformedPassedID).NotTo(Equal(fakePersistedDiff.ID)) }) - Describe("when non-matching header found", func() { - var ( - blockNumber int - fakePersistedDiff types.PersistedDiff - ) - - BeforeEach(func() { - blockNumber = rand.Int() - fakeRawDiff := types.RawDiff{ - HashedAddress: hashedAddress, - BlockHash: test_data.FakeHash(), - BlockHeight: blockNumber, - StorageKey: test_data.FakeHash(), - StorageValue: test_data.FakeHash(), - } - mockHeaderRepository.GetHeaderByBlockNumberReturnID = int64(blockNumber) - mockHeaderRepository.GetHeaderByBlockNumberReturnHash = test_data.FakeHash().Hex() - - fakePersistedDiff = types.PersistedDiff{ - RawDiff: fakeRawDiff, - ID: rand.Int63(), - } - mockDiffsRepository.GetNewDiffsDiffs = []types.PersistedDiff{fakePersistedDiff} - }) + It("marks diff 'noncanonical' if block height less than max known block height minus reorg window", func() { + mockHeaderRepository.MostRecentHeaderBlockNumber = int64(blockNumber + watcher.ReorgWindow + 1) + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{nil, fakes.FakeError}) - It("does not mark diff checked if getting max known block height fails", func() { - mockHeaderRepository.MostRecentHeaderBlockNumberErr = errors.New("getting max header failed") - mockDiffsRepository.GetNewDiffsErrors = []error{nil, fakes.FakeError} + err := storageWatcher.Execute() - err := storageWatcher.Execute() + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + Expect(mockDiffsRepository.MarkNoncanonicalPassedID).To(Equal(fakePersistedDiff.ID)) + }) - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) - Expect(mockDiffsRepository.MarkCheckedPassedID).NotTo(Equal(fakePersistedDiff.ID)) - }) + It("does not mark diff as 'noncanonical' if block height is within reorg window", func() { + mockHeaderRepository.MostRecentHeaderBlockNumber = int64(blockNumber + watcher.ReorgWindow) + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{nil, fakes.FakeError}) - It("marks diff checked if block height less than max known block height minus reorg window", func() { - mockHeaderRepository.MostRecentHeaderBlockNumber = int64(blockNumber + watcher.ReorgWindow + 1) - mockDiffsRepository.GetNewDiffsErrors = []error{nil, fakes.FakeError} + err := storageWatcher.Execute() - err := storageWatcher.Execute() + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + Expect(mockDiffsRepository.MarkTransformedPassedID).NotTo(Equal(fakePersistedDiff.ID)) + }) + }) - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) - Expect(mockDiffsRepository.MarkCheckedPassedID).To(Equal(fakePersistedDiff.ID)) - }) + Describe("When a header with a matching hash exists", func() { + var fakePersistedDiff types.PersistedDiff - It("does not mark diff checked if block height is within reorg window", func() { - mockHeaderRepository.MostRecentHeaderBlockNumber = int64(blockNumber + watcher.ReorgWindow) - mockDiffsRepository.GetNewDiffsErrors = []error{nil, fakes.FakeError} + BeforeEach(func() { + fakeBlockHash := test_data.FakeHash() + fakeRawDiff := types.RawDiff{ + Address: contractAddress, + BlockHash: fakeBlockHash, + } - err := storageWatcher.Execute() + mockHeaderRepository.GetHeaderByBlockNumberReturnID = rand.Int63() + mockHeaderRepository.GetHeaderByBlockNumberReturnHash = fakeBlockHash.Hex() - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) - Expect(mockDiffsRepository.MarkCheckedPassedID).NotTo(Equal(fakePersistedDiff.ID)) - }) + fakePersistedDiff = types.PersistedDiff{ + RawDiff: fakeRawDiff, + ID: rand.Int63(), + } + setDiffsToReturn(storageWatcher.DiffStatus, mockDiffsRepository, []types.PersistedDiff{fakePersistedDiff}) }) - Describe("when matching header exists", func() { - var fakePersistedDiff types.PersistedDiff - - BeforeEach(func() { - fakeBlockHash := test_data.FakeHash() - fakeRawDiff := types.RawDiff{ - HashedAddress: hashedAddress, - BlockHash: fakeBlockHash, - } + It("does not change diff's status if transformer execution fails", func() { + executeErr := errors.New("execute failed") + mockTransformer.ExecuteErr = executeErr + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{nil}) - mockHeaderRepository.GetHeaderByBlockNumberReturnID = rand.Int63() - mockHeaderRepository.GetHeaderByBlockNumberReturnHash = fakeBlockHash.Hex() + err := storageWatcher.Execute() - fakePersistedDiff = types.PersistedDiff{ - RawDiff: fakeRawDiff, - ID: rand.Int63(), - } - mockDiffsRepository.GetNewDiffsDiffs = []types.PersistedDiff{fakePersistedDiff} - }) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(executeErr)) + Expect(mockDiffsRepository.MarkTransformedPassedID).NotTo(Equal(fakePersistedDiff.ID)) + }) - It("does not mark diff checked if transformer execution fails", func() { - mockTransformer.ExecuteErr = errors.New("execute failed") - mockDiffsRepository.GetNewDiffsErrors = []error{nil, fakes.FakeError} + It("marks diff as 'unrecognized' when transforming the diff returns a ErrKeyNotFound error", func() { + mockTransformer.ExecuteErr = types.ErrKeyNotFound + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{nil, types.ErrKeyNotFound}) - err := storageWatcher.Execute() + err := storageWatcher.Execute() - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) - Expect(mockDiffsRepository.MarkCheckedPassedID).NotTo(Equal(fakePersistedDiff.ID)) - }) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(types.ErrKeyNotFound)) + Expect(mockDiffsRepository.MarkUnrecognizedPassedID).To(Equal(fakePersistedDiff.ID)) + }) - It("marks diff checked if transformer execution doesn't fail", func() { - mockDiffsRepository.GetNewDiffsDiffs = []types.PersistedDiff{fakePersistedDiff} - mockDiffsRepository.GetNewDiffsErrors = []error{nil, fakes.FakeError} + It("marks diff transformed if transformer execution doesn't fail", func() { + setDiffsToReturn(storageWatcher.DiffStatus, mockDiffsRepository, []types.PersistedDiff{fakePersistedDiff}) + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{nil, fakes.FakeError}) - err := storageWatcher.Execute() + err := storageWatcher.Execute() - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) - Expect(mockDiffsRepository.MarkCheckedPassedID).To(Equal(fakePersistedDiff.ID)) - }) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + Expect(mockDiffsRepository.MarkTransformedPassedID).To(Equal(fakePersistedDiff.ID)) }) }) }) -}) +} + +func setGetDiffsErrors(diffStatus watcher.DiffStatusToWatch, mockDiffsRepo *mocks.MockStorageDiffRepository, diffErrors []error) { + switch diffStatus { + case watcher.New: + mockDiffsRepo.GetNewDiffsErrors = diffErrors + case watcher.Unrecognized: + mockDiffsRepo.GetUnrecognizedDiffsErrors = diffErrors + } +} +func setDiffsToReturn(diffStatus watcher.DiffStatusToWatch, mockDiffsRepo *mocks.MockStorageDiffRepository, diffs []types.PersistedDiff) { + switch diffStatus { + case watcher.New: + mockDiffsRepo.GetNewDiffsToReturn = diffs + case watcher.Unrecognized: + mockDiffsRepo.GetUnrecognizedDiffsToReturn = diffs + } +} + +func assertGetDiffsLimits(diffStatus watcher.DiffStatusToWatch, mockDiffRepo *mocks.MockStorageDiffRepository, watcherLimit int) { + switch diffStatus { + case watcher.New: + Expect(mockDiffRepo.GetNewDiffsPassedLimits).To(ConsistOf(watcherLimit)) + case watcher.Unrecognized: + Expect(mockDiffRepo.GetUnrecognizedDiffsPassedLimits).To(ConsistOf(watcherLimit)) + } +} + +func assertGetDiffsMinIDs(diffStatus watcher.DiffStatusToWatch, mockDiffRepo *mocks.MockStorageDiffRepository, passedMinIDs []int) { + switch diffStatus { + case watcher.New: + Expect(mockDiffRepo.GetNewDiffsPassedMinIDs).To(ConsistOf(passedMinIDs)) + case watcher.Unrecognized: + Expect(mockDiffRepo.GetUnrecognizedDiffsPassedMinIDs).To(ConsistOf(passedMinIDs)) + } +} diff --git a/pkg/config/plugin.go b/pkg/config/plugin.go index 25f0d6014..fe9423803 100644 --- a/pkg/config/plugin.go +++ b/pkg/config/plugin.go @@ -31,6 +31,7 @@ type Plugin struct { FileName string Save bool Home string + Schema string } type Transformer struct { diff --git a/pkg/contract_watcher/helpers/test_helpers/database.go b/pkg/contract_watcher/helpers/test_helpers/database.go index 60c9920f9..a82172e9e 100644 --- a/pkg/contract_watcher/helpers/test_helpers/database.go +++ b/pkg/contract_watcher/helpers/test_helpers/database.go @@ -17,19 +17,12 @@ package test_helpers import ( - "github.com/ethereum/go-ethereum/ethclient" - "github.com/ethereum/go-ethereum/rpc" "github.com/makerdao/vulcanizedb/pkg/config" "github.com/makerdao/vulcanizedb/pkg/contract_watcher/constants" "github.com/makerdao/vulcanizedb/pkg/contract_watcher/contract" "github.com/makerdao/vulcanizedb/pkg/contract_watcher/helpers/test_helpers/mocks" "github.com/makerdao/vulcanizedb/pkg/core" "github.com/makerdao/vulcanizedb/pkg/datastore/postgres" - "github.com/makerdao/vulcanizedb/pkg/eth" - "github.com/makerdao/vulcanizedb/pkg/eth/client" - "github.com/makerdao/vulcanizedb/pkg/eth/converters" - "github.com/makerdao/vulcanizedb/pkg/eth/node" - "github.com/makerdao/vulcanizedb/test_config" . "github.com/onsi/gomega" ) @@ -55,28 +48,6 @@ type NewOwnerLog struct { RawLog []byte `db:"raw_log"` } -func SetupDBandBC() (*postgres.DB, core.BlockChain) { - con := test_config.TestClient - testIPC := con.IPCPath - rawRPCClient, err := rpc.Dial(testIPC) - Expect(err).NotTo(HaveOccurred()) - rpcClient := client.NewRpcClient(rawRPCClient, testIPC) - ethClient := ethclient.NewClient(rawRPCClient) - blockChainClient := client.NewEthClient(ethClient) - madeNode := node.MakeNode(rpcClient) - transactionConverter := converters.NewTransactionConverter(ethClient) - blockChain := eth.NewBlockChain(blockChainClient, rpcClient, madeNode, transactionConverter) - - db, err := postgres.NewDB(config.Database{ - Hostname: "localhost", - Name: "vulcanize_testing", - Port: 5432, - }, blockChain.Node()) - Expect(err).NotTo(HaveOccurred()) - - return db, blockChain -} - func SetupTusdRepo(wantedEvents []string) (*postgres.DB, *contract.Contract) { db, err := postgres.NewDB(config.Database{ Hostname: "localhost", @@ -167,14 +138,6 @@ func TearDown(db *postgres.DB) { _, err = tx.Exec(`DELETE FROM public.receipts`) Expect(err).NotTo(HaveOccurred()) - _, err = tx.Exec(`DROP TABLE public.checked_headers`) - Expect(err).NotTo(HaveOccurred()) - - _, err = tx.Exec(`CREATE TABLE public.checked_headers ( - id SERIAL PRIMARY KEY, - header_id INTEGER UNIQUE NOT NULL REFERENCES headers (id) ON DELETE CASCADE);`) - Expect(err).NotTo(HaveOccurred()) - _, err = tx.Exec(`DROP SCHEMA IF EXISTS cw_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e CASCADE`) Expect(err).NotTo(HaveOccurred()) @@ -184,6 +147,25 @@ func TearDown(db *postgres.DB) { err = tx.Commit() Expect(err).NotTo(HaveOccurred()) + err = ResetCheckedHeadersTableSchema(db) + Expect(err).NotTo(HaveOccurred()) + _, err = db.Exec(`VACUUM public.checked_headers`) Expect(err).NotTo(HaveOccurred()) } + +func ResetCheckedHeadersTableSchema(db *postgres.DB) error { + _, dropTableErr := db.Exec(`DROP TABLE public.checked_headers`) + if dropTableErr != nil { + return dropTableErr + } + + _, createTableErr := db.Exec(`CREATE TABLE public.checked_headers ( + id SERIAL PRIMARY KEY, + header_id INTEGER UNIQUE NOT NULL REFERENCES headers (id) ON DELETE CASCADE);`) + if createTableErr != nil { + return createTableErr + } + + return nil +} diff --git a/pkg/contract_watcher/repository/header_repository_test.go b/pkg/contract_watcher/repository/header_repository_test.go index 05d854ce7..02e1d7086 100644 --- a/pkg/contract_watcher/repository/header_repository_test.go +++ b/pkg/contract_watcher/repository/header_repository_test.go @@ -24,15 +24,15 @@ import ( "github.com/makerdao/vulcanizedb/pkg/contract_watcher/repository" "github.com/makerdao/vulcanizedb/pkg/core" "github.com/makerdao/vulcanizedb/pkg/datastore" - "github.com/makerdao/vulcanizedb/pkg/datastore/postgres" "github.com/makerdao/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/makerdao/vulcanizedb/test_config" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) var _ = Describe("Repository", func() { var ( - db *postgres.DB + db = test_config.NewTestDB(test_config.NewTestNode()) contractHeaderRepo repository.HeaderRepository // contract_watcher header repository coreHeaderRepo datastore.HeaderRepository // pkg/datastore header repository eventIDs = []string{ @@ -43,13 +43,14 @@ var _ = Describe("Repository", func() { ) BeforeEach(func() { - db, _ = test_helpers.SetupDBandBC() + test_config.CleanTestDB(db) contractHeaderRepo = repository.NewHeaderRepository(db) coreHeaderRepo = repositories.NewHeaderRepository(db) }) AfterEach(func() { - test_helpers.TearDown(db) + err := test_helpers.ResetCheckedHeadersTableSchema(db) + Expect(err).NotTo(HaveOccurred(), "Failed to reset checked headers table in teardown") }) Describe("AddCheckColumn", func() { diff --git a/pkg/core/eth_client.go b/pkg/core/eth_client.go index 183d224bd..deb3fceb1 100644 --- a/pkg/core/eth_client.go +++ b/pkg/core/eth_client.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/filters" ) type EthClient interface { @@ -30,6 +31,7 @@ type EthClient interface { CallContract(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) FilterLogs(ctx context.Context, q ethereum.FilterQuery) ([]types.Log, error) HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) + SubscribeNewStateChanges(ctx context.Context, q ethereum.FilterQuery, ch chan<- filters.Payload) (ethereum.Subscription, error) TransactionSender(ctx context.Context, tx *types.Transaction, block common.Hash, index uint) (common.Address, error) TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) } diff --git a/pkg/datastore/postgres/repositories/checked_headers_repository.go b/pkg/datastore/postgres/repositories/checked_headers_repository.go index 25446e5a3..d51604a1a 100644 --- a/pkg/datastore/postgres/repositories/checked_headers_repository.go +++ b/pkg/datastore/postgres/repositories/checked_headers_repository.go @@ -17,31 +17,59 @@ package repositories import ( + "fmt" + "github.com/makerdao/vulcanizedb/pkg/core" "github.com/makerdao/vulcanizedb/pkg/datastore/postgres" ) const ( - insertCheckedHeaderQuery = `UPDATE public.headers SET check_count = (SELECT check_count WHERE id = $1) + 1 WHERE id = $1` + insertCheckedHeaderQuery = ` +INSERT INTO %s.checked_headers (check_count, header_id) +VALUES (1, $1) +ON CONFLICT (header_id) DO + UPDATE SET check_count = + (SELECT checked_headers.check_count WHERE checked_headers.header_id = $1) + 1 + WHERE checked_headers.header_id = $1` ) +type SchemaQuery struct { + SchemaName string `db:"schema_name"` +} + type CheckedHeadersRepository struct { - db *postgres.DB + db *postgres.DB + schemaName string } -func NewCheckedHeadersRepository(db *postgres.DB) CheckedHeadersRepository { - return CheckedHeadersRepository{db: db} +func NewCheckedHeadersRepository(db *postgres.DB, schemaName string) (*CheckedHeadersRepository, error) { + var sq SchemaQuery + err := db.Get(&sq, "SELECT schema_name FROM information_schema.schemata WHERE schema_name = $1", schemaName) + if err != nil { + return nil, fmt.Errorf("error creating checked headers repository - invalid schema %w", err) + } + + // Use the SchemaName from the actual information_schema, just in case the + // passed in schemaName doesn't exist + return &CheckedHeadersRepository{db: db, schemaName: sq.SchemaName}, nil } // Increment check_count for header func (repo CheckedHeadersRepository) MarkHeaderChecked(headerID int64) error { - _, err := repo.db.Exec(insertCheckedHeaderQuery, headerID) + queryString := fmt.Sprintf(insertCheckedHeaderQuery, repo.schemaName) + _, err := repo.db.Exec(queryString, headerID) return err } // Zero out check count for header with the given block number func (repo CheckedHeadersRepository) MarkSingleHeaderUnchecked(blockNumber int64) error { - _, err := repo.db.Exec(`UPDATE public.headers SET check_count = 0 WHERE block_number = $1`, blockNumber) + queryString := fmt.Sprintf(`UPDATE %s.checked_headers ch + SET check_count = 0 + FROM public.headers h + WHERE ch.header_id = h.id + AND h.block_number = $1`, repo.schemaName) + + _, err := repo.db.Exec(queryString, blockNumber) return err } @@ -49,30 +77,30 @@ func (repo CheckedHeadersRepository) MarkSingleHeaderUnchecked(blockNumber int64 func (repo CheckedHeadersRepository) UncheckedHeaders(startingBlockNumber, endingBlockNumber, checkCount int64) ([]core.Header, error) { var ( result []core.Header - query string err error recheckOffsetMultiplier = 15 ) + joinQuery := fmt.Sprintf(` +WITH checked_headers AS ( + SELECT h.id, h.block_number, h.hash, COALESCE(ch.check_count, 0) AS check_count + FROM public.headers h + LEFT JOIN %s.checked_headers ch + ON ch.header_id = h.id + WHERE h.block_number >= $1 +) +SELECT id, block_number, hash +FROM checked_headers +WHERE ( check_count < 1 + OR (check_count < $2 + AND block_number <= ((SELECT MAX(block_number) FROM public.headers) - ($3 * check_count * (check_count + 1) / 2)))) +`, repo.schemaName) + if endingBlockNumber == -1 { - query = `SELECT id, block_number, hash - FROM public.headers - WHERE (check_count < 1 - AND block_number >= $1) - OR (check_count < $2 - AND block_number <= ((SELECT MAX(block_number) FROM public.headers) - ($3 * check_count * (check_count + 1) / 2)))` - err = repo.db.Select(&result, query, startingBlockNumber, checkCount, recheckOffsetMultiplier) + err = repo.db.Select(&result, joinQuery, startingBlockNumber, checkCount, recheckOffsetMultiplier) } else { - query = `SELECT id, block_number, hash - FROM public.headers - WHERE (check_count < 1 - AND block_number >= $1 - AND block_number <= $2) - OR (check_count < $3 - AND block_number >= $1 - AND block_number <= $2 - AND block_number <= ((SELECT MAX(block_number) FROM public.headers) - ($4 * (check_count * (check_count + 1) / 2))))` - err = repo.db.Select(&result, query, startingBlockNumber, endingBlockNumber, checkCount, recheckOffsetMultiplier) + endingBlockQuery := fmt.Sprintf(`%s AND block_number <= $4`, joinQuery) + err = repo.db.Select(&result, endingBlockQuery, startingBlockNumber, checkCount, recheckOffsetMultiplier, endingBlockNumber) } return result, err diff --git a/pkg/datastore/postgres/repositories/checked_headers_repository_test.go b/pkg/datastore/postgres/repositories/checked_headers_repository_test.go index 7fae6b331..d3b748db0 100644 --- a/pkg/datastore/postgres/repositories/checked_headers_repository_test.go +++ b/pkg/datastore/postgres/repositories/checked_headers_repository_test.go @@ -17,10 +17,12 @@ package repositories_test import ( + "fmt" "math/rand" "github.com/makerdao/vulcanizedb/pkg/core" "github.com/makerdao/vulcanizedb/pkg/datastore" + "github.com/makerdao/vulcanizedb/pkg/datastore/postgres" "github.com/makerdao/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/makerdao/vulcanizedb/pkg/fakes" "github.com/makerdao/vulcanizedb/test_config" @@ -30,313 +32,378 @@ import ( var _ = Describe("Checked Headers repository", func() { var ( - db = test_config.NewTestDB(test_config.NewTestNode()) - repo datastore.CheckedHeadersRepository + db = test_config.NewTestDB(test_config.NewTestNode()) + repo datastore.CheckedHeadersRepository + pluginSchemaName = "plugin" ) - BeforeEach(func() { - test_config.CleanTestDB(db) - repo = repositories.NewCheckedHeadersRepository(db) + Describe("without a valid setup", func() { + BeforeEach(func() { + test_config.CleanTestDB(db) + }) + + It("errors for a schema that isn't present", func() { + var err error + repo, err = repositories.NewCheckedHeadersRepository(db, pluginSchemaName) + Expect(err.Error()).To(ContainSubstring("invalid schema")) + Expect(repo).To(BeNil()) + }) }) - Describe("MarkHeaderChecked", func() { - It("marks passed header as checked on insert", func() { - headerRepository := repositories.NewHeaderRepository(db) - headerID, headerErr := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) - Expect(headerErr).NotTo(HaveOccurred()) + Describe("with a valid database schema", func() { - err := repo.MarkHeaderChecked(headerID) + BeforeEach(func() { + test_config.CleanTestDB(db) + Expect(createPluginCheckedHeadersTable(db, pluginSchemaName)).To(Succeed()) - Expect(err).NotTo(HaveOccurred()) - var checkedCount int - fetchErr := db.Get(&checkedCount, `SELECT check_count FROM public.headers WHERE id = $1`, headerID) - Expect(fetchErr).NotTo(HaveOccurred()) - Expect(checkedCount).To(Equal(1)) + var repoErr error + repo, repoErr = repositories.NewCheckedHeadersRepository(db, pluginSchemaName) + Expect(repoErr).NotTo(HaveOccurred()) }) - It("increments check count on update", func() { - headerRepository := repositories.NewHeaderRepository(db) - headerID, headerErr := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) - Expect(headerErr).NotTo(HaveOccurred()) + AfterEach(func() { + Expect(clearPluginSchema(db, pluginSchemaName)).To(Succeed()) + }) - insertErr := repo.MarkHeaderChecked(headerID) - Expect(insertErr).NotTo(HaveOccurred()) + Describe("MarkHeaderChecked", func() { + It("marks passed header as checked on insert", func() { + headerRepository := repositories.NewHeaderRepository(db) + headerID, headerErr := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) + Expect(headerErr).NotTo(HaveOccurred()) - updateErr := repo.MarkHeaderChecked(headerID) - Expect(updateErr).NotTo(HaveOccurred()) + Expect(repo.MarkHeaderChecked(headerID)).To(Succeed()) - var checkedCount int - fetchErr := db.Get(&checkedCount, `SELECT check_count FROM public.headers WHERE id = $1`, headerID) - Expect(fetchErr).NotTo(HaveOccurred()) - Expect(checkedCount).To(Equal(2)) - }) - }) + Expect(selectCheckedHeaders(db, pluginSchemaName, headerID)).To(Equal(1)) + }) - Describe("MarkSingleHeaderUnchecked", func() { - It("marks headers with matching block number as unchecked", func() { - blockNumberOne := rand.Int63() - blockNumberTwo := blockNumberOne + 1 - blockNumberThree := blockNumberOne + 2 - fakeHeaderOne := fakes.GetFakeHeader(blockNumberOne) - fakeHeaderTwo := fakes.GetFakeHeader(blockNumberTwo) - fakeHeaderThree := fakes.GetFakeHeader(blockNumberThree) - headerRepository := repositories.NewHeaderRepository(db) - // insert three headers with incrementing block number - headerIdOne, insertHeaderOneErr := headerRepository.CreateOrUpdateHeader(fakeHeaderOne) - Expect(insertHeaderOneErr).NotTo(HaveOccurred()) - headerIdTwo, insertHeaderTwoErr := headerRepository.CreateOrUpdateHeader(fakeHeaderTwo) - Expect(insertHeaderTwoErr).NotTo(HaveOccurred()) - headerIdThree, insertHeaderThreeErr := headerRepository.CreateOrUpdateHeader(fakeHeaderThree) - Expect(insertHeaderThreeErr).NotTo(HaveOccurred()) - // mark all headers checked - markHeaderOneCheckedErr := repo.MarkHeaderChecked(headerIdOne) - Expect(markHeaderOneCheckedErr).NotTo(HaveOccurred()) - markHeaderTwoCheckedErr := repo.MarkHeaderChecked(headerIdTwo) - Expect(markHeaderTwoCheckedErr).NotTo(HaveOccurred()) - markHeaderThreeCheckedErr := repo.MarkHeaderChecked(headerIdThree) - Expect(markHeaderThreeCheckedErr).NotTo(HaveOccurred()) - - // mark header from blockNumberTwo unchecked - err := repo.MarkSingleHeaderUnchecked(blockNumberTwo) - - Expect(err).NotTo(HaveOccurred()) - var headerOneCheckCount, headerTwoCheckCount, headerThreeCheckCount int - getHeaderOneErr := db.Get(&headerOneCheckCount, `SELECT check_count FROM public.headers WHERE id = $1`, headerIdOne) - Expect(getHeaderOneErr).NotTo(HaveOccurred()) - Expect(headerOneCheckCount).To(Equal(1)) - getHeaderTwoErr := db.Get(&headerTwoCheckCount, `SELECT check_count FROM public.headers WHERE id = $1`, headerIdTwo) - Expect(getHeaderTwoErr).NotTo(HaveOccurred()) - Expect(headerTwoCheckCount).To(BeZero()) - getHeaderThreeErr := db.Get(&headerThreeCheckCount, `SELECT check_count FROM public.headers WHERE id = $1`, headerIdThree) - Expect(getHeaderThreeErr).NotTo(HaveOccurred()) - Expect(headerThreeCheckCount).To(Equal(1)) - }) - }) + It("increments check count on update", func() { + headerRepository := repositories.NewHeaderRepository(db) + headerID, headerErr := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) + Expect(headerErr).NotTo(HaveOccurred()) - Describe("UncheckedHeaders", func() { - var ( - headerRepository datastore.HeaderRepository - firstBlock, - secondBlock, - thirdBlock, - lastBlock, - secondHeaderID, - thirdHeaderID int64 - blockNumbers []int64 - headerIDs []int64 - err error - uncheckedCheckCount = int64(1) - recheckCheckCount = int64(2) - ) + Expect(repo.MarkHeaderChecked(headerID)).To(Succeed()) + Expect(repo.MarkHeaderChecked(headerID)).To(Succeed()) - BeforeEach(func() { - headerRepository = repositories.NewHeaderRepository(db) - - lastBlock = rand.Int63() - thirdBlock = lastBlock - 15 - secondBlock = lastBlock - (15 + 30) - firstBlock = lastBlock - (15 + 30 + 45) - - blockNumbers = []int64{firstBlock, secondBlock, thirdBlock, lastBlock} - - headerIDs = []int64{} - for _, n := range blockNumbers { - headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(n)) - headerIDs = append(headerIDs, headerID) - Expect(err).NotTo(HaveOccurred()) - } - secondHeaderID = headerIDs[1] - thirdHeaderID = headerIDs[2] + Expect(selectCheckedHeaders(db, pluginSchemaName, headerID)).To(Equal(2)) + }) }) - Describe("when ending block is specified", func() { - It("excludes headers that are out of range", func() { - headers, err := repo.UncheckedHeaders(firstBlock, thirdBlock, uncheckedCheckCount) - Expect(err).NotTo(HaveOccurred()) - - headerBlockNumbers := getBlockNumbers(headers) - Expect(headerBlockNumbers).To(ConsistOf(firstBlock, secondBlock, thirdBlock)) - Expect(headerBlockNumbers).NotTo(ContainElement(lastBlock)) - }) + Describe("MarkSingleHeaderUnchecked", func() { + It("marks headers with matching block number as unchecked", func() { + blockNumber := rand.Int63() + fakeHeader := fakes.GetFakeHeader(blockNumber) + headerRepository := repositories.NewHeaderRepository(db) + headerID, insertHeaderErr := headerRepository.CreateOrUpdateHeader(fakeHeader) + Expect(insertHeaderErr).NotTo(HaveOccurred()) + Expect(repo.MarkHeaderChecked(headerID)).To(Succeed()) - It("excludes headers that have been checked more than the check count", func() { - _, err = db.Exec(`UPDATE public.headers SET check_count = 1 WHERE id = $1`, secondHeaderID) - Expect(err).NotTo(HaveOccurred()) + Expect(repo.MarkSingleHeaderUnchecked(blockNumber)).To(Succeed()) - headers, err := repo.UncheckedHeaders(firstBlock, thirdBlock, uncheckedCheckCount) - Expect(err).NotTo(HaveOccurred()) + Expect(selectCheckedHeaders(db, pluginSchemaName, headerID)).To(BeZero()) + }) - headerBlockNumbers := getBlockNumbers(headers) - Expect(headerBlockNumbers).To(ConsistOf(firstBlock, thirdBlock)) - Expect(headerBlockNumbers).NotTo(ContainElement(secondBlock)) + It("leaves headers without a matching block number alone", func() { + checkedBlockNumber := rand.Int63() + uncheckedBlockNumber := checkedBlockNumber + 1 + checkedHeader := fakes.GetFakeHeader(checkedBlockNumber) + uncheckedHeader := fakes.GetFakeHeader(uncheckedBlockNumber) + headerRepository := repositories.NewHeaderRepository(db) + checkedHeaderID, insertCheckedHeaderErr := headerRepository.CreateOrUpdateHeader(checkedHeader) + Expect(insertCheckedHeaderErr).NotTo(HaveOccurred()) + uncheckedHeaderID, insertUncheckedHeaderErr := headerRepository.CreateOrUpdateHeader(uncheckedHeader) + Expect(insertUncheckedHeaderErr).NotTo(HaveOccurred()) + + // mark both headers as checked + Expect(repo.MarkHeaderChecked(checkedHeaderID)).To(Succeed()) + Expect(repo.MarkHeaderChecked(uncheckedHeaderID)).To(Succeed()) + + // re-mark unchecked header as unchecked + Expect(repo.MarkSingleHeaderUnchecked(uncheckedBlockNumber)).To(Succeed()) + + // Verify the other block was not checked (1 checked header) + Expect(selectCheckedHeaders(db, pluginSchemaName, checkedHeaderID)).To(Equal(1)) }) + }) - Describe("when header has already been checked", func() { - It("includes header with block number > 15 back from latest with check count of 1", func() { - _, err = db.Exec(`UPDATE public.headers SET check_count = 1 WHERE id = $1`, thirdHeaderID) + Describe("UncheckedHeaders", func() { + var ( + headerRepository datastore.HeaderRepository + firstBlock, + secondBlock, + thirdBlock, + lastBlock, + secondHeaderID, + thirdHeaderID int64 + blockNumbers []int64 + headerIDs []int64 + err error + uncheckedCheckCount = int64(1) + recheckCheckCount = int64(2) + ) + + BeforeEach(func() { + headerRepository = repositories.NewHeaderRepository(db) + + lastBlock = rand.Int63() + thirdBlock = lastBlock - 15 + secondBlock = lastBlock - (15 + 30) + firstBlock = lastBlock - (15 + 30 + 45) + + blockNumbers = []int64{firstBlock, secondBlock, thirdBlock, lastBlock} + + headerIDs = []int64{} + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(n)) Expect(err).NotTo(HaveOccurred()) + headerIDs = append(headerIDs, headerID) + } + secondHeaderID = headerIDs[1] + thirdHeaderID = headerIDs[2] + }) - headers, err := repo.UncheckedHeaders(firstBlock, lastBlock, recheckCheckCount) + Describe("when ending block is specified", func() { + It("excludes headers that are out of range", func() { + headers, err := repo.UncheckedHeaders(firstBlock, thirdBlock, uncheckedCheckCount) Expect(err).NotTo(HaveOccurred()) headerBlockNumbers := getBlockNumbers(headers) - Expect(headerBlockNumbers).To(ContainElement(thirdBlock)) + Expect(headerBlockNumbers).To(ConsistOf(firstBlock, secondBlock, thirdBlock)) + Expect(headerBlockNumbers).NotTo(ContainElement(lastBlock)) }) - It("excludes header with block number < 15 back from latest with check count of 1", func() { - excludedHeader := fakes.GetFakeHeader(thirdBlock + 1) - excludedHeaderID, createHeaderErr := headerRepository.CreateOrUpdateHeader(excludedHeader) - Expect(createHeaderErr).NotTo(HaveOccurred()) - _, updateHeaderErr := db.Exec(`UPDATE public.headers SET check_count = 1 WHERE id = $1`, excludedHeaderID) - Expect(updateHeaderErr).NotTo(HaveOccurred()) + It("includes headers that have had their check-count reset", func() { + Expect(repo.MarkHeaderChecked(secondHeaderID)).To(Succeed()) + Expect(repo.MarkSingleHeaderUnchecked(secondBlock)).To(Succeed()) - headers, err := repo.UncheckedHeaders(firstBlock, lastBlock, recheckCheckCount) + headers, err := repo.UncheckedHeaders(firstBlock, thirdBlock, uncheckedCheckCount) Expect(err).NotTo(HaveOccurred()) headerBlockNumbers := getBlockNumbers(headers) - Expect(headerBlockNumbers).NotTo(ContainElement(excludedHeader.BlockNumber)) + Expect(headerBlockNumbers).To(ConsistOf(firstBlock, secondBlock, thirdBlock)) + Expect(headerBlockNumbers).NotTo(ContainElement(lastBlock)) }) - It("includes header with block number > 45 back from latest with check count of 2", func() { - _, err = db.Exec(`UPDATE public.headers SET check_count = 1 WHERE id = $1`, secondHeaderID) - Expect(err).NotTo(HaveOccurred()) - - headers, err := repo.UncheckedHeaders(firstBlock, lastBlock, recheckCheckCount) + It("excludes headers that are before the first block number", func() { + headers, err := repo.UncheckedHeaders(firstBlock+1, thirdBlock, uncheckedCheckCount) Expect(err).NotTo(HaveOccurred()) headerBlockNumbers := getBlockNumbers(headers) - Expect(headerBlockNumbers).To(ContainElement(secondBlock)) + Expect(headerBlockNumbers).To(ConsistOf(secondBlock, thirdBlock)) + Expect(headerBlockNumbers).NotTo(ContainElement(firstBlock)) + Expect(headerBlockNumbers).NotTo(ContainElement(lastBlock)) }) - It("excludes header with block number < 45 back from latest with check count of 2", func() { - excludedHeader := fakes.GetFakeHeader(secondBlock + 1) - excludedHeaderID, createHeaderErr := headerRepository.CreateOrUpdateHeader(excludedHeader) - Expect(createHeaderErr).NotTo(HaveOccurred()) - _, updateHeaderErr := db.Exec(`UPDATE public.headers SET check_count = 2 WHERE id = $1`, excludedHeaderID) - Expect(updateHeaderErr).NotTo(HaveOccurred()) + It("excludes headers that have been checked more than the check count", func() { + Expect(repo.MarkHeaderChecked(secondHeaderID)).To(Succeed()) - headers, err := repo.UncheckedHeaders(firstBlock, lastBlock, 3) + headers, err := repo.UncheckedHeaders(firstBlock, thirdBlock, uncheckedCheckCount) Expect(err).NotTo(HaveOccurred()) headerBlockNumbers := getBlockNumbers(headers) - Expect(headerBlockNumbers).NotTo(ContainElement(excludedHeader.BlockNumber)) + Expect(headerBlockNumbers).To(ConsistOf(firstBlock, thirdBlock)) + Expect(headerBlockNumbers).NotTo(ContainElement(secondBlock)) }) - }) - It("only returns headers associated with any node", func() { - dbTwo := test_config.NewTestDB(core.Node{ID: "second"}) - headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo) - repoTwo := repositories.NewCheckedHeadersRepository(dbTwo) - for _, n := range blockNumbers { - _, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n + 10)) - Expect(err).NotTo(HaveOccurred()) - } - allHeaders := []int64{firstBlock, firstBlock + 10, secondBlock, secondBlock + 10, thirdBlock, thirdBlock + 10} + Describe("when header has already been checked", func() { + It("excludes headers that are before the first block number", func() { + Expect(repo.MarkHeaderChecked(thirdHeaderID)).To(Succeed()) - nodeOneMissingHeaders, err := repo.UncheckedHeaders(firstBlock, thirdBlock+10, uncheckedCheckCount) - Expect(err).NotTo(HaveOccurred()) - nodeOneHeaderBlockNumbers := getBlockNumbers(nodeOneMissingHeaders) - Expect(nodeOneHeaderBlockNumbers).To(ConsistOf(allHeaders)) + headers, err := repo.UncheckedHeaders(thirdBlock+1, lastBlock, recheckCheckCount) + Expect(err).NotTo(HaveOccurred()) - nodeTwoMissingHeaders, err := repoTwo.UncheckedHeaders(firstBlock, thirdBlock+10, uncheckedCheckCount) - Expect(err).NotTo(HaveOccurred()) - nodeTwoHeaderBlockNumbers := getBlockNumbers(nodeTwoMissingHeaders) - Expect(nodeTwoHeaderBlockNumbers).To(ConsistOf(allHeaders)) - }) - }) + headerBlockNumbers := getBlockNumbers(headers) + Expect(headerBlockNumbers).NotTo(ContainElement(thirdBlock)) + }) - Describe("when ending block is -1", func() { - It("includes all non-checked headers when ending block is -1 ", func() { - headers, err := repo.UncheckedHeaders(firstBlock, -1, uncheckedCheckCount) - Expect(err).NotTo(HaveOccurred()) + It("includes header with block number >= 15 back from latest with check count of 1", func() { + Expect(repo.MarkHeaderChecked(thirdHeaderID)).To(Succeed()) - headerBlockNumbers := getBlockNumbers(headers) - Expect(headerBlockNumbers).To(ConsistOf(firstBlock, secondBlock, thirdBlock, lastBlock)) - }) + headers, err := repo.UncheckedHeaders(firstBlock, lastBlock, recheckCheckCount) + Expect(err).NotTo(HaveOccurred()) - It("excludes headers that have been checked more than the check count", func() { - _, err = db.Exec(`UPDATE public.headers SET check_count = 1 WHERE id = $1`, headerIDs[1]) - Expect(err).NotTo(HaveOccurred()) + headerBlockNumbers := getBlockNumbers(headers) + Expect(headerBlockNumbers).To(ContainElement(thirdBlock)) + }) - headers, err := repo.UncheckedHeaders(firstBlock, -1, uncheckedCheckCount) - Expect(err).NotTo(HaveOccurred()) + It("excludes header with block number < 15 back from latest with check count of 1", func() { + excludedHeader := fakes.GetFakeHeader(thirdBlock + 1) + excludedHeaderID, createHeaderErr := headerRepository.CreateOrUpdateHeader(excludedHeader) + Expect(createHeaderErr).NotTo(HaveOccurred()) + Expect(repo.MarkHeaderChecked(excludedHeaderID)).To(Succeed()) - headerBlockNumbers := getBlockNumbers(headers) - Expect(headerBlockNumbers).To(ConsistOf(firstBlock, thirdBlock, lastBlock)) - Expect(headerBlockNumbers).NotTo(ContainElement(secondBlock)) - }) + headers, err := repo.UncheckedHeaders(firstBlock, lastBlock, recheckCheckCount) + Expect(err).NotTo(HaveOccurred()) + + headerBlockNumbers := getBlockNumbers(headers) + Expect(headerBlockNumbers).NotTo(ContainElement(excludedHeader.BlockNumber)) + }) + + It("includes header with block number > 45 back from latest with check count of 2", func() { + Expect(repo.MarkHeaderChecked(secondHeaderID)).To(Succeed()) + Expect(repo.MarkHeaderChecked(secondHeaderID)).To(Succeed()) + + headers, err := repo.UncheckedHeaders(firstBlock, lastBlock, 3) + Expect(err).NotTo(HaveOccurred()) + + headerBlockNumbers := getBlockNumbers(headers) + Expect(headerBlockNumbers).To(ContainElement(secondBlock)) + }) + + It("excludes header with block number < 45 back from latest with check count of 2", func() { + excludedHeader := fakes.GetFakeHeader(secondBlock + 1) + excludedHeaderID, createHeaderErr := headerRepository.CreateOrUpdateHeader(excludedHeader) + Expect(createHeaderErr).NotTo(HaveOccurred()) + + Expect(repo.MarkHeaderChecked(excludedHeaderID)).To(Succeed()) + Expect(repo.MarkHeaderChecked(excludedHeaderID)).To(Succeed()) + + headers, err := repo.UncheckedHeaders(firstBlock, lastBlock, 3) + Expect(err).NotTo(HaveOccurred()) + + headerBlockNumbers := getBlockNumbers(headers) + Expect(headerBlockNumbers).NotTo(ContainElement(excludedHeader.BlockNumber)) + }) + }) - Describe("when header has already been checked", func() { - It("includes header with block number > 15 back from latest with check count of 1", func() { - _, err = db.Exec(`UPDATE public.headers SET check_count = 1 WHERE id = $1`, thirdHeaderID) + It("only returns headers associated with any node", func() { + dbTwo := test_config.NewTestDB(core.Node{ID: "second"}) + headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo) + repoTwo, repoErr := repositories.NewCheckedHeadersRepository(dbTwo, pluginSchemaName) + Expect(repoErr).NotTo(HaveOccurred()) + + for _, n := range blockNumbers { + _, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n + 10)) + Expect(err).NotTo(HaveOccurred()) + } + allHeaders := []int64{firstBlock, firstBlock + 10, secondBlock, secondBlock + 10, thirdBlock, thirdBlock + 10} + + nodeOneMissingHeaders, err := repo.UncheckedHeaders(firstBlock, thirdBlock+10, uncheckedCheckCount) Expect(err).NotTo(HaveOccurred()) + nodeOneHeaderBlockNumbers := getBlockNumbers(nodeOneMissingHeaders) + Expect(nodeOneHeaderBlockNumbers).To(ConsistOf(allHeaders)) + + nodeTwoMissingHeaders, err := repoTwo.UncheckedHeaders(firstBlock, thirdBlock+10, uncheckedCheckCount) + Expect(err).NotTo(HaveOccurred()) + nodeTwoHeaderBlockNumbers := getBlockNumbers(nodeTwoMissingHeaders) + Expect(nodeTwoHeaderBlockNumbers).To(ConsistOf(allHeaders)) + }) + }) - headers, err := repo.UncheckedHeaders(firstBlock, -1, recheckCheckCount) + Describe("when ending block is -1", func() { + It("includes all non-checked headers when ending block is -1 ", func() { + headers, err := repo.UncheckedHeaders(firstBlock, -1, uncheckedCheckCount) Expect(err).NotTo(HaveOccurred()) headerBlockNumbers := getBlockNumbers(headers) - Expect(headerBlockNumbers).To(ContainElement(thirdBlock)) + Expect(headerBlockNumbers).To(ConsistOf(firstBlock, secondBlock, thirdBlock, lastBlock)) }) - It("excludes header with block number < 15 back from latest with check count of 1", func() { - excludedHeader := fakes.GetFakeHeader(thirdBlock + 1) - excludedHeaderID, createHeaderErr := headerRepository.CreateOrUpdateHeader(excludedHeader) - Expect(createHeaderErr).NotTo(HaveOccurred()) - _, updateHeaderErr := db.Exec(`UPDATE public.headers SET check_count = 1 WHERE id = $1`, excludedHeaderID) - Expect(updateHeaderErr).NotTo(HaveOccurred()) + It("includes headers that have had their check-count reset", func() { + Expect(repo.MarkHeaderChecked(secondHeaderID)).To(Succeed()) + Expect(repo.MarkSingleHeaderUnchecked(secondBlock)).To(Succeed()) - headers, err := repo.UncheckedHeaders(firstBlock, -1, recheckCheckCount) + headers, err := repo.UncheckedHeaders(firstBlock, thirdBlock, uncheckedCheckCount) Expect(err).NotTo(HaveOccurred()) headerBlockNumbers := getBlockNumbers(headers) - Expect(headerBlockNumbers).NotTo(ContainElement(excludedHeader.BlockNumber)) + Expect(headerBlockNumbers).To(ConsistOf(firstBlock, secondBlock, thirdBlock)) + Expect(headerBlockNumbers).NotTo(ContainElement(lastBlock)) }) - It("includes header with block number > 45 back from latest with check count of 2", func() { - _, err = db.Exec(`UPDATE public.headers SET check_count = 1 WHERE id = $1`, secondHeaderID) - Expect(err).NotTo(HaveOccurred()) + It("excludes headers that have been checked more than the check count", func() { + Expect(repo.MarkHeaderChecked(headerIDs[1])).To(Succeed()) - headers, err := repo.UncheckedHeaders(firstBlock, -1, recheckCheckCount) + headers, err := repo.UncheckedHeaders(firstBlock, -1, uncheckedCheckCount) Expect(err).NotTo(HaveOccurred()) headerBlockNumbers := getBlockNumbers(headers) - Expect(headerBlockNumbers).To(ContainElement(secondBlock)) + Expect(headerBlockNumbers).To(ConsistOf(firstBlock, thirdBlock, lastBlock)) + Expect(headerBlockNumbers).NotTo(ContainElement(secondBlock)) }) - It("excludes header with block number < 45 back from latest with check count of 2", func() { - excludedHeader := fakes.GetFakeHeader(secondBlock + 1) - excludedHeaderID, createHeaderErr := headerRepository.CreateOrUpdateHeader(excludedHeader) - Expect(createHeaderErr).NotTo(HaveOccurred()) - _, updateHeaderErr := db.Exec(`UPDATE public.headers SET check_count = 2 WHERE id = $1`, excludedHeaderID) - Expect(updateHeaderErr).NotTo(HaveOccurred()) - - headers, err := repo.UncheckedHeaders(firstBlock, -1, 3) + It("excludes headers that are before the first block number", func() { + repo.MarkSingleHeaderUnchecked(secondBlock) + headers, err := repo.UncheckedHeaders(firstBlock+1, -1, uncheckedCheckCount) Expect(err).NotTo(HaveOccurred()) headerBlockNumbers := getBlockNumbers(headers) - Expect(headerBlockNumbers).NotTo(ContainElement(excludedHeader.BlockNumber)) + Expect(headerBlockNumbers).To(ConsistOf(secondBlock, thirdBlock, lastBlock)) + Expect(headerBlockNumbers).NotTo(ContainElement(firstBlock)) }) - }) - It("returns headers associated with any node", func() { - dbTwo := test_config.NewTestDB(core.Node{ID: "second"}) - headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo) - repoTwo := repositories.NewCheckedHeadersRepository(dbTwo) - for _, n := range blockNumbers { - _, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n + 10)) - Expect(err).NotTo(HaveOccurred()) - } - allHeaders := []int64{firstBlock, firstBlock + 10, secondBlock, secondBlock + 10, thirdBlock, thirdBlock + 10, lastBlock, lastBlock + 10} + Describe("when header has already been checked", func() { + It("includes header with block number > 15 back from latest with check count of 1", func() { + Expect(repo.MarkHeaderChecked(thirdHeaderID)).To(Succeed()) + + headers, err := repo.UncheckedHeaders(firstBlock, -1, recheckCheckCount) + Expect(err).NotTo(HaveOccurred()) + + headerBlockNumbers := getBlockNumbers(headers) + Expect(headerBlockNumbers).To(ContainElement(thirdBlock)) + }) + + It("excludes header with block number < 15 back from latest with check count of 1", func() { + excludedHeader := fakes.GetFakeHeader(thirdBlock + 1) + excludedHeaderID, createHeaderErr := headerRepository.CreateOrUpdateHeader(excludedHeader) + Expect(createHeaderErr).NotTo(HaveOccurred()) + Expect(repo.MarkHeaderChecked(excludedHeaderID)).To(Succeed()) + + headers, err := repo.UncheckedHeaders(firstBlock, -1, recheckCheckCount) + Expect(err).NotTo(HaveOccurred()) + + headerBlockNumbers := getBlockNumbers(headers) + Expect(headerBlockNumbers).NotTo(ContainElement(excludedHeader.BlockNumber)) + }) + + It("includes header with block number > 45 back from latest with check count of 2", func() { + Expect(repo.MarkHeaderChecked(secondHeaderID)).To(Succeed()) + Expect(repo.MarkHeaderChecked(secondHeaderID)).To(Succeed()) + + headers, err := repo.UncheckedHeaders(firstBlock, -1, 3) + Expect(err).NotTo(HaveOccurred()) + + headerBlockNumbers := getBlockNumbers(headers) + Expect(headerBlockNumbers).To(ContainElement(secondBlock)) + }) - nodeOneMissingHeaders, err := repo.UncheckedHeaders(firstBlock, -1, uncheckedCheckCount) - Expect(err).NotTo(HaveOccurred()) - nodeOneBlockNumbers := getBlockNumbers(nodeOneMissingHeaders) - Expect(nodeOneBlockNumbers).To(ConsistOf(allHeaders)) + It("excludes header with block number < 45 back from latest with check count of 2", func() { + excludedHeader := fakes.GetFakeHeader(secondBlock + 1) + excludedHeaderID, createHeaderErr := headerRepository.CreateOrUpdateHeader(excludedHeader) + Expect(createHeaderErr).NotTo(HaveOccurred()) - nodeTwoMissingHeaders, err := repoTwo.UncheckedHeaders(firstBlock, -1, uncheckedCheckCount) - Expect(err).NotTo(HaveOccurred()) - nodeTwoBlockNumbers := getBlockNumbers(nodeTwoMissingHeaders) - Expect(nodeTwoBlockNumbers).To(ConsistOf(allHeaders)) + Expect(repo.MarkHeaderChecked(excludedHeaderID)).To(Succeed()) + Expect(repo.MarkHeaderChecked(excludedHeaderID)).To(Succeed()) + + headers, err := repo.UncheckedHeaders(firstBlock, -1, 3) + Expect(err).NotTo(HaveOccurred()) + + headerBlockNumbers := getBlockNumbers(headers) + Expect(headerBlockNumbers).NotTo(ContainElement(excludedHeader.BlockNumber)) + }) + }) + + It("returns headers associated with any node", func() { + dbTwo := test_config.NewTestDB(core.Node{ID: "second"}) + headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo) + repoTwo, repoErr := repositories.NewCheckedHeadersRepository(dbTwo, pluginSchemaName) + Expect(repoErr).NotTo(HaveOccurred()) + + for _, n := range blockNumbers { + _, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n + 10)) + Expect(err).NotTo(HaveOccurred()) + } + allHeaders := []int64{firstBlock, firstBlock + 10, secondBlock, secondBlock + 10, thirdBlock, thirdBlock + 10, lastBlock, lastBlock + 10} + + nodeOneMissingHeaders, err := repo.UncheckedHeaders(firstBlock, -1, uncheckedCheckCount) + Expect(err).NotTo(HaveOccurred()) + nodeOneBlockNumbers := getBlockNumbers(nodeOneMissingHeaders) + Expect(nodeOneBlockNumbers).To(ConsistOf(allHeaders)) + + nodeTwoMissingHeaders, err := repoTwo.UncheckedHeaders(firstBlock, -1, uncheckedCheckCount) + Expect(err).NotTo(HaveOccurred()) + nodeTwoBlockNumbers := getBlockNumbers(nodeTwoMissingHeaders) + Expect(nodeTwoBlockNumbers).To(ConsistOf(allHeaders)) + }) }) }) }) @@ -349,3 +416,39 @@ func getBlockNumbers(headers []core.Header) []int64 { } return headerBlockNumbers } + +func selectCheckedHeaders(db *postgres.DB, schemaName string, headerID int64) (int, error) { + var checkedCount int + queryString := fmt.Sprintf(`SELECT check_count FROM %s.checked_headers WHERE header_id = $1`, schemaName) + fetchErr := db.Get(&checkedCount, queryString, headerID) + return checkedCount, fetchErr +} + +func createPluginSchema(db *postgres.DB, schemaName string) error { + prepareSchema := ` +CREATE SCHEMA IF NOT EXISTS %[1]s; +` + + _, schemaError := db.Exec(fmt.Sprintf(prepareSchema, schemaName)) + return schemaError +} + +func createPluginCheckedHeadersTable(db *postgres.DB, schemaName string) error { + prepareSchema := ` +CREATE SCHEMA IF NOT EXISTS %[1]s; + +CREATE TABLE %[1]s.checked_headers ( + id SERIAL PRIMARY KEY, + check_count INTEGER NOT NULL DEFAULT 0, + header_id INTEGER NOT NULL REFERENCES public.headers(id) ON DELETE CASCADE, + UNIQUE (header_id) +); +` + _, schemaError := db.Exec(fmt.Sprintf(prepareSchema, schemaName)) + return schemaError +} + +func clearPluginSchema(db *postgres.DB, schemaName string) error { + _, err := db.Exec(fmt.Sprintf("DROP SCHEMA IF EXISTS %s CASCADE;", schemaName)) + return err +} diff --git a/pkg/datastore/postgres/repositories/event_log_repository.go b/pkg/datastore/postgres/repositories/event_log_repository.go index ae40fcde6..6bf5d6fec 100644 --- a/pkg/datastore/postgres/repositories/event_log_repository.go +++ b/pkg/datastore/postgres/repositories/event_log_repository.go @@ -60,7 +60,9 @@ type rawEventLog struct { func (repo EventLogRepository) GetUntransformedEventLogs(minID, limit int) ([]core.EventLog, error) { var rawLogs []rawEventLog - query := fmt.Sprintf("SELECT * FROM public.event_logs WHERE transformed is false AND id > %d ORDER BY id ASC LIMIT %d", minID, limit) + query := fmt.Sprintf("SELECT id, header_id, address, topics, data, block_number, block_hash,"+ + "tx_hash, tx_index, log_index, transformed, raw FROM public.event_logs "+ + "WHERE transformed = false AND id > %d ORDER BY id ASC LIMIT %d", minID, limit) err := repo.db.Select(&rawLogs, query) if err != nil { return nil, err diff --git a/pkg/datastore/postgres/repositories/event_log_repository_test.go b/pkg/datastore/postgres/repositories/event_log_repository_test.go index 57240e6ee..0cc1448b3 100644 --- a/pkg/datastore/postgres/repositories/event_log_repository_test.go +++ b/pkg/datastore/postgres/repositories/event_log_repository_test.go @@ -71,7 +71,8 @@ var _ = Describe("Event log repository", func() { Expect(err).NotTo(HaveOccurred()) var dbLog rawEventLog - lookupErr := db.Get(&dbLog, `SELECT * FROM public.event_logs`) + lookupErr := db.Get(&dbLog, `SELECT id, header_id, address, topics, data, block_number, block_hash, + tx_hash, tx_index, log_index, transformed, raw FROM public.event_logs`) Expect(lookupErr).NotTo(HaveOccurred()) Expect(dbLog.ID).NotTo(BeZero()) Expect(dbLog.HeaderID).To(Equal(headerID)) @@ -119,7 +120,8 @@ var _ = Describe("Event log repository", func() { Expect(err).NotTo(HaveOccurred()) var dbLog rawEventLog - lookupErr := db.Get(&dbLog, `SELECT * FROM public.event_logs`) + lookupErr := db.Get(&dbLog, `SELECT id, header_id, address, topics, data, block_number, block_hash, + tx_hash, tx_index, log_index, transformed, raw FROM public.event_logs`) Expect(lookupErr).NotTo(HaveOccurred()) var logTopics []common.Hash diff --git a/pkg/eth/client/eth_client.go b/pkg/eth/client/eth_client.go index 0b335ee6f..b708d3e31 100644 --- a/pkg/eth/client/eth_client.go +++ b/pkg/eth/client/eth_client.go @@ -18,11 +18,13 @@ package client import ( "context" + "math/big" + "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/ethclient" - "math/big" ) type EthClient struct { @@ -49,6 +51,10 @@ func (client EthClient) HeaderByNumber(ctx context.Context, number *big.Int) (*t return client.client.HeaderByNumber(ctx, number) } +func (client EthClient) SubscribeNewStateChanges(ctx context.Context, q ethereum.FilterQuery, ch chan<- filters.Payload) (ethereum.Subscription, error) { + return client.client.SubscribeNewStateChanges(ctx, q, ch) +} + func (client EthClient) TransactionSender(ctx context.Context, tx *types.Transaction, block common.Hash, index uint) (common.Address, error) { return client.client.TransactionSender(ctx, tx, block, index) } diff --git a/pkg/fakes/mock_eth_client.go b/pkg/fakes/mock_eth_client.go index 59d09cb5f..4edc4b958 100644 --- a/pkg/fakes/mock_eth_client.go +++ b/pkg/fakes/mock_eth_client.go @@ -23,38 +23,43 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/filters" + "github.com/ethereum/go-ethereum/rpc" "github.com/makerdao/vulcanizedb/pkg/core" . "github.com/onsi/gomega" ) type MockEthClient struct { - callContractErr error - callContractPassedContext context.Context - callContractPassedMsg ethereum.CallMsg - callContractPassedNumber *big.Int - callContractReturnBytes []byte - blockByNumberErr error - blockByNumberPassedContext context.Context - blockByNumberPassedNumber *big.Int - blockByNumberReturnBlock *types.Block - headerByNumberErr error - headerByNumberPassedContext context.Context - headerByNumberPassedNumber *big.Int - headerByNumberReturnHeader *types.Header - headerByNumbersReturnHeader []*types.Header - headerByNumbersPassedNumber []*big.Int - filterLogsErr error - filterLogsPassedContext context.Context - filterLogsPassedQuery ethereum.FilterQuery - filterLogsReturnLogs []types.Log - transactionReceipts map[string]*types.Receipt - err error - passedBatch []core.BatchElem - passedMethod string - transactionSenderErr error - transactionReceiptErr error - passedAddress common.Address - passedBlockNumber *big.Int + callContractErr error + callContractPassedContext context.Context + callContractPassedMsg ethereum.CallMsg + callContractPassedNumber *big.Int + callContractReturnBytes []byte + blockByNumberErr error + blockByNumberPassedContext context.Context + blockByNumberPassedNumber *big.Int + blockByNumberReturnBlock *types.Block + headerByNumberErr error + headerByNumberPassedContext context.Context + headerByNumberPassedNumber *big.Int + headerByNumberReturnHeader *types.Header + headerByNumbersReturnHeader []*types.Header + headerByNumbersPassedNumber []*big.Int + filterLogsErr error + filterLogsPassedContext context.Context + filterLogsPassedQuery ethereum.FilterQuery + filterLogsReturnLogs []types.Log + transactionReceipts map[string]*types.Receipt + err error + passedBatch []core.BatchElem + passedMethod string + transactionSenderErr error + transactionReceiptErr error + passedAddress common.Address + passedBlockNumber *big.Int + subscribeStateChangeErr error + passedStateChangeFilterQuery ethereum.FilterQuery + passedStateChangePayloadCh chan<- filters.Payload } func NewMockEthClient() *MockEthClient { @@ -148,6 +153,13 @@ func (client *MockEthClient) FilterLogs(ctx context.Context, q ethereum.FilterQu return client.filterLogsReturnLogs, client.filterLogsErr } +func (client *MockEthClient) SubscribeNewStateChanges(ctx context.Context, q ethereum.FilterQuery, ch chan<- filters.Payload) (ethereum.Subscription, error) { + client.passedStateChangeFilterQuery = q + client.passedStateChangePayloadCh = ch + subscription := rpc.ClientSubscription{} + return &subscription, nil +} + func (client *MockEthClient) TransactionSender(ctx context.Context, tx *types.Transaction, block common.Hash, index uint) (common.Address, error) { return common.HexToAddress("0x123"), client.transactionSenderErr } @@ -187,3 +199,8 @@ func (client *MockEthClient) AssertFilterLogsCalledWith(ctx context.Context, q e func (client *MockEthClient) AssertBatchCalledWith(method string) { Expect(client.passedMethod).To(Equal(method)) } + +func (client *MockEthClient) AssertSubscribeNewStateChangesCalledWith(filterQuery ethereum.FilterQuery, payloadCh chan<- filters.Payload) { + Expect(client.passedStateChangeFilterQuery).To(Equal(filterQuery)) + Expect(client.passedStateChangePayloadCh).To(Equal(payloadCh)) +} diff --git a/pkg/fakes/mock_header_repository.go b/pkg/fakes/mock_header_repository.go index 50f9e6443..0d934e4ab 100644 --- a/pkg/fakes/mock_header_repository.go +++ b/pkg/fakes/mock_header_repository.go @@ -32,9 +32,9 @@ type MockHeaderRepository struct { GetHeaderByIDError error GetHeaderByIDHeaderToReturn core.Header GetHeaderPassedBlockNumber int64 - GetHeadersInRangeEndingBlock int64 + GetHeadersInRangeEndingBlocks []int64 GetHeadersInRangeError error - GetHeadersInRangeStartingBlock int64 + GetHeadersInRangeStartingBlocks []int64 MostRecentHeaderBlockNumber int64 MostRecentHeaderBlockNumberErr error createOrUpdateHeaderCallCount int @@ -90,8 +90,8 @@ func (mock *MockHeaderRepository) GetHeaderByID(id int64) (core.Header, error) { } func (mock *MockHeaderRepository) GetHeadersInRange(startingBlock, endingBlock int64) ([]core.Header, error) { - mock.GetHeadersInRangeStartingBlock = startingBlock - mock.GetHeadersInRangeEndingBlock = endingBlock + mock.GetHeadersInRangeStartingBlocks = append(mock.GetHeadersInRangeStartingBlocks, startingBlock) + mock.GetHeadersInRangeEndingBlocks = append(mock.GetHeadersInRangeEndingBlocks, endingBlock) return mock.AllHeaders, mock.GetHeadersInRangeError } diff --git a/pkg/fakes/mock_rpc_client.go b/pkg/fakes/mock_rpc_client.go index f18b4b886..dbe5f710c 100644 --- a/pkg/fakes/mock_rpc_client.go +++ b/pkg/fakes/mock_rpc_client.go @@ -23,9 +23,9 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/rpc" - "github.com/ethereum/go-ethereum/statediff" "github.com/makerdao/vulcanizedb/pkg/core" "github.com/makerdao/vulcanizedb/pkg/eth/client" . "github.com/onsi/gomega" @@ -45,7 +45,7 @@ type MockRpcClient struct { passedResult interface{} passedBatch []core.BatchElem passedNamespace string - passedPayloadChan chan statediff.Payload + passedPayloadChan chan filters.Payload passedSubscribeArgs []interface{} lengthOfBatch int returnPOAHeader core.POAHeader @@ -61,7 +61,7 @@ func NewMockRpcClient() *MockRpcClient { func (c *MockRpcClient) Subscribe(namespace string, payloadChan interface{}, args ...interface{}) (core.Subscription, error) { c.passedNamespace = namespace - passedPayloadChan, ok := payloadChan.(chan statediff.Payload) + passedPayloadChan, ok := payloadChan.(chan filters.Payload) if !ok { return nil, errors.New("passed in channel is not of the correct type") } @@ -75,7 +75,7 @@ func (c *MockRpcClient) Subscribe(namespace string, payloadChan interface{}, arg return client.Subscription{RpcSubscription: &subscription}, nil } -func (c *MockRpcClient) AssertSubscribeCalledWith(namespace string, payloadChan chan statediff.Payload, args []interface{}) { +func (c *MockRpcClient) AssertSubscribeCalledWith(namespace string, payloadChan chan filters.Payload, args []interface{}) { Expect(c.passedNamespace).To(Equal(namespace)) Expect(c.passedPayloadChan).To(Equal(payloadChan)) Expect(c.passedSubscribeArgs).To(Equal(args)) diff --git a/pkg/plugin/builder/builder.go b/pkg/plugin/builder/builder.go index 67157fa4d..0b2030539 100644 --- a/pkg/plugin/builder/builder.go +++ b/pkg/plugin/builder/builder.go @@ -66,7 +66,7 @@ func (b *builder) BuildPlugin() error { } // Build the .go file into a .so plugin - execErr := exec.Command("go", "build", "-buildmode=plugin", "-o", soFile, b.goFile).Run() + execErr := exec.Command("go", "build", "-mod=mod", "-buildmode=plugin", "-o", soFile, b.goFile).Run() if execErr != nil { return errors.New(fmt.Sprintf("unable to build .so file: %s", execErr.Error())) } diff --git a/pkg/plugin/generator.go b/pkg/plugin/generator.go index 842fd37e2..695c6e140 100644 --- a/pkg/plugin/generator.go +++ b/pkg/plugin/generator.go @@ -18,6 +18,7 @@ package plugin import ( "errors" + "github.com/makerdao/vulcanizedb/pkg/config" "github.com/makerdao/vulcanizedb/pkg/plugin/builder" "github.com/makerdao/vulcanizedb/pkg/plugin/manager" @@ -27,6 +28,7 @@ import ( // Generator is the top-level interface for creating transformer plugins type Generator interface { GenerateExporterPlugin() error + SetMigrationManager(manager.MigrationManager) } type generator struct { @@ -66,3 +68,8 @@ func (g *generator) GenerateExporterPlugin() error { // Perform db migrations for the transformers return g.MigrationManager.RunMigrations() } + +// Sets the Migration manager - typically used for testing +func (g *generator) SetMigrationManager(mgr manager.MigrationManager) { + g.MigrationManager = mgr +} diff --git a/pkg/plugin/manager/manager.go b/pkg/plugin/manager/manager.go index 6e8b7f148..34ae6296f 100644 --- a/pkg/plugin/manager/manager.go +++ b/pkg/plugin/manager/manager.go @@ -60,34 +60,77 @@ func (m *manager) setDB() error { } dbConnector, err := pq.NewConnector(pgStr) if err != nil { - return errors.New(fmt.Sprintf("can't connect to db: %s", err.Error())) + return fmt.Errorf("can't connect to db: %w", err) } m.db = sql.OpenDB(dbConnector) return nil } func (m *manager) RunMigrations() error { + if len(m.GenConfig.Schema) <= 0 { + return errors.New("config is missing a schema, required for plugin migrations") + } + // Get paths to db migrations from the plugin config paths, err := m.GenConfig.GetMigrationsPaths() if err != nil { - return err + return fmt.Errorf("could not get migration paths %w", err) } if len(paths) < 1 { return nil } + // Init directory for temporary copies of migrations - err = m.setupMigrationEnv() - if err != nil { - return err + setupErr := m.setupMigrationEnv() + if setupErr != nil { + return fmt.Errorf("could not setup migration env %w", setupErr) } defer m.cleanUp() + + // First run the public schema migrations + migrationErr := m.runPublicMigrations() + if migrationErr != nil { + return fmt.Errorf("could not run public migrations %w", migrationErr) + } + // Creates copies of migrations for all the plugin's transformers in a tmp dir - err = m.createMigrationCopies(paths) - if err != nil { - return err + schemaMigrationErr := m.createMigrationCopies(paths) + if schemaMigrationErr != nil { + return fmt.Errorf("could not run schema migrations %w", schemaMigrationErr) + } + + return nil +} + +func (m *manager) runPublicMigrations() error { + // Setup DB if not set + if m.db == nil { + setErr := m.setDB() + if setErr != nil { + return fmt.Errorf("could not open db: %w", setErr) + } + } + + goose.SetTableName("public.goose_db_version") + + path, pathErr := helpers.CleanPath(filepath.Join("$GOPATH/src", m.GenConfig.Home, "db", "migrations")) + + if pathErr != nil { + return fmt.Errorf("could not construct filepath %w", pathErr) } + fixErr := goose.Fix(path) + if fixErr != nil { + return fmt.Errorf("version fixing for plugin migrations at %s failed: %w", path, fixErr) + } + + // Run the copied migrations with goose + upErr := goose.Up(m.db, path) + if upErr != nil { + return fmt.Errorf("db migrations for plugin transformers at %s failed: %w", path, upErr) + } return nil + } // Setup a temporary directory to hold transformer db migrations @@ -95,17 +138,17 @@ func (m *manager) setupMigrationEnv() error { var err error m.tmpMigDir, err = helpers.CleanPath(filepath.Join("$GOPATH/src", m.GenConfig.Home, ".plugin_migrations")) if err != nil { - return err + return fmt.Errorf("unable to construct clean path %w", err) } removeErr := os.RemoveAll(m.tmpMigDir) if removeErr != nil { - removeErrString := "unable to remove file found at %s where tmp directory needs to be written: %s" - return errors.New(fmt.Sprintf(removeErrString, m.tmpMigDir, removeErr.Error())) + removeErrString := "unable to remove file found at %s where tmp directory needs to be written: %w" + return fmt.Errorf(removeErrString, m.tmpMigDir, removeErr) } mkdirErr := os.Mkdir(m.tmpMigDir, os.ModePerm) if mkdirErr != nil { - mkdirErrString := "unable to create temporary migration directory %s: %s" - return errors.New(fmt.Sprintf(mkdirErrString, m.tmpMigDir, mkdirErr.Error())) + mkdirErrString := "unable to create temporary migration directory %s: %w" + return fmt.Errorf(mkdirErrString, m.tmpMigDir, mkdirErr) } return nil @@ -146,18 +189,27 @@ func (m *manager) fixAndRun(path string) error { if m.db == nil { setErr := m.setDB() if setErr != nil { - return errors.New(fmt.Sprintf("could not open db: %s", setErr.Error())) + return fmt.Errorf("could not open db: %w", setErr) } } + + _, execErr := m.db.Exec(fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS %s", m.GenConfig.Schema)) + if execErr != nil { + return fmt.Errorf("could not create schema %s, %w", m.GenConfig.Schema, execErr) + } + + goose.SetTableName(fmt.Sprintf("%s.goose_db_version", m.GenConfig.Schema)) + // Fix the migrations fixErr := goose.Fix(m.tmpMigDir) if fixErr != nil { - return errors.New(fmt.Sprintf("version fixing for plugin migrations at %s failed: %s", path, fixErr.Error())) + return fmt.Errorf("version fixing for plugin migrations at %s failed: %w", path, fixErr) } + // Run the copied migrations with goose upErr := goose.Up(m.db, m.tmpMigDir) if upErr != nil { - return errors.New(fmt.Sprintf("db migrations for plugin transformers at %s failed: %s", path, upErr.Error())) + return fmt.Errorf("db migrations for plugin transformers at %s failed: %w", path, upErr) } return nil } diff --git a/test_config/test_config.go b/test_config/test_config.go index 468bf0e18..4a681c10a 100644 --- a/test_config/test_config.go +++ b/test_config/test_config.go @@ -17,7 +17,6 @@ package test_config import ( - "errors" "fmt" "os" @@ -31,7 +30,6 @@ import ( var TestConfig *viper.Viper var DBConfig config.Database -var TestClient config.Client var ABIFilePath string func init() { @@ -47,16 +45,6 @@ func setTestConfig() { if err != nil { logrus.Fatal(err) } - ipc := TestConfig.GetString("client.ipcPath") - - // If we don't have an ipc path in the config file, check the env variable - if ipc == "" { - TestConfig.BindEnv("url", "CLIENT_IPCPATH") - ipc = TestConfig.GetString("url") - } - if ipc == "" { - logrus.Fatal(errors.New("testing.toml IPC path or $CLIENT_IPCPATH env variable need to be set")) - } hn := TestConfig.GetString("database.hostname") port := TestConfig.GetInt("database.port") @@ -67,9 +55,6 @@ func setTestConfig() { Name: name, Port: port, } - TestClient = config.Client{ - IPCPath: ipc, - } } func setABIPath() {