From ef1055300ca9263e44ede2452c5839a6a77bff3c Mon Sep 17 00:00:00 2001 From: Rob Mulholand Date: Thu, 16 Jul 2020 11:51:36 -0500 Subject: [PATCH 01/66] (VDB-1532) Isolate beta env - Don't update on merges to staging, so that breaking changes can flow through staging => prod without applying to existing env --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a8ebb0648..320dd1f7f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,7 @@ deploy: - provider: script script: bash ./.travis/deploy.sh staging on: - branch: staging + branch: beta - provider: script script: bash ./.travis/deploy.sh prod on: From 18e4dbac32004697f531a2ebabc36ddea9ac7ad9 Mon Sep 17 00:00:00 2001 From: Rob Mulholand Date: Wed, 15 Jul 2020 16:41:45 -0500 Subject: [PATCH 02/66] (VDB-1516) Use "=" instead of "is" in events index - for untransformed events, so that auto-generated queries using the equality operator have access to the index --- db/migrations/00008_create_event_logs_table.sql | 2 +- db/schema.sql | 2 +- pkg/datastore/postgres/repositories/event_log_repository.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/db/migrations/00008_create_event_logs_table.sql b/db/migrations/00008_create_event_logs_table.sql index 0e867529c..64a473dd5 100644 --- a/db/migrations/00008_create_event_logs_table.sql +++ b/db/migrations/00008_create_event_logs_table.sql @@ -23,7 +23,7 @@ CREATE INDEX event_logs_transaction ON event_logs (tx_hash); CREATE INDEX event_logs_untransformed ON event_logs (transformed) - WHERE transformed is false; + WHERE transformed = false; -- +goose Down -- SQL in this section is executed when the migration is rolled back. diff --git a/db/schema.sql b/db/schema.sql index b42b2a761..24dd4702a 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -709,7 +709,7 @@ CREATE INDEX event_logs_transaction ON public.event_logs USING btree (tx_hash); -- Name: event_logs_untransformed; Type: INDEX; Schema: public; Owner: - -- -CREATE INDEX event_logs_untransformed ON public.event_logs USING btree (transformed) WHERE (transformed IS FALSE); +CREATE INDEX event_logs_untransformed ON public.event_logs USING btree (transformed) WHERE (transformed = false); -- diff --git a/pkg/datastore/postgres/repositories/event_log_repository.go b/pkg/datastore/postgres/repositories/event_log_repository.go index ae40fcde6..13c896042 100644 --- a/pkg/datastore/postgres/repositories/event_log_repository.go +++ b/pkg/datastore/postgres/repositories/event_log_repository.go @@ -60,7 +60,7 @@ type rawEventLog struct { func (repo EventLogRepository) GetUntransformedEventLogs(minID, limit int) ([]core.EventLog, error) { var rawLogs []rawEventLog - query := fmt.Sprintf("SELECT * FROM public.event_logs WHERE transformed is false AND id > %d ORDER BY id ASC LIMIT %d", minID, limit) + query := fmt.Sprintf("SELECT * FROM public.event_logs WHERE transformed = false AND id > %d ORDER BY id ASC LIMIT %d", minID, limit) err := repo.db.Select(&rawLogs, query) if err != nil { return nil, err From c43434f6a040c16e0a7d175287df85ffa45b5777 Mon Sep 17 00:00:00 2001 From: Rob Mulholand Date: Wed, 15 Jul 2020 16:46:54 -0500 Subject: [PATCH 03/66] (VDB-1517) Use "=" instead of "is" in storage index - for unchecked diffs, so that auto-generated queries using the equality operator have access to the index --- db/migrations/20200414084022_add_storage_diff_checked_index.sql | 2 +- db/schema.sql | 2 +- libraries/shared/storage/diff_repository.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/db/migrations/20200414084022_add_storage_diff_checked_index.sql b/db/migrations/20200414084022_add_storage_diff_checked_index.sql index 615382de0..2a9a16850 100644 --- a/db/migrations/20200414084022_add_storage_diff_checked_index.sql +++ b/db/migrations/20200414084022_add_storage_diff_checked_index.sql @@ -1,5 +1,5 @@ -- +goose Up -CREATE INDEX storage_diff_checked_index ON public.storage_diff (checked) WHERE checked is false; +CREATE INDEX storage_diff_checked_index ON public.storage_diff (checked) WHERE checked = false; -- +goose Down diff --git a/db/schema.sql b/db/schema.sql index 24dd4702a..bb65902da 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -758,7 +758,7 @@ CREATE INDEX receipts_transaction ON public.receipts USING btree (transaction_id -- Name: storage_diff_checked_index; Type: INDEX; Schema: public; Owner: - -- -CREATE INDEX storage_diff_checked_index ON public.storage_diff USING btree (checked) WHERE (checked IS FALSE); +CREATE INDEX storage_diff_checked_index ON public.storage_diff USING btree (checked) WHERE (checked = false); -- diff --git a/libraries/shared/storage/diff_repository.go b/libraries/shared/storage/diff_repository.go index 0e43c38e5..648bf7037 100644 --- a/libraries/shared/storage/diff_repository.go +++ b/libraries/shared/storage/diff_repository.go @@ -65,7 +65,7 @@ func (repository diffRepository) CreateBackFilledStorageValue(rawDiff types.RawD func (repository diffRepository) GetNewDiffs(minID, limit int) ([]types.PersistedDiff, error) { var result []types.PersistedDiff - query := fmt.Sprintf("SELECT * FROM public.storage_diff WHERE checked IS false and id > %d ORDER BY id ASC LIMIT %d", minID, limit) + query := fmt.Sprintf("SELECT * FROM public.storage_diff WHERE checked = false and id > %d ORDER BY id ASC LIMIT %d", minID, limit) err := repository.db.Select(&result, query) if err != nil { return nil, fmt.Errorf("error getting unchecked storage diffs with id greater than %d: %w", minID, err) From 23a2a4465281597439d7d9abcee73f5f2b3498e5 Mon Sep 17 00:00:00 2001 From: Rob Mulholand Date: Wed, 29 Jul 2020 16:08:00 -0500 Subject: [PATCH 04/66] (VDB-1447) Rename migrations with numeric indexes - remove timestamps to avoid applying migrations later than those in plugins when spinning up core VDB processes - fixes issue where plugin migrations with earlier timestamps don't get applied after core VDB migrations have run --- ...unction.sql => 00010_create_get_or_create_header_function.sql} | 0 ...tion.sql => 00011_create_create_back_filled_diff_function.sql} | 0 ...checked_index.sql => 00012_add_storage_diff_checked_index.sql} | 0 ...tion.sql => 00013_update_create_back_filled_diff_function.sql} | 0 ...tamp_index.sql => 00014_add_headers_block_timestamp_index.sql} | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename db/migrations/{20200305202405_create_get_or_create_header_function.sql => 00010_create_get_or_create_header_function.sql} (100%) rename db/migrations/{20200406181508_create_create_back_filled_diff_function.sql => 00011_create_create_back_filled_diff_function.sql} (100%) rename db/migrations/{20200414084022_add_storage_diff_checked_index.sql => 00012_add_storage_diff_checked_index.sql} (100%) rename db/migrations/{20200415113349_update_create_back_filled_diff_function.sql => 00013_update_create_back_filled_diff_function.sql} (100%) rename db/migrations/{20200603213514_add_headers_block_timestamp_index.sql => 00014_add_headers_block_timestamp_index.sql} (100%) diff --git a/db/migrations/20200305202405_create_get_or_create_header_function.sql b/db/migrations/00010_create_get_or_create_header_function.sql similarity index 100% rename from db/migrations/20200305202405_create_get_or_create_header_function.sql rename to db/migrations/00010_create_get_or_create_header_function.sql diff --git a/db/migrations/20200406181508_create_create_back_filled_diff_function.sql b/db/migrations/00011_create_create_back_filled_diff_function.sql similarity index 100% rename from db/migrations/20200406181508_create_create_back_filled_diff_function.sql rename to db/migrations/00011_create_create_back_filled_diff_function.sql diff --git a/db/migrations/20200414084022_add_storage_diff_checked_index.sql b/db/migrations/00012_add_storage_diff_checked_index.sql similarity index 100% rename from db/migrations/20200414084022_add_storage_diff_checked_index.sql rename to db/migrations/00012_add_storage_diff_checked_index.sql diff --git a/db/migrations/20200415113349_update_create_back_filled_diff_function.sql b/db/migrations/00013_update_create_back_filled_diff_function.sql similarity index 100% rename from db/migrations/20200415113349_update_create_back_filled_diff_function.sql rename to db/migrations/00013_update_create_back_filled_diff_function.sql diff --git a/db/migrations/20200603213514_add_headers_block_timestamp_index.sql b/db/migrations/00014_add_headers_block_timestamp_index.sql similarity index 100% rename from db/migrations/20200603213514_add_headers_block_timestamp_index.sql rename to db/migrations/00014_add_headers_block_timestamp_index.sql From 4ed52e0d752219f60dbc666c5792b0f60cbabefc Mon Sep 17 00:00:00 2001 From: Rob Mulholand Date: Wed, 29 Jul 2020 16:23:05 -0500 Subject: [PATCH 05/66] (VDB-1449) Squash migrations - Don't include migrations that are modifying earlier migrations, instead updating the original migration - Also remove extraneous comments and index dropping --- db/migrations/00003_create_headers_table.sql | 16 ++-- .../00005_create_storage_diffs_table.sql | 3 + .../00006_create_transactions_table.sql | 1 - db/migrations/00007_create_receipts_table.sql | 2 - .../00008_create_event_logs_table.sql | 5 -- .../00009_create_watched_logs_table.sql | 2 - ...0_create_get_or_create_header_function.sql | 5 +- ...reate_create_back_filled_diff_function.sql | 5 +- .../00012_add_storage_diff_checked_index.sql | 6 -- ...pdate_create_back_filled_diff_function.sql | 78 ------------------- ...0014_add_headers_block_timestamp_index.sql | 6 -- db/schema.sql | 7 ++ 12 files changed, 22 insertions(+), 114 deletions(-) delete mode 100644 db/migrations/00012_add_storage_diff_checked_index.sql delete mode 100644 db/migrations/00013_update_create_back_filled_diff_function.sql delete mode 100644 db/migrations/00014_add_headers_block_timestamp_index.sql diff --git a/db/migrations/00003_create_headers_table.sql b/db/migrations/00003_create_headers_table.sql index 76aec7436..b1e1d6f07 100644 --- a/db/migrations/00003_create_headers_table.sql +++ b/db/migrations/00003_create_headers_table.sql @@ -29,17 +29,17 @@ CREATE TRIGGER header_updated FOR EACH ROW EXECUTE PROCEDURE set_header_updated(); --- Index is removed when table is -CREATE INDEX headers_block_number ON public.headers (block_number); -CREATE INDEX headers_check_count ON public.headers (check_count); -CREATE INDEX headers_eth_node ON public.headers (eth_node_id); +CREATE INDEX headers_block_number + ON public.headers (block_number); +CREATE INDEX headers_block_timestamp_index + ON public.headers (block_timestamp); +CREATE INDEX headers_check_count + ON public.headers (check_count); +CREATE INDEX headers_eth_node + ON public.headers (eth_node_id); -- +goose Down -DROP INDEX headers_block_number; -DROP INDEX headers_check_count; -DROP INDEX headers_eth_node; - DROP TRIGGER header_updated ON public.headers; DROP FUNCTION set_header_updated(); diff --git a/db/migrations/00005_create_storage_diffs_table.sql b/db/migrations/00005_create_storage_diffs_table.sql index 43806a3ea..2e91e4c86 100644 --- a/db/migrations/00005_create_storage_diffs_table.sql +++ b/db/migrations/00005_create_storage_diffs_table.sql @@ -12,5 +12,8 @@ CREATE TABLE public.storage_diff UNIQUE (block_height, block_hash, hashed_address, storage_key, storage_value) ); +CREATE INDEX storage_diff_checked_index + ON public.storage_diff (checked) WHERE checked = false; + -- +goose Down DROP TABLE public.storage_diff; \ No newline at end of file diff --git a/db/migrations/00006_create_transactions_table.sql b/db/migrations/00006_create_transactions_table.sql index 0c2831506..676892070 100644 --- a/db/migrations/00006_create_transactions_table.sql +++ b/db/migrations/00006_create_transactions_table.sql @@ -19,5 +19,4 @@ CREATE INDEX transactions_header ON transactions (header_id); -- +goose Down -DROP INDEX transactions_header; DROP TABLE transactions; diff --git a/db/migrations/00007_create_receipts_table.sql b/db/migrations/00007_create_receipts_table.sql index 2686f3ea6..bc57f10f9 100644 --- a/db/migrations/00007_create_receipts_table.sql +++ b/db/migrations/00007_create_receipts_table.sql @@ -20,6 +20,4 @@ CREATE INDEX receipts_transaction ON public.receipts (transaction_id); -- +goose Down -DROP INDEX receipts_transaction; -DROP INDEX receipts_contract_address; DROP TABLE receipts; diff --git a/db/migrations/00008_create_event_logs_table.sql b/db/migrations/00008_create_event_logs_table.sql index 64a473dd5..9cf50b9c7 100644 --- a/db/migrations/00008_create_event_logs_table.sql +++ b/db/migrations/00008_create_event_logs_table.sql @@ -1,5 +1,4 @@ -- +goose Up --- SQL in this section is executed when the migration is applied. CREATE TABLE public.event_logs ( id SERIAL PRIMARY KEY, @@ -26,8 +25,4 @@ CREATE INDEX event_logs_untransformed WHERE transformed = false; -- +goose Down --- SQL in this section is executed when the migration is rolled back. -DROP INDEX event_logs_transaction; -DROP INDEX event_logs_address; -DROP INDEX event_logs_untransformed; DROP TABLE event_logs; \ No newline at end of file diff --git a/db/migrations/00009_create_watched_logs_table.sql b/db/migrations/00009_create_watched_logs_table.sql index 4268a68a1..0e463e27f 100644 --- a/db/migrations/00009_create_watched_logs_table.sql +++ b/db/migrations/00009_create_watched_logs_table.sql @@ -1,5 +1,4 @@ -- +goose Up --- SQL in this section is executed when the migration is applied. CREATE TABLE public.watched_logs ( id SERIAL PRIMARY KEY, @@ -8,5 +7,4 @@ CREATE TABLE public.watched_logs ); -- +goose Down --- SQL in this section is executed when the migration is rolled back. DROP TABLE public.watched_logs; diff --git a/db/migrations/00010_create_get_or_create_header_function.sql b/db/migrations/00010_create_get_or_create_header_function.sql index 76a35eb60..b995cfd87 100644 --- a/db/migrations/00010_create_get_or_create_header_function.sql +++ b/db/migrations/00010_create_get_or_create_header_function.sql @@ -1,6 +1,4 @@ -- +goose Up --- SQL in this section is executed when the migration is applied. - -- +goose StatementBegin CREATE OR REPLACE FUNCTION public.get_or_create_header(block_number BIGINT, hash VARCHAR(66), raw JSONB, block_timestamp NUMERIC, eth_node_id INTEGER) RETURNS INTEGER AS @@ -47,7 +45,8 @@ $$ LANGUAGE plpgsql; -- +goose StatementEnd +COMMENT ON FUNCTION public.get_or_create_header(block_number BIGINT, hash VARCHAR, raw JSONB, block_timestamp NUMERIC, eth_node_id INTEGER) + IS E'@omit'; -- +goose Down --- SQL in this section is executed when the migration is rolled back. DROP FUNCTION public.get_or_create_header(block_number BIGINT, hash VARCHAR, raw JSONB, block_timestamp NUMERIC, eth_node_id INTEGER); diff --git a/db/migrations/00011_create_create_back_filled_diff_function.sql b/db/migrations/00011_create_create_back_filled_diff_function.sql index aadb6a238..b4e8150da 100644 --- a/db/migrations/00011_create_create_back_filled_diff_function.sql +++ b/db/migrations/00011_create_create_back_filled_diff_function.sql @@ -1,5 +1,4 @@ -- +goose Up --- SQL in this section is executed when the migration is applied. -- +goose StatementBegin CREATE OR REPLACE FUNCTION public.create_back_filled_diff(block_height BIGINT, block_hash BYTEA, hashed_address BYTEA, storage_key BYTEA, storage_value BYTEA) RETURNS VOID AS @@ -8,7 +7,8 @@ DECLARE last_storage_value BYTEA := ( SELECT storage_diff.storage_value FROM public.storage_diff - WHERE storage_diff.hashed_address = create_back_filled_diff.hashed_address + WHERE storage_diff.block_height <= create_back_filled_diff.block_height + AND storage_diff.hashed_address = create_back_filled_diff.hashed_address AND storage_diff.storage_key = create_back_filled_diff.storage_key ORDER BY storage_diff.block_height DESC LIMIT 1 @@ -42,5 +42,4 @@ COMMENT ON FUNCTION public.create_back_filled_diff(block_height BIGINT, block_ha IS E'@omit'; -- +goose Down --- SQL in this section is executed when the migration is rolled back. DROP FUNCTION public.create_back_filled_diff(block_height BIGINT, block_hash BYTEA, hashed_address BYTEA, storage_key BYTEA, storage_value BYTEA); diff --git a/db/migrations/00012_add_storage_diff_checked_index.sql b/db/migrations/00012_add_storage_diff_checked_index.sql deleted file mode 100644 index 2a9a16850..000000000 --- a/db/migrations/00012_add_storage_diff_checked_index.sql +++ /dev/null @@ -1,6 +0,0 @@ --- +goose Up -CREATE INDEX storage_diff_checked_index ON public.storage_diff (checked) WHERE checked = false; - - --- +goose Down -DROP INDEX storage_diff_checked_index; \ No newline at end of file diff --git a/db/migrations/00013_update_create_back_filled_diff_function.sql b/db/migrations/00013_update_create_back_filled_diff_function.sql deleted file mode 100644 index 2d613707c..000000000 --- a/db/migrations/00013_update_create_back_filled_diff_function.sql +++ /dev/null @@ -1,78 +0,0 @@ --- +goose Up --- +goose StatementBegin -CREATE OR REPLACE FUNCTION public.create_back_filled_diff(block_height BIGINT, block_hash BYTEA, hashed_address BYTEA, - storage_key BYTEA, storage_value BYTEA) RETURNS VOID AS -$$ -DECLARE - last_storage_value BYTEA := ( - SELECT storage_diff.storage_value - FROM public.storage_diff - WHERE storage_diff.block_height <= create_back_filled_diff.block_height - AND storage_diff.hashed_address = create_back_filled_diff.hashed_address - AND storage_diff.storage_key = create_back_filled_diff.storage_key - ORDER BY storage_diff.block_height DESC - LIMIT 1 - ); - empty_storage_value BYTEA := ( - SELECT '\x0000000000000000000000000000000000000000000000000000000000000000'::BYTEA - ); -BEGIN - IF last_storage_value = create_back_filled_diff.storage_value THEN - RETURN; - END IF; - - IF last_storage_value is null and create_back_filled_diff.storage_value = empty_storage_value THEN - RETURN; - END IF; - - INSERT INTO public.storage_diff (block_height, block_hash, hashed_address, storage_key, storage_value, - from_backfill) - VALUES (create_back_filled_diff.block_height, create_back_filled_diff.block_hash, - create_back_filled_diff.hashed_address, create_back_filled_diff.storage_key, - create_back_filled_diff.storage_value, true) - ON CONFLICT DO NOTHING; - - RETURN; -END -$$ - LANGUAGE plpgsql; --- +goose StatementEnd - --- +goose Down --- +goose StatementBegin -CREATE OR REPLACE FUNCTION public.create_back_filled_diff(block_height BIGINT, block_hash BYTEA, hashed_address BYTEA, - storage_key BYTEA, storage_value BYTEA) RETURNS VOID AS -$$ -DECLARE - last_storage_value BYTEA := ( - SELECT storage_diff.storage_value - FROM public.storage_diff - WHERE storage_diff.hashed_address = create_back_filled_diff.hashed_address - AND storage_diff.storage_key = create_back_filled_diff.storage_key - ORDER BY storage_diff.block_height DESC - LIMIT 1 - ); - empty_storage_value BYTEA := ( - SELECT '\x0000000000000000000000000000000000000000000000000000000000000000'::BYTEA - ); -BEGIN - IF last_storage_value = create_back_filled_diff.storage_value THEN - RETURN; - END IF; - - IF last_storage_value is null and create_back_filled_diff.storage_value = empty_storage_value THEN - RETURN; - END IF; - - INSERT INTO public.storage_diff (block_height, block_hash, hashed_address, storage_key, storage_value, - from_backfill) - VALUES (create_back_filled_diff.block_height, create_back_filled_diff.block_hash, - create_back_filled_diff.hashed_address, create_back_filled_diff.storage_key, - create_back_filled_diff.storage_value, true) - ON CONFLICT DO NOTHING; - - RETURN; -END -$$ - LANGUAGE plpgsql; --- +goose StatementEnd \ No newline at end of file diff --git a/db/migrations/00014_add_headers_block_timestamp_index.sql b/db/migrations/00014_add_headers_block_timestamp_index.sql deleted file mode 100644 index afa9d60a8..000000000 --- a/db/migrations/00014_add_headers_block_timestamp_index.sql +++ /dev/null @@ -1,6 +0,0 @@ --- +goose Up -CREATE INDEX headers_block_timestamp_index ON public.headers (block_timestamp); - - --- +goose Down -DROP INDEX headers_block_timestamp_index; \ No newline at end of file diff --git a/db/schema.sql b/db/schema.sql index bb65902da..f0b6363ef 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -112,6 +112,13 @@ END $$; +-- +-- Name: FUNCTION get_or_create_header(block_number bigint, hash character varying, raw jsonb, block_timestamp numeric, eth_node_id integer); Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON FUNCTION public.get_or_create_header(block_number bigint, hash character varying, raw jsonb, block_timestamp numeric, eth_node_id integer) IS '@omit'; + + -- -- Name: set_header_updated(); Type: FUNCTION; Schema: public; Owner: - -- From a9ae4957b1fbb21f89cafa279a3032eb5a1f8047 Mon Sep 17 00:00:00 2001 From: Elizabeth Date: Mon, 10 Aug 2020 15:42:54 -0500 Subject: [PATCH 06/66] Vdb 1425 remove hashed keys (#120) * Remove hashing storage keys on storage diffs * Remove adding hashed storage keys to keys lookup * Remove keccaking storage key from storage backfill --- libraries/shared/factories/storage/README.md | 1 - .../shared/factories/storage/keys_lookup.go | 2 - .../factories/storage/keys_lookup_test.go | 12 ----- .../storage/backfill/storage_value_loader.go | 2 +- .../backfill/storage_value_loader_test.go | 10 ++-- .../fetcher/geth_rpc_storage_fetcher_test.go | 6 +-- libraries/shared/storage/keys_lookup.go | 38 --------------- libraries/shared/storage/keys_lookup_test.go | 48 ------------------- libraries/shared/storage/types/diff.go | 2 +- libraries/shared/storage/types/diff_test.go | 2 +- 10 files changed, 11 insertions(+), 112 deletions(-) delete mode 100644 libraries/shared/storage/keys_lookup.go delete mode 100644 libraries/shared/storage/keys_lookup_test.go diff --git a/libraries/shared/factories/storage/README.md b/libraries/shared/factories/storage/README.md index 9fcc824d7..295c7c3e1 100644 --- a/libraries/shared/factories/storage/README.md +++ b/libraries/shared/factories/storage/README.md @@ -71,7 +71,6 @@ func (lookup *keysLookup) refreshMappings() error { if err != nil { return err } - lookup.mappings = utils.AddHashedKeys(lookup.mappings) return nil } ``` diff --git a/libraries/shared/factories/storage/keys_lookup.go b/libraries/shared/factories/storage/keys_lookup.go index ca90ec8e6..13d3ad513 100644 --- a/libraries/shared/factories/storage/keys_lookup.go +++ b/libraries/shared/factories/storage/keys_lookup.go @@ -20,7 +20,6 @@ import ( "fmt" "github.com/ethereum/go-ethereum/common" - "github.com/makerdao/vulcanizedb/libraries/shared/storage" "github.com/makerdao/vulcanizedb/libraries/shared/storage/types" "github.com/makerdao/vulcanizedb/pkg/datastore/postgres" ) @@ -59,7 +58,6 @@ func (lookup *keysLookup) Lookup(key common.Hash) (types.ValueMetadata, error) { if refreshErr != nil { return metadata, fmt.Errorf("error refreshing mappings in keys lookup: %w", refreshErr) } - lookup.mappings = storage.AddHashedKeys(lookup.mappings) metadata, ok = lookup.mappings[key] if !ok { return metadata, fmt.Errorf("%w: %s", types.ErrKeyNotFound, key.Hex()) diff --git a/libraries/shared/factories/storage/keys_lookup_test.go b/libraries/shared/factories/storage/keys_lookup_test.go index 5da469df6..f60990214 100644 --- a/libraries/shared/factories/storage/keys_lookup_test.go +++ b/libraries/shared/factories/storage/keys_lookup_test.go @@ -18,7 +18,6 @@ package storage_test import ( "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" "github.com/makerdao/vulcanizedb/libraries/shared/factories/storage" "github.com/makerdao/vulcanizedb/libraries/shared/mocks" "github.com/makerdao/vulcanizedb/libraries/shared/storage/types" @@ -91,17 +90,6 @@ var _ = Describe("Storage keys lookup", func() { Expect(metadata).To(Equal(fakeMetadata)) }) - It("returns metadata for hashed version of key (accommodates keys emitted from Geth)", func() { - fakeKey := test_data.FakeHash() - loader.StorageKeyMappings = map[common.Hash]types.ValueMetadata{fakeKey: fakeMetadata} - - hashedKey := common.BytesToHash(crypto.Keccak256(fakeKey.Bytes())) - metadata, err := lookup.Lookup(hashedKey) - - Expect(err).NotTo(HaveOccurred()) - Expect(metadata).To(Equal(fakeMetadata)) - }) - It("returns key not found error if key not found", func() { fakeKey := test_data.FakeHash() _, err := lookup.Lookup(fakeKey) diff --git a/libraries/shared/storage/backfill/storage_value_loader.go b/libraries/shared/storage/backfill/storage_value_loader.go index 14cd09365..62477cbe5 100644 --- a/libraries/shared/storage/backfill/storage_value_loader.go +++ b/libraries/shared/storage/backfill/storage_value_loader.go @@ -128,7 +128,7 @@ func (r *StorageValueLoader) getAndPersistStorageValues(blockNumber int64, heade HashedAddress: keccakOfAddress, BlockHash: blockHash, BlockHeight: int(blockNumber), - StorageKey: crypto.Keccak256Hash(key.Bytes()), + StorageKey: key, StorageValue: newValueHash, } createDiffErr := r.StorageDiffRepo.CreateBackFilledStorageValue(diff) diff --git a/libraries/shared/storage/backfill/storage_value_loader_test.go b/libraries/shared/storage/backfill/storage_value_loader_test.go index c9692f240..c8699c731 100644 --- a/libraries/shared/storage/backfill/storage_value_loader_test.go +++ b/libraries/shared/storage/backfill/storage_value_loader_test.go @@ -223,14 +223,14 @@ var _ = Describe("StorageValueLoader", func() { BlockHeight: int(blockOne), BlockHash: headerHashBytes, HashedAddress: crypto.Keccak256Hash(addressOne[:]), - StorageKey: crypto.Keccak256Hash(keyOne.Bytes()), + StorageKey: keyOne, StorageValue: valueOne, } expectedDiffTwo := types.RawDiff{ BlockHeight: int(blockOne), BlockHash: headerHashBytes, HashedAddress: crypto.Keccak256Hash(addressTwo[:]), - StorageKey: crypto.Keccak256Hash(keyTwo.Bytes()), + StorageKey: keyTwo, StorageValue: valueTwo, } @@ -256,21 +256,21 @@ var _ = Describe("StorageValueLoader", func() { BlockHeight: int(blockOne), BlockHash: headerHashBytes, HashedAddress: crypto.Keccak256Hash(addressOne[:]), - StorageKey: crypto.Keccak256Hash(keyOne.Bytes()), + StorageKey: keyOne, StorageValue: valueOne, } expectedDiffTwo := types.RawDiff{ BlockHeight: int(blockOne), BlockHash: headerHashBytes, HashedAddress: crypto.Keccak256Hash(addressTwo[:]), - StorageKey: crypto.Keccak256Hash(keyTwo.Bytes()), + StorageKey: keyTwo, StorageValue: valueTwo, } expectedDiffThree := types.RawDiff{ BlockHeight: int(blockTwo), BlockHash: common.HexToHash(blockTwoHeader.Hash), HashedAddress: crypto.Keccak256Hash(addressOne[:]), - StorageKey: crypto.Keccak256Hash(keyOne.Bytes()), + StorageKey: keyOne, StorageValue: valueTwo, } diff --git a/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher_test.go b/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher_test.go index 0364122de..489cab2a1 100644 --- a/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher_test.go +++ b/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher_test.go @@ -249,21 +249,21 @@ var _ = Describe("Geth RPC Storage Fetcher", func() { HashedAddress: crypto.Keccak256Hash(test_data.ContractLeafKey[:]), BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), BlockHeight: intHeight, - StorageKey: crypto.Keccak256Hash(test_data.StorageKey), + StorageKey: common.BytesToHash(test_data.StorageKey), StorageValue: common.BytesToHash(test_data.SmallStorageValue), } expectedDiff2 := types.RawDiff{ HashedAddress: crypto.Keccak256Hash(test_data.AnotherContractLeafKey[:]), BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), BlockHeight: intHeight, - StorageKey: crypto.Keccak256Hash(test_data.StorageKey), + StorageKey: common.BytesToHash(test_data.StorageKey), StorageValue: common.BytesToHash(test_data.LargeStorageValue), } expectedDiff3 := types.RawDiff{ HashedAddress: crypto.Keccak256Hash(test_data.AnotherContractLeafKey[:]), BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), BlockHeight: intHeight, - StorageKey: crypto.Keccak256Hash(test_data.StorageKey), + StorageKey: common.BytesToHash(test_data.StorageKey), StorageValue: common.BytesToHash(test_data.SmallStorageValue), } diff --git a/libraries/shared/storage/keys_lookup.go b/libraries/shared/storage/keys_lookup.go deleted file mode 100644 index 1344b94bf..000000000 --- a/libraries/shared/storage/keys_lookup.go +++ /dev/null @@ -1,38 +0,0 @@ -// VulcanizeDB -// Copyright © 2019 Vulcanize - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. - -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package storage - -import ( - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" - "github.com/makerdao/vulcanizedb/libraries/shared/storage/types" -) - -func AddHashedKeys(currentMappings map[common.Hash]types.ValueMetadata) map[common.Hash]types.ValueMetadata { - copyOfCurrentMappings := make(map[common.Hash]types.ValueMetadata) - for k, v := range currentMappings { - copyOfCurrentMappings[k] = v - } - for k, v := range copyOfCurrentMappings { - currentMappings[hashKey(k)] = v - } - return currentMappings -} - -func hashKey(key common.Hash) common.Hash { - return crypto.Keccak256Hash(key.Bytes()) -} diff --git a/libraries/shared/storage/keys_lookup_test.go b/libraries/shared/storage/keys_lookup_test.go deleted file mode 100644 index 76d80768c..000000000 --- a/libraries/shared/storage/keys_lookup_test.go +++ /dev/null @@ -1,48 +0,0 @@ -// VulcanizeDB -// Copyright © 2019 Vulcanize - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. - -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package storage_test - -import ( - "github.com/ethereum/go-ethereum/common" - "github.com/makerdao/vulcanizedb/libraries/shared/storage" - "github.com/makerdao/vulcanizedb/libraries/shared/storage/types" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -var _ = Describe("Storage keys lookup utils", func() { - Describe("AddHashedKeys", func() { - It("returns a copy of the map with an additional slot for the hashed version of every key", func() { - fakeMap := map[common.Hash]types.ValueMetadata{} - fakeStorageKey := common.HexToHash("72c72de6b203d67cb6cd54fc93300109fcc6fd6eac88e390271a3d548794d800") - var fakeMappingKey types.Key = "fakeKey" - fakeMetadata := types.ValueMetadata{ - Name: "fakeName", - Keys: map[types.Key]string{fakeMappingKey: "fakeValue"}, - Type: types.Uint48, - } - fakeMap[fakeStorageKey] = fakeMetadata - - result := storage.AddHashedKeys(fakeMap) - - Expect(len(result)).To(Equal(2)) - expectedHashedStorageKey := common.HexToHash("2165edb4e1c37b99b60fa510d84f939dd35d5cd1d1c8f299d6456ea09df65a76") - Expect(fakeMap[fakeStorageKey]).To(Equal(fakeMetadata)) - Expect(fakeMap[expectedHashedStorageKey]).To(Equal(fakeMetadata)) - }) - }) -}) diff --git a/libraries/shared/storage/types/diff.go b/libraries/shared/storage/types/diff.go index 9353cf641..bf9d1b0e8 100644 --- a/libraries/shared/storage/types/diff.go +++ b/libraries/shared/storage/types/diff.go @@ -87,7 +87,7 @@ func FromNewGethStateDiff(account statediff.AccountDiff, stateDiff *statediff.St HashedAddress: crypto.Keccak256Hash(account.Key), BlockHash: stateDiff.BlockHash, BlockHeight: int(stateDiff.BlockNumber.Int64()), - StorageKey: crypto.Keccak256Hash(storage.Key), + StorageKey: common.BytesToHash(storage.Key), StorageValue: common.BytesToHash(decodedRLPStorageValue), }, nil } diff --git a/libraries/shared/storage/types/diff_test.go b/libraries/shared/storage/types/diff_test.go index 7860045cf..d8fe99a59 100644 --- a/libraries/shared/storage/types/diff_test.go +++ b/libraries/shared/storage/types/diff_test.go @@ -148,7 +148,7 @@ var _ = Describe("Storage row parsing", func() { Expect(result.BlockHash).To(Equal(fakes.FakeHash)) expectedBlockHeight := int(stateDiff.BlockNumber.Int64()) Expect(result.BlockHeight).To(Equal(expectedBlockHeight)) - expectedStorageKey := crypto.Keccak256Hash(storageDiff.Key) + expectedStorageKey := common.BytesToHash(storageDiff.Key) Expect(result.StorageKey).To(Equal(expectedStorageKey)) expectedStorageValue := common.BytesToHash(storageValueBytes) Expect(result.StorageValue).To(Equal(expectedStorageValue)) From 89a4c0b00a08f7baf6ee56879a82a412cc4ed39b Mon Sep 17 00:00:00 2001 From: Elizabeth Date: Tue, 11 Aug 2020 13:29:37 -0500 Subject: [PATCH 07/66] VDB-1368: add new geth patch as diff source (#119) * Update geth version to updated statediff patch * Update geth statediff pkg references to filters pkg * Update AccountDiffs for new geth patch - remove CreatedAccounts and DeletedAccounts from StateDiff - remove Path and Proofs from StorageDiff - remove BlockRlp from Payload * Allow for watched storage addresses to be passed in as CLI arg * Use new geth statediff patch tag * Update geth fetcher logging * Log number of watched addresses * Log watched address strings * Remove dockerfile for extractDiffs command * Use config file for extract diffs watched addresses --- .travis/deploy.sh | 14 -- cmd/extractDiffs.go | 56 ++++-- dockerfiles/extract_diffs/Dockerfile | 48 ----- dockerfiles/extract_diffs/startup_script.sh | 41 ---- go.mod | 7 +- go.sum | 14 ++ libraries/shared/mocks/streamer.go | 10 +- .../fetcher/geth_rpc_storage_fetcher.go | 78 +++----- .../fetcher/geth_rpc_storage_fetcher_test.go | 182 +++--------------- libraries/shared/storage/types/diff.go | 27 +-- libraries/shared/storage/types/diff_test.go | 73 +------ ...f_streamer.go => state_change_streamer.go} | 27 +-- ..._test.go => state_change_streamer_test.go} | 19 +- libraries/shared/test_data/statediff.go | 82 ++++---- pkg/core/eth_client.go | 2 + pkg/eth/client/eth_client.go | 8 +- pkg/fakes/mock_eth_client.go | 71 ++++--- pkg/fakes/mock_rpc_client.go | 8 +- 18 files changed, 247 insertions(+), 520 deletions(-) delete mode 100644 dockerfiles/extract_diffs/Dockerfile delete mode 100755 dockerfiles/extract_diffs/startup_script.sh rename libraries/shared/streamer/{statediff_streamer.go => state_change_streamer.go} (53%) rename libraries/shared/streamer/{statediff_streamer_test.go => state_change_streamer_test.go} (60%) diff --git a/.travis/deploy.sh b/.travis/deploy.sh index e54ae76ac..c2c7788e2 100755 --- a/.travis/deploy.sh +++ b/.travis/deploy.sh @@ -29,9 +29,6 @@ fi message BUILDING HEADER-SYNC docker build -f dockerfiles/header_sync/Dockerfile . -t makerdao/vdb-headersync:$TAG -message BUILDING EXTRACT-DIFFS -docker build -f dockerfiles/extract_diffs/Dockerfile . -t makerdao/vdb-extract-diffs:$TAG - message BUILDING RESET-HEADER-CHECK docker build -f dockerfiles/reset_header_check_count/Dockerfile . -t makerdao/vdb-reset-header-check:$TAG @@ -42,9 +39,6 @@ echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USER" --password-stdi message PUSHING HEADER-SYNC docker push makerdao/vdb-headersync:$TAG -message PUSHING EXTRACT-DIFFS -docker push makerdao/vdb-extract-diffs:$TAG - message PUSHING RESET-HEADER-CHECK docker push makerdao/vdb-reset-header-check:$TAG @@ -53,17 +47,9 @@ if [ "$ENVIRONMENT" == "prod" ]; then message DEPLOYING HEADER-SYNC aws ecs update-service --cluster vdb-cluster-$ENVIRONMENT --service vdb-header-sync-$ENVIRONMENT --force-new-deployment --endpoint https://ecs.$PROD_REGION.amazonaws.com --region $PROD_REGION - message DEPLOYING EXTRACT-DIFFS - aws ecs update-service --cluster vdb-cluster-$ENVIRONMENT --service vdb-extract-diffs-$ENVIRONMENT --force-new-deployment --endpoint https://ecs.$PROD_REGION.amazonaws.com --region $PROD_REGION elif [ "$ENVIRONMENT" == "staging" ]; then message DEPLOYING HEADER-SYNC aws ecs update-service --cluster vdb-cluster-$ENVIRONMENT --service vdb-header-sync-$ENVIRONMENT --force-new-deployment --endpoint https://ecs.$STAGING_REGION.amazonaws.com --region $STAGING_REGION - - message DEPLOYING EXTRACT-DIFFS - aws ecs update-service --cluster vdb-cluster-$ENVIRONMENT --service vdb-extract-diffs-$ENVIRONMENT --force-new-deployment --endpoint https://ecs.$STAGING_REGION.amazonaws.com --region $STAGING_REGION - - message DEPLOYING EXTRACT-DIFFS-NEW-GETH - aws ecs update-service --cluster vdb-cluster-$ENVIRONMENT --service vdb-extract-diffs2-$ENVIRONMENT --force-new-deployment --endpoint https://ecs.$STAGING_REGION.amazonaws.com --region $STAGING_REGION else message UNKNOWN ENVIRONMENT fi diff --git a/cmd/extractDiffs.go b/cmd/extractDiffs.go index 844811aa6..02959e141 100644 --- a/cmd/extractDiffs.go +++ b/cmd/extractDiffs.go @@ -1,7 +1,11 @@ package cmd import ( - "github.com/ethereum/go-ethereum/statediff" + "strings" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/makerdao/vulcanizedb/libraries/shared/storage" "github.com/makerdao/vulcanizedb/libraries/shared/storage/fetcher" "github.com/makerdao/vulcanizedb/libraries/shared/streamer" @@ -9,6 +13,7 @@ import ( "github.com/makerdao/vulcanizedb/utils" "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "github.com/spf13/viper" ) // extractDiffsCmd represents the extractDiffs command @@ -29,10 +34,22 @@ func init() { rootCmd.AddCommand(extractDiffsCmd) } +func getContractAddresses() []string { + LogWithCommand.Info("Getting contract addresses from config file") + contracts := viper.GetStringMap("contract") + var addresses []string + for contractName := range contracts { + address := viper.GetStringMapString("contract." + contractName)["address"] + addresses = append(addresses, address) + } + return addresses +} + func extractDiffs() { // Setup bc and db objects blockChain := getBlockChain() db := utils.LoadPostgres(databaseConfig, blockChain.Node()) + addressesToWatch := getContractAddresses() healthCheckFile := "/tmp/connection" msg := []byte("geth storage fetcher connection established\n") @@ -40,23 +57,15 @@ func extractDiffs() { // initialize fetcher var storageFetcher fetcher.IStorageFetcher + logrus.Debug("fetching storage diffs from geth") switch storageDiffsSource { case "geth": - logrus.Info("Using original geth patch") - logrus.Debug("fetching storage diffs from geth pub sub") - rpcClient, _ := getClients() - stateDiffStreamer := streamer.NewStateDiffStreamer(rpcClient) - payloadChan := make(chan statediff.Payload) - - storageFetcher = fetcher.NewGethRpcStorageFetcher(&stateDiffStreamer, payloadChan, fetcher.OldGethPatch, gethStatusWriter) - case "new-geth": - logrus.Info("Using new geth patch") - logrus.Debug("fetching storage diffs from geth pub sub") - rpcClient, _ := getClients() - stateDiffStreamer := streamer.NewStateDiffStreamer(rpcClient) - payloadChan := make(chan statediff.Payload) - - storageFetcher = fetcher.NewGethRpcStorageFetcher(&stateDiffStreamer, payloadChan, fetcher.NewGethPatch, gethStatusWriter) + logrus.Info("Using new geth patch with filters event system") + _, ethClient := getClients() + filterQuery := createFilterQuery(addressesToWatch) + stateDiffStreamer := streamer.NewEthStateChangeStreamer(ethClient, filterQuery) + payloadChan := make(chan filters.Payload) + storageFetcher = fetcher.NewGethRpcStorageFetcher(&stateDiffStreamer, payloadChan, gethStatusWriter) default: logrus.Debug("fetching storage diffs from csv") tailer := fs.FileTailer{Path: storageDiffsPath} @@ -73,3 +82,18 @@ func extractDiffs() { LogWithCommand.Fatalf("extracting diffs failed: %s", err.Error()) } } + +func createFilterQuery(watchedAddresses []string) ethereum.FilterQuery { + logrus.Infof("Creating a filter query for %d watched addresses", len(watchedAddresses)) + addressesToLog := strings.Join(watchedAddresses[:], ", ") + logrus.Infof("Watched addresses: %s", addressesToLog) + + var addresses []common.Address + for _, addressString := range watchedAddresses { + addresses = append(addresses, common.HexToAddress(addressString)) + } + + return ethereum.FilterQuery{ + Addresses: addresses, + } +} diff --git a/dockerfiles/extract_diffs/Dockerfile b/dockerfiles/extract_diffs/Dockerfile deleted file mode 100644 index c9ea7d2cc..000000000 --- a/dockerfiles/extract_diffs/Dockerfile +++ /dev/null @@ -1,48 +0,0 @@ -FROM golang:alpine as builder - -RUN apk --update --no-cache add g++ git linux-headers - -# Build migration tool -WORKDIR /go -RUN GO111MODULE=auto go get -u -d github.com/pressly/goose/cmd/goose -WORKDIR /go/src/github.com/pressly/goose/cmd/goose -RUN GO111MODULE=auto go build -a -ldflags '-s' -tags='no_mysql no_sqlite' -o goose - -ENV GO111MODULE on - -WORKDIR /vulcanizedb -COPY . . -RUN go build - - -FROM golang:alpine - -WORKDIR /app - -# add certificates for node requests via https -# bash for wait-for-it.sh -RUN apk update \ - && apk upgrade \ - && apk add --no-cache \ - ca-certificates \ - bash \ - && update-ca-certificates 2>/dev/null || true - -# Direct logs to stdout for docker log driver -RUN ln -sf /dev/stdout /app/vulcanizedb.log - -# setup environment -ENV GO111MODULE on - -# add required files -COPY --from=builder /vulcanizedb/vulcanizedb . -COPY --from=builder /vulcanizedb/dockerfiles/extract_diffs/startup_script.sh . -COPY --from=builder /vulcanizedb/db/migrations/* db/migrations/ -COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose -# needed for waiting until postgres is ready before starting from docker-compose -COPY --from=builder /vulcanizedb/dockerfiles/wait-for-it.sh . - -HEALTHCHECK CMD test -f /tmp/connection - -# need to execute with a shell to access env variables -CMD ["./startup_script.sh"] \ No newline at end of file diff --git a/dockerfiles/extract_diffs/startup_script.sh b/dockerfiles/extract_diffs/startup_script.sh deleted file mode 100755 index 87b52adf7..000000000 --- a/dockerfiles/extract_diffs/startup_script.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/sh -# Starts the extractDiffs command - -# Verify required args present -MISSING_VAR_MESSAGE=" is required and no value was given" - -function testDatabaseVariables() { - for a in DATABASE_NAME DATABASE_HOSTNAME DATABASE_PORT DATABASE_USER DATABASE_PASSWORD - do - eval arg="$"$a - test $arg - if [ $? -ne 0 ]; then - echo $a $MISSING_VAR_MESSAGE - exit 1 - fi - done -} - -if test -z "$VDB_PG_CONNECT"; then - # Exits if the variable tests fail - testDatabaseVariables - if [ $? -ne 0 ]; then - exit 1 - fi - - # Construct the connection string for postgres - VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable -fi - -# Run the DB migrations -echo "Connecting with: $VDB_PG_CONNECT" -./goose -dir db/migrations postgres "$VDB_PG_CONNECT" up - -if [ $? -ne 0 ]; then - echo "Could not run migrations. Are the database details correct?" - exit 1 -fi - -# Run extractDiffs -echo "Running extractDiffs..." -./vulcanizedb extractDiffs \ No newline at end of file diff --git a/go.mod b/go.mod index fa9c9b253..17417fa18 100644 --- a/go.mod +++ b/go.mod @@ -17,9 +17,8 @@ require ( github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 // indirect github.com/gorilla/websocket v1.4.1 // indirect - github.com/hashicorp/golang-lru v0.5.3 + github.com/hashicorp/golang-lru v0.5.4 github.com/hpcloud/tail v1.0.0 - github.com/huin/goupnp v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/influxdata/influxdb v1.7.9 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect @@ -44,12 +43,12 @@ require ( github.com/spf13/viper v1.3.2 github.com/status-im/keycard-go v0.0.0-20191119114148-6dd40a46baa0 // indirect github.com/tyler-smith/go-bip39 v1.0.2 // indirect - golang.org/x/net v0.0.0-20200301022130-244492dfa37a + golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0 golang.org/x/sync v0.0.0-20190423024810-112230192c58 google.golang.org/appengine v1.6.5 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 ) -replace github.com/ethereum/go-ethereum => github.com/makerdao/go-ethereum v1.9.11-rc2 +replace github.com/ethereum/go-ethereum => github.com/makerdao/go-ethereum v1.9.15-statechange-filter replace gopkg.in/urfave/cli.v1 => gopkg.in/urfave/cli.v1 v1.20.0 diff --git a/go.sum b/go.sum index bfe995883..709b5d2a9 100644 --- a/go.sum +++ b/go.sum @@ -158,6 +158,8 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/golang-lru v0.0.0-20160813221303-0a025b7e63ad/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk= github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/howeyc/fsnotify v0.9.0 h1:0gtV5JmOKH4A8SsFxG2BczSeXWWPvcMT0euZt5gDAxY= @@ -215,8 +217,12 @@ github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDe github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/makerdao/go-ethereum v1.5.10-0.20200508151415-a332a1649dff h1:oH6AkDyruMAXlE1WELwaAc5iEnRBsQfQALS+gGzDHIM= github.com/makerdao/go-ethereum v1.5.10-0.20200508151415-a332a1649dff/go.mod h1:qeDFnA9rW6VM6KC91PIlcpXMr8V+KY4udYi/RSZtPT4= +github.com/makerdao/go-ethereum v1.5.10-0.20200521202712-d5657e3ff589 h1:a6bFsUnwxMBq7MLW4vZDBj1DnEpWyFQRMGp71t1aJj4= +github.com/makerdao/go-ethereum v1.5.10-0.20200521202712-d5657e3ff589/go.mod h1:oP8FC5+TbICUyftkTWs+8JryntjIJLJvWvApK3z2AYw= github.com/makerdao/go-ethereum v1.9.11-rc2 h1:ZjxmFdkar8P43EkK17ni8y3vXxHHFZjOd12HqNalf8M= github.com/makerdao/go-ethereum v1.9.11-rc2/go.mod h1:7oC0Ni6dosMv5pxMigm6s0hN8g4haJMBnqmmo0D9YfQ= +github.com/makerdao/go-ethereum v1.9.15-statechange-filter h1:jLX9CleT+i/gUn5LLTUoor1NXYLHSTnZM1pOFKrg2IA= +github.com/makerdao/go-ethereum v1.9.15-statechange-filter/go.mod h1:slT8bPPRhXsyNTwHQxrOnjuTZ1sDXRajW11EkJ84QJ0= github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -281,6 +287,7 @@ github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= @@ -316,6 +323,8 @@ github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubrpZHtQLnOf6EwhN2hEMusxZOhcW9H3UQ= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/shirou/gopsutil v2.20.5-0.20200531151128-663af789c085+incompatible h1:+gAR1bMhuoQnZMTWFIvp7ukynULPsteLzG+siZKLtD8= +github.com/shirou/gopsutil v2.20.5-0.20200531151128-663af789c085+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -402,6 +411,8 @@ golang.org/x/net v0.0.0-20190912160710-24e19bdeb0f2 h1:4dVFTC832rPn4pomLSz1vA+ar golang.org/x/net v0.0.0-20190912160710-24e19bdeb0f2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0 h1:Jcxah/M+oLZ/R4/z5RzfPzGbPXnVDPkEDtf2JnuxN+U= +golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -433,6 +444,8 @@ golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9Y golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -469,6 +482,7 @@ gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190213234257-ec84240a7772/go.mod h1:uAJ gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190709231704-1e4459ed25ff h1:uuol9OUzSvZntY1v963NAbVd7A+PHLMz1FlCe3Lorcs= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190709231704-1e4459ed25ff/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200316214253-d7b0ff38cac9/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= +gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200603215123-a4a8cb9d2cbc/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/redis.v4 v4.2.4/go.mod h1:8KREHdypkCEojGKQcjMqAODMICIVwZAONWq8RowTITA= gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= diff --git a/libraries/shared/mocks/streamer.go b/libraries/shared/mocks/streamer.go index 9c7fd53b8..0b69f5e29 100644 --- a/libraries/shared/mocks/streamer.go +++ b/libraries/shared/mocks/streamer.go @@ -1,7 +1,7 @@ package mocks import ( - "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/makerdao/vulcanizedb/pkg/core" "github.com/makerdao/vulcanizedb/pkg/fakes" ) @@ -9,11 +9,11 @@ import ( type MockStoragediffStreamer struct { subscribeError error ClientSubscription *fakes.MockSubscription - PassedPayloadChan chan statediff.Payload - streamPayloads []statediff.Payload + PassedPayloadChan chan filters.Payload + streamPayloads []filters.Payload } -func (streamer *MockStoragediffStreamer) Stream(statediffPayloadChan chan statediff.Payload) (core.Subscription, error) { +func (streamer *MockStoragediffStreamer) Stream(statediffPayloadChan chan filters.Payload) (core.Subscription, error) { streamer.PassedPayloadChan = statediffPayloadChan go func() { @@ -29,6 +29,6 @@ func (streamer *MockStoragediffStreamer) SetSubscribeError(err error) { streamer.subscribeError = err } -func (streamer *MockStoragediffStreamer) SetPayloads(payloads []statediff.Payload) { +func (streamer *MockStoragediffStreamer) SetPayloads(payloads []filters.Payload) { streamer.streamPayloads = payloads } diff --git a/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher.go b/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher.go index 8b09c5b8a..c9cb62661 100644 --- a/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher.go +++ b/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher.go @@ -17,37 +17,34 @@ package fetcher import ( "fmt" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/statediff" "github.com/makerdao/vulcanizedb/libraries/shared/storage/types" "github.com/makerdao/vulcanizedb/libraries/shared/streamer" "github.com/makerdao/vulcanizedb/pkg/fs" "github.com/sirupsen/logrus" ) -type GethPatchVersion int - -const ( - OldGethPatch GethPatchVersion = iota - NewGethPatch -) - type GethRpcStorageFetcher struct { - statediffPayloadChan chan statediff.Payload + statediffPayloadChan chan filters.Payload streamer streamer.Streamer - gethVersion GethPatchVersion statusWriter fs.StatusWriter } -func NewGethRpcStorageFetcher(streamer streamer.Streamer, statediffPayloadChan chan statediff.Payload, gethVersion GethPatchVersion, statusWriter fs.StatusWriter) GethRpcStorageFetcher { +func NewGethRpcStorageFetcher(streamer streamer.Streamer, statediffPayloadChan chan filters.Payload, statusWriter fs.StatusWriter) GethRpcStorageFetcher { return GethRpcStorageFetcher{ statediffPayloadChan: statediffPayloadChan, streamer: streamer, - gethVersion: gethVersion, statusWriter: statusWriter, } } +var ( + processingDiffsLogString = "processing %d storage diffs for account %s" + addingDiffsLogString = "adding storage diff to out channel. keccak of address: %v, block height: %v, storage key: %v, storage value: %v" +) + func (fetcher GethRpcStorageFetcher) FetchStorageDiffs(out chan<- types.RawDiff, errs chan<- error) { ethStatediffPayloadChan := fetcher.statediffPayloadChan clientSubscription, clientSubErr := fetcher.streamer.Stream(ethStatediffPayloadChan) @@ -67,45 +64,32 @@ func (fetcher GethRpcStorageFetcher) FetchStorageDiffs(out chan<- types.RawDiff, case err := <-clientSubscription.Err(): logrus.Errorf("error with client subscription: %s", err.Error()) errs <- err - case diff := <-ethStatediffPayloadChan: - logrus.Trace("received a statediff") - stateDiff := new(statediff.StateDiff) - decodeErr := rlp.DecodeBytes(diff.StateDiffRlp, stateDiff) - logrus.Tracef("received a statediff from block: %v", stateDiff.BlockNumber) - if decodeErr != nil { - logrus.Warn("Error decoding state diff into RLP: ", decodeErr) - errs <- decodeErr - } - - accounts := getAccountsFromDiff(*stateDiff) - logrus.Trace(fmt.Sprintf("iterating through %d accounts on stateDiff for block %d", len(accounts), stateDiff.BlockNumber)) - for _, account := range accounts { - logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account", len(account.Storage))) - for _, accountStorage := range account.Storage { - diff, formatErr := fetcher.formatDiff(account, stateDiff, accountStorage) - logrus.Tracef("adding storage diff to out channel. keccak of address: %v, block height: %v, storage key: %v, storage value: %v", - diff.HashedAddress.Hex(), diff.BlockHeight, diff.StorageKey.Hex(), diff.StorageValue.Hex()) - if formatErr != nil { - errs <- formatErr - } - - out <- diff - } - } + case diffPayload := <-ethStatediffPayloadChan: + logrus.Trace("received a statediff payload") + fetcher.handleDiffPayload(diffPayload, out, errs) } - } } -func (fetcher GethRpcStorageFetcher) formatDiff(account statediff.AccountDiff, stateDiff *statediff.StateDiff, storage statediff.StorageDiff) (types.RawDiff, error) { - if fetcher.gethVersion == OldGethPatch { - return types.FromOldGethStateDiff(account, stateDiff, storage) - } else { - return types.FromNewGethStateDiff(account, stateDiff, storage) +func (fetcher GethRpcStorageFetcher) handleDiffPayload(payload filters.Payload, out chan<- types.RawDiff, errs chan<- error) { + var stateDiff filters.StateDiff + decodeErr := rlp.DecodeBytes(payload.StateDiffRlp, &stateDiff) + if decodeErr != nil { + errs <- fmt.Errorf("error decoding storage diff from geth payload: %w", decodeErr) + return } -} -func getAccountsFromDiff(stateDiff statediff.StateDiff) []statediff.AccountDiff { - accounts := append(stateDiff.CreatedAccounts, stateDiff.UpdatedAccounts...) - return append(accounts, stateDiff.DeletedAccounts...) + for _, account := range stateDiff.UpdatedAccounts { + logrus.Infof(processingDiffsLogString, len(account.Storage), common.Bytes2Hex(account.Key)) + for _, accountStorage := range account.Storage { + rawDiff, formatErr := types.FromGethStateDiff(account, &stateDiff, accountStorage) + if formatErr != nil { + errs <- formatErr + return + } + + logrus.Tracef(addingDiffsLogString, rawDiff.HashedAddress.Hex(), rawDiff.BlockHeight, rawDiff.StorageKey.Hex(), rawDiff.StorageValue.Hex()) + out <- rawDiff + } + } } diff --git a/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher_test.go b/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher_test.go index 489cab2a1..2eb049084 100644 --- a/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher_test.go +++ b/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher_test.go @@ -15,10 +15,13 @@ package fetcher_test import ( + "fmt" + "io" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/statediff" "github.com/makerdao/vulcanizedb/libraries/shared/mocks" "github.com/makerdao/vulcanizedb/libraries/shared/storage/fetcher" "github.com/makerdao/vulcanizedb/libraries/shared/storage/types" @@ -31,156 +34,27 @@ import ( var _ = Describe("Geth RPC Storage Fetcher", func() { var ( streamer *mocks.MockStoragediffStreamer - statediffPayloadChan chan statediff.Payload + statediffPayloadChan chan filters.Payload statediffFetcher fetcher.GethRpcStorageFetcher storagediffChan chan types.RawDiff subscription *fakes.MockSubscription errorChan chan error statusWriter fakes.MockStatusWriter + stateDiffPayloads []filters.Payload + badStateDiffPayloads = []filters.Payload{{}} //This empty payload is "bad" because it does not contain the required StateDiffRlp ) - Describe("StorageFetcher for the Old Geth Patch", func() { - BeforeEach(func() { - subscription = &fakes.MockSubscription{Errs: make(chan error)} - streamer = &mocks.MockStoragediffStreamer{ClientSubscription: subscription} - statediffPayloadChan = make(chan statediff.Payload, 1) - statusWriter = fakes.MockStatusWriter{} - statediffFetcher = fetcher.NewGethRpcStorageFetcher(streamer, statediffPayloadChan, fetcher.OldGethPatch, &statusWriter) - storagediffChan = make(chan types.RawDiff) - errorChan = make(chan error) - }) - - It("adds errors to errors channel if the streamer fails to subscribe", func(done Done) { - streamer.SetSubscribeError(fakes.FakeError) - - go func() { - failedSub := func() { - statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan) - } - Expect(failedSub).To(Panic()) - }() - - Expect(<-errorChan).To(MatchError(fakes.FakeError)) - close(done) - }) - - It("streams StatediffPayloads from a Geth RPC subscription", func(done Done) { - streamer.SetPayloads([]statediff.Payload{test_data.MockStatediffPayload}) - - go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan) - - streamedPayload := <-statediffPayloadChan - Expect(streamedPayload).To(Equal(test_data.MockStatediffPayload)) - Expect(streamer.PassedPayloadChan).To(Equal(statediffPayloadChan)) - close(done) - }) - - Describe("when subscription established", func() { - It("creates file for health check when connection established", func(done Done) { - go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan) - - Eventually(func() bool { - return statusWriter.WriteCalled - }).Should(BeTrue()) - close(done) - }) - - It("adds error to errors channel if the subscription fails", func(done Done) { - go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan) - - subscription.Errs <- fakes.FakeError - - Expect(<-errorChan).To(MatchError(fakes.FakeError)) - close(done) - }) - - It("adds errors to error channel if decoding the state diff RLP fails", func(done Done) { - badStatediffPayload := statediff.Payload{} - streamer.SetPayloads([]statediff.Payload{badStatediffPayload}) - - go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan) - - Expect(<-errorChan).To(MatchError("EOF")) - - close(done) - }) - - It("adds parsed statediff payloads to the out channel for the old geth patch", func(done Done) { - streamer.SetPayloads([]statediff.Payload{test_data.MockStatediffPayload}) - - go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan) - - height := test_data.BlockNumber - intHeight := int(height.Int64()) - createdExpectedStorageDiff := types.RawDiff{ - HashedAddress: common.BytesToHash(test_data.ContractLeafKey[:]), - BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), - BlockHeight: intHeight, - StorageKey: common.BytesToHash(test_data.StorageKey), - StorageValue: common.BytesToHash(test_data.SmallStorageValue), - } - updatedExpectedStorageDiff := types.RawDiff{ - HashedAddress: common.BytesToHash(test_data.AnotherContractLeafKey[:]), - BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), - BlockHeight: intHeight, - StorageKey: common.BytesToHash(test_data.StorageKey), - StorageValue: common.BytesToHash(test_data.LargeStorageValue), - } - deletedExpectedStorageDiff := types.RawDiff{ - HashedAddress: common.BytesToHash(test_data.AnotherContractLeafKey[:]), - BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), - BlockHeight: intHeight, - StorageKey: common.BytesToHash(test_data.StorageKey), - StorageValue: common.BytesToHash(test_data.SmallStorageValue), - } - - createdStateDiff := <-storagediffChan - updatedStateDiff := <-storagediffChan - deletedStateDiff := <-storagediffChan - - Expect(createdStateDiff).To(Equal(createdExpectedStorageDiff)) - Expect(updatedStateDiff).To(Equal(updatedExpectedStorageDiff)) - Expect(deletedStateDiff).To(Equal(deletedExpectedStorageDiff)) - - close(done) - }) - - It("adds errors to error channel if formatting the diff as a StateDiff object fails", func(done Done) { - accountDiffs := test_data.CreatedAccountDiffs - accountDiffs[0].Storage = []statediff.StorageDiff{test_data.StorageWithBadValue} - - stateDiff := statediff.StateDiff{ - BlockNumber: test_data.BlockNumber, - BlockHash: common.HexToHash(test_data.BlockHash), - CreatedAccounts: accountDiffs, - } - - stateDiffRlp, err := rlp.EncodeToBytes(stateDiff) - Expect(err).NotTo(HaveOccurred()) - - badStatediffPayload := statediff.Payload{ - StateDiffRlp: stateDiffRlp, - } - streamer.SetPayloads([]statediff.Payload{badStatediffPayload}) - - go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan) - - Expect(<-errorChan).To(MatchError("rlp: input contains more than one value")) - - close(done) - }) - }) - }) - Describe("StorageFetcher for the New Geth Patch", func() { + // This tests fetching diff payloads from the updated simplified geth patch: https://github.com/makerdao/go-ethereum/tree/allow-state-diff-subscription + // - diffs are formatted with the FromGethStateDiff method BeforeEach(func() { subscription = &fakes.MockSubscription{Errs: make(chan error)} streamer = &mocks.MockStoragediffStreamer{ClientSubscription: subscription} - statediffPayloadChan = make(chan statediff.Payload, 1) - statusWriter = fakes.MockStatusWriter{} - statediffFetcher = fetcher.NewGethRpcStorageFetcher(streamer, statediffPayloadChan, fetcher.NewGethPatch, &statusWriter) + statediffPayloadChan = make(chan filters.Payload, 1) + statediffFetcher = fetcher.NewGethRpcStorageFetcher(streamer, statediffPayloadChan, &statusWriter) storagediffChan = make(chan types.RawDiff) errorChan = make(chan error) + stateDiffPayloads = []filters.Payload{test_data.MockStatediffPayload} }) It("adds errors to errors channel if the streamer fails to subscribe", func(done Done) { @@ -198,7 +72,7 @@ var _ = Describe("Geth RPC Storage Fetcher", func() { }) It("streams StatediffPayloads from a Geth RPC subscription", func(done Done) { - streamer.SetPayloads([]statediff.Payload{test_data.MockStatediffPayload}) + streamer.SetPayloads(stateDiffPayloads) go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan) @@ -228,23 +102,23 @@ var _ = Describe("Geth RPC Storage Fetcher", func() { }) It("adds errors to error channel if decoding the state diff RLP fails", func(done Done) { - badStatediffPayload := statediff.Payload{} - streamer.SetPayloads([]statediff.Payload{badStatediffPayload}) + streamer.SetPayloads(badStateDiffPayloads) go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan) - Expect(<-errorChan).To(MatchError("EOF")) - + expectedErr := fmt.Errorf("error decoding storage diff from geth payload: %w", io.EOF) + Expect(<-errorChan).To(MatchError(expectedErr)) close(done) }) - It("adds parsed statediff payloads to the out channel for the new geth patch", func(done Done) { - streamer.SetPayloads([]statediff.Payload{test_data.MockStatediffPayload}) + It("adds parsed statediff payloads to the out channel", func(done Done) { + streamer.SetPayloads(stateDiffPayloads) go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan) height := test_data.BlockNumber intHeight := int(height.Int64()) + expectedDiff1 := types.RawDiff{ HashedAddress: crypto.Keccak256Hash(test_data.ContractLeafKey[:]), BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), @@ -279,26 +153,16 @@ var _ = Describe("Geth RPC Storage Fetcher", func() { }) It("adds errors to error channel if formatting the diff as a StateDiff object fails", func(done Done) { - accountDiffs := test_data.CreatedAccountDiffs - accountDiffs[0].Storage = []statediff.StorageDiff{test_data.StorageWithBadValue} - - stateDiff := statediff.StateDiff{ - BlockNumber: test_data.BlockNumber, - BlockHash: common.HexToHash(test_data.BlockHash), - CreatedAccounts: accountDiffs, - } - + stateDiff := test_data.StateDiffWithBadStorageValue stateDiffRlp, err := rlp.EncodeToBytes(stateDiff) Expect(err).NotTo(HaveOccurred()) + payloadToReturn := filters.Payload{StateDiffRlp: stateDiffRlp} - badStatediffPayload := statediff.Payload{ - StateDiffRlp: stateDiffRlp, - } - streamer.SetPayloads([]statediff.Payload{badStatediffPayload}) + streamer.SetPayloads([]filters.Payload{payloadToReturn}) go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan) - Expect(<-errorChan).To(MatchError("rlp: input contains more than one value")) + Expect(<-errorChan).To(MatchError(rlp.ErrMoreThanOneValue)) close(done) }) diff --git a/libraries/shared/storage/types/diff.go b/libraries/shared/storage/types/diff.go index bf9d1b0e8..92f00fffa 100644 --- a/libraries/shared/storage/types/diff.go +++ b/libraries/shared/storage/types/diff.go @@ -21,8 +21,8 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/statediff" ) const ExpectedRowLength = 5 @@ -60,23 +60,7 @@ func FromParityCsvRow(csvRow []string) (RawDiff, error) { }, nil } -func FromOldGethStateDiff(account statediff.AccountDiff, stateDiff *statediff.StateDiff, storage statediff.StorageDiff) (RawDiff, error) { - var decodedRLPStorageValue []byte - err := rlp.DecodeBytes(storage.Value, &decodedRLPStorageValue) - if err != nil { - return RawDiff{}, err - } - - return RawDiff{ - HashedAddress: common.BytesToHash(account.Key), - BlockHash: stateDiff.BlockHash, - BlockHeight: int(stateDiff.BlockNumber.Int64()), - StorageKey: common.BytesToHash(storage.Key), - StorageValue: common.BytesToHash(decodedRLPStorageValue), - }, nil -} - -func FromNewGethStateDiff(account statediff.AccountDiff, stateDiff *statediff.StateDiff, storage statediff.StorageDiff) (RawDiff, error) { +func FromGethStateDiff(account filters.AccountDiff, stateDiff *filters.StateDiff, storage filters.StorageDiff) (RawDiff, error) { var decodedRLPStorageValue []byte err := rlp.DecodeBytes(storage.Value, &decodedRLPStorageValue) if err != nil { @@ -92,13 +76,6 @@ func FromNewGethStateDiff(account statediff.AccountDiff, stateDiff *statediff.St }, nil } -func ToPersistedDiff(raw RawDiff, id int64) PersistedDiff { - return PersistedDiff{ - RawDiff: raw, - ID: id, - } -} - func HexToKeccak256Hash(hex string) common.Hash { return crypto.Keccak256Hash(common.FromHex(hex)) } diff --git a/libraries/shared/storage/types/diff_test.go b/libraries/shared/storage/types/diff_test.go index d8fe99a59..194500643 100644 --- a/libraries/shared/storage/types/diff_test.go +++ b/libraries/shared/storage/types/diff_test.go @@ -22,8 +22,8 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/statediff" "github.com/makerdao/vulcanizedb/libraries/shared/storage/types" "github.com/makerdao/vulcanizedb/libraries/shared/test_data" "github.com/makerdao/vulcanizedb/pkg/fakes" @@ -66,10 +66,10 @@ var _ = Describe("Storage row parsing", func() { }) }) - Describe("FromOldGethStateDiff", func() { + Describe("FromGethStateDiff", func() { var ( - accountDiff = statediff.AccountDiff{Key: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}} - stateDiff = &statediff.StateDiff{ + accountDiff = filters.AccountDiff{Key: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}} + stateDiff = &filters.StateDiff{ BlockNumber: big.NewInt(rand.Int63()), BlockHash: fakes.FakeHash, } @@ -80,67 +80,12 @@ var _ = Describe("Storage row parsing", func() { storageValueRlp, encodeErr := rlp.EncodeToBytes(storageValueBytes) Expect(encodeErr).NotTo(HaveOccurred()) - storageDiff := statediff.StorageDiff{ + storageDiff := filters.StorageDiff{ Key: []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1}, Value: storageValueRlp, } - result, err := types.FromOldGethStateDiff(accountDiff, stateDiff, storageDiff) - Expect(err).NotTo(HaveOccurred()) - - expectedAddress := common.BytesToHash(accountDiff.Key) - Expect(result.HashedAddress).To(Equal(expectedAddress)) - Expect(result.BlockHash).To(Equal(fakes.FakeHash)) - expectedBlockHeight := int(stateDiff.BlockNumber.Int64()) - Expect(result.BlockHeight).To(Equal(expectedBlockHeight)) - expectedStorageKey := common.BytesToHash(storageDiff.Key) - Expect(result.StorageKey).To(Equal(expectedStorageKey)) - expectedStorageValue := common.BytesToHash(storageValueBytes) - Expect(result.StorageValue).To(Equal(expectedStorageValue)) - }) - - It("handles decoding large storage values from their RLP", func() { - storageValueBytes := []byte{1, 2, 3, 4, 5, 0, 9, 8, 7, 6} - storageValueRlp, encodeErr := rlp.EncodeToBytes(storageValueBytes) - Expect(encodeErr).NotTo(HaveOccurred()) - - storageDiff := statediff.StorageDiff{ - Key: []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1}, - Value: storageValueRlp, - } - - result, err := types.FromOldGethStateDiff(accountDiff, stateDiff, storageDiff) - Expect(err).NotTo(HaveOccurred()) - Expect(result.StorageValue).To(Equal(common.BytesToHash(storageValueBytes))) - }) - - It("returns an err if decoding the storage value Rlp fails", func() { - _, err := types.FromOldGethStateDiff(accountDiff, stateDiff, test_data.StorageWithBadValue) - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError("rlp: input contains more than one value")) - }) - }) - - Describe("FromNewGethStateDiff", func() { - var ( - accountDiff = statediff.AccountDiff{Key: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}} - stateDiff = &statediff.StateDiff{ - BlockNumber: big.NewInt(rand.Int63()), - BlockHash: fakes.FakeHash, - } - ) - - It("adds relevant fields to diff", func() { - storageValueBytes := []byte{3} - storageValueRlp, encodeErr := rlp.EncodeToBytes(storageValueBytes) - Expect(encodeErr).NotTo(HaveOccurred()) - - storageDiff := statediff.StorageDiff{ - Key: []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1}, - Value: storageValueRlp, - } - - result, err := types.FromNewGethStateDiff(accountDiff, stateDiff, storageDiff) + result, err := types.FromGethStateDiff(accountDiff, stateDiff, storageDiff) Expect(err).NotTo(HaveOccurred()) expectedHashedAddress := crypto.Keccak256Hash(accountDiff.Key) @@ -159,18 +104,18 @@ var _ = Describe("Storage row parsing", func() { storageValueRlp, encodeErr := rlp.EncodeToBytes(storageValueBytes) Expect(encodeErr).NotTo(HaveOccurred()) - storageDiff := statediff.StorageDiff{ + storageDiff := filters.StorageDiff{ Key: []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1}, Value: storageValueRlp, } - result, err := types.FromNewGethStateDiff(accountDiff, stateDiff, storageDiff) + result, err := types.FromGethStateDiff(accountDiff, stateDiff, storageDiff) Expect(err).NotTo(HaveOccurred()) Expect(result.StorageValue).To(Equal(common.BytesToHash(storageValueBytes))) }) It("returns an err if decoding the storage value Rlp fails", func() { - _, err := types.FromNewGethStateDiff(accountDiff, stateDiff, test_data.StorageWithBadValue) + _, err := types.FromGethStateDiff(accountDiff, stateDiff, test_data.StorageWithBadValue) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError("rlp: input contains more than one value")) }) diff --git a/libraries/shared/streamer/statediff_streamer.go b/libraries/shared/streamer/state_change_streamer.go similarity index 53% rename from libraries/shared/streamer/statediff_streamer.go rename to libraries/shared/streamer/state_change_streamer.go index 130cea22f..7089794f2 100644 --- a/libraries/shared/streamer/statediff_streamer.go +++ b/libraries/shared/streamer/state_change_streamer.go @@ -15,26 +15,31 @@ package streamer import ( - "github.com/ethereum/go-ethereum/statediff" + "context" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/makerdao/vulcanizedb/pkg/core" "github.com/sirupsen/logrus" ) type Streamer interface { - Stream(chan statediff.Payload) (core.Subscription, error) + Stream(chan filters.Payload) (core.Subscription, error) } -type StateDiffStreamer struct { - client core.RpcClient +type EthStateChangeStreamer struct { + ethClient core.EthClient + filterQuery ethereum.FilterQuery } -func (streamer *StateDiffStreamer) Stream(payloadChan chan statediff.Payload) (core.Subscription, error) { - logrus.Info("streaming diffs from geth") - return streamer.client.Subscribe("statediff", payloadChan, "stream") +func NewEthStateChangeStreamer(ethClient core.EthClient, filterQuery ethereum.FilterQuery) EthStateChangeStreamer { + return EthStateChangeStreamer{ + ethClient: ethClient, + filterQuery: filterQuery, + } } -func NewStateDiffStreamer(client core.RpcClient) StateDiffStreamer { - return StateDiffStreamer{ - client: client, - } +func (streamer *EthStateChangeStreamer) Stream(payloadChan chan filters.Payload) (core.Subscription, error) { + logrus.Info("streaming diffs from geth") + return streamer.ethClient.SubscribeNewStateChanges(context.Background(), streamer.filterQuery, payloadChan) } diff --git a/libraries/shared/streamer/statediff_streamer_test.go b/libraries/shared/streamer/state_change_streamer_test.go similarity index 60% rename from libraries/shared/streamer/statediff_streamer_test.go rename to libraries/shared/streamer/state_change_streamer_test.go index 3a6c8e6b3..468791962 100644 --- a/libraries/shared/streamer/statediff_streamer_test.go +++ b/libraries/shared/streamer/state_change_streamer_test.go @@ -15,21 +15,26 @@ package streamer_test import ( - "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/makerdao/vulcanizedb/libraries/shared/streamer" "github.com/makerdao/vulcanizedb/pkg/fakes" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) -var _ = Describe("StateDiff Streamer", func() { - It("subscribes to the geth statediff service", func() { - client := &fakes.MockRpcClient{} - streamer := streamer.NewStateDiffStreamer(client) - payloadChan := make(chan statediff.Payload) +var _ = Describe("State Change Streamer", func() { + It("subscribes to the geth state change subscription", func() { + ethClient := &fakes.MockEthClient{} + filterQuery := ethereum.FilterQuery{ + Addresses: []common.Address{fakes.FakeAddress}, + } + streamer := streamer.NewEthStateChangeStreamer(ethClient, filterQuery) + payloadChan := make(chan filters.Payload) _, err := streamer.Stream(payloadChan) Expect(err).NotTo(HaveOccurred()) - client.AssertSubscribeCalledWith("statediff", payloadChan, []interface{}{"stream"}) + ethClient.AssertSubscribeNewStateChangesCalledWith(filterQuery, payloadChan) }) }) diff --git a/libraries/shared/test_data/statediff.go b/libraries/shared/test_data/statediff.go index e51a51077..256c82a8c 100644 --- a/libraries/shared/test_data/statediff.go +++ b/libraries/shared/test_data/statediff.go @@ -20,10 +20,9 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/statediff" ) var ( @@ -37,22 +36,18 @@ var ( StorageKey = common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes() SmallStorageValue = common.Hex2Bytes("03") SmallStorageValueRlp, _ = rlp.EncodeToBytes(SmallStorageValue) - storageWithSmallValue = []statediff.StorageDiff{{ + storageWithSmallValue = []filters.StorageDiff{{ Key: StorageKey, Value: SmallStorageValueRlp, - Path: StoragePath, - Proof: [][]byte{}, }} LargeStorageValue = common.Hex2Bytes("00191b53778c567b14b50ba0000") LargeStorageValueRlp, rlpErr = rlp.EncodeToBytes(LargeStorageValue) - storageWithLargeValue = []statediff.StorageDiff{{ + storageWithLargeValue = []filters.StorageDiff{{ Key: StorageKey, Value: LargeStorageValueRlp, - Path: StoragePath, - Proof: [][]byte{}, }} - EmptyStorage = make([]statediff.StorageDiff, 0) - StorageWithBadValue = statediff.StorageDiff{ + EmptyStorage = make([]filters.StorageDiff, 0) + StorageWithBadValue = filters.StorageDiff{ Key: StorageKey, Value: []byte{0, 1, 2}, // this storage value will fail to be decoded as an RLP with the following error message: @@ -69,56 +64,49 @@ var ( Root: ContractRoot, CodeHash: CodeHash, } - valueBytes, _ = rlp.EncodeToBytes(testAccount) - CreatedAccountDiffs = []statediff.AccountDiff{ - { - Key: ContractLeafKey.Bytes(), - Value: valueBytes, - Storage: storageWithSmallValue, - }, + AccountValueBytes, _ = rlp.EncodeToBytes(testAccount) + testAccountDiff1 = filters.AccountDiff{ + Key: ContractLeafKey.Bytes(), + Value: AccountValueBytes, + Storage: storageWithSmallValue, } - - UpdatedAccountDiffs = []statediff.AccountDiff{{ + testAccountDiff2 = filters.AccountDiff{ Key: AnotherContractLeafKey.Bytes(), - Value: valueBytes, + Value: AccountValueBytes, Storage: storageWithLargeValue, - }} - - DeletedAccountDiffs = []statediff.AccountDiff{{ + } + testAccountDiff3 = filters.AccountDiff{ Key: AnotherContractLeafKey.Bytes(), - Value: valueBytes, + Value: AccountValueBytes, Storage: storageWithSmallValue, - }} + } + UpdatedAccountDiffs = []filters.AccountDiff{testAccountDiff1, testAccountDiff2, testAccountDiff3} - MockStateDiff = statediff.StateDiff{ + MockStateDiff = filters.StateDiff{ BlockNumber: BlockNumber, BlockHash: common.HexToHash(BlockHash), - CreatedAccounts: CreatedAccountDiffs, - DeletedAccounts: DeletedAccountDiffs, UpdatedAccounts: UpdatedAccountDiffs, } MockStateDiffBytes, _ = rlp.EncodeToBytes(MockStateDiff) - mockTransaction1 = types.NewTransaction(0, common.HexToAddress("0x0"), big.NewInt(1000), 50, big.NewInt(100), nil) - mockTransaction2 = types.NewTransaction(1, common.HexToAddress("0x1"), big.NewInt(2000), 100, big.NewInt(200), nil) - MockTransactions = types.Transactions{mockTransaction1, mockTransaction2} - - mockReceipt1 = types.NewReceipt(common.HexToHash("0x0").Bytes(), false, 50) - mockReceipt2 = types.NewReceipt(common.HexToHash("0x1").Bytes(), false, 100) - MockReceipts = types.Receipts{mockReceipt1, mockReceipt2} - - MockHeader = types.Header{ - Time: 0, - Number: BlockNumber, - Root: common.HexToHash("0x0"), - TxHash: common.HexToHash("0x0"), - ReceiptHash: common.HexToHash("0x0"), + MockStatediffPayload = filters.Payload{ + StateDiffRlp: MockStateDiffBytes, } - MockBlock = types.NewBlock(&MockHeader, MockTransactions, nil, MockReceipts) - MockBlockRlp, _ = rlp.EncodeToBytes(MockBlock) - MockStatediffPayload = statediff.Payload{ - BlockRlp: MockBlockRlp, - StateDiffRlp: MockStateDiffBytes, + storageWithBadValue = filters.StorageDiff{ + Key: StorageKey, + Value: []byte{0, 1, 2}, + // this storage value will fail to be decoded as an RLP with the following error message: + // "rlp: input contains more than one value" + } + testAccountDiffWithBadStorageValue = filters.AccountDiff{ + Key: ContractLeafKey.Bytes(), + Value: AccountValueBytes, + Storage: []filters.StorageDiff{storageWithBadValue}, + } + StateDiffWithBadStorageValue = filters.StateDiff{ + BlockNumber: BlockNumber, + BlockHash: common.HexToHash(BlockHash), + UpdatedAccounts: []filters.AccountDiff{testAccountDiffWithBadStorageValue}, } ) diff --git a/pkg/core/eth_client.go b/pkg/core/eth_client.go index 183d224bd..deb3fceb1 100644 --- a/pkg/core/eth_client.go +++ b/pkg/core/eth_client.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/filters" ) type EthClient interface { @@ -30,6 +31,7 @@ type EthClient interface { CallContract(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) FilterLogs(ctx context.Context, q ethereum.FilterQuery) ([]types.Log, error) HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) + SubscribeNewStateChanges(ctx context.Context, q ethereum.FilterQuery, ch chan<- filters.Payload) (ethereum.Subscription, error) TransactionSender(ctx context.Context, tx *types.Transaction, block common.Hash, index uint) (common.Address, error) TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) } diff --git a/pkg/eth/client/eth_client.go b/pkg/eth/client/eth_client.go index 0b335ee6f..b708d3e31 100644 --- a/pkg/eth/client/eth_client.go +++ b/pkg/eth/client/eth_client.go @@ -18,11 +18,13 @@ package client import ( "context" + "math/big" + "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/ethclient" - "math/big" ) type EthClient struct { @@ -49,6 +51,10 @@ func (client EthClient) HeaderByNumber(ctx context.Context, number *big.Int) (*t return client.client.HeaderByNumber(ctx, number) } +func (client EthClient) SubscribeNewStateChanges(ctx context.Context, q ethereum.FilterQuery, ch chan<- filters.Payload) (ethereum.Subscription, error) { + return client.client.SubscribeNewStateChanges(ctx, q, ch) +} + func (client EthClient) TransactionSender(ctx context.Context, tx *types.Transaction, block common.Hash, index uint) (common.Address, error) { return client.client.TransactionSender(ctx, tx, block, index) } diff --git a/pkg/fakes/mock_eth_client.go b/pkg/fakes/mock_eth_client.go index 59d09cb5f..4edc4b958 100644 --- a/pkg/fakes/mock_eth_client.go +++ b/pkg/fakes/mock_eth_client.go @@ -23,38 +23,43 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/filters" + "github.com/ethereum/go-ethereum/rpc" "github.com/makerdao/vulcanizedb/pkg/core" . "github.com/onsi/gomega" ) type MockEthClient struct { - callContractErr error - callContractPassedContext context.Context - callContractPassedMsg ethereum.CallMsg - callContractPassedNumber *big.Int - callContractReturnBytes []byte - blockByNumberErr error - blockByNumberPassedContext context.Context - blockByNumberPassedNumber *big.Int - blockByNumberReturnBlock *types.Block - headerByNumberErr error - headerByNumberPassedContext context.Context - headerByNumberPassedNumber *big.Int - headerByNumberReturnHeader *types.Header - headerByNumbersReturnHeader []*types.Header - headerByNumbersPassedNumber []*big.Int - filterLogsErr error - filterLogsPassedContext context.Context - filterLogsPassedQuery ethereum.FilterQuery - filterLogsReturnLogs []types.Log - transactionReceipts map[string]*types.Receipt - err error - passedBatch []core.BatchElem - passedMethod string - transactionSenderErr error - transactionReceiptErr error - passedAddress common.Address - passedBlockNumber *big.Int + callContractErr error + callContractPassedContext context.Context + callContractPassedMsg ethereum.CallMsg + callContractPassedNumber *big.Int + callContractReturnBytes []byte + blockByNumberErr error + blockByNumberPassedContext context.Context + blockByNumberPassedNumber *big.Int + blockByNumberReturnBlock *types.Block + headerByNumberErr error + headerByNumberPassedContext context.Context + headerByNumberPassedNumber *big.Int + headerByNumberReturnHeader *types.Header + headerByNumbersReturnHeader []*types.Header + headerByNumbersPassedNumber []*big.Int + filterLogsErr error + filterLogsPassedContext context.Context + filterLogsPassedQuery ethereum.FilterQuery + filterLogsReturnLogs []types.Log + transactionReceipts map[string]*types.Receipt + err error + passedBatch []core.BatchElem + passedMethod string + transactionSenderErr error + transactionReceiptErr error + passedAddress common.Address + passedBlockNumber *big.Int + subscribeStateChangeErr error + passedStateChangeFilterQuery ethereum.FilterQuery + passedStateChangePayloadCh chan<- filters.Payload } func NewMockEthClient() *MockEthClient { @@ -148,6 +153,13 @@ func (client *MockEthClient) FilterLogs(ctx context.Context, q ethereum.FilterQu return client.filterLogsReturnLogs, client.filterLogsErr } +func (client *MockEthClient) SubscribeNewStateChanges(ctx context.Context, q ethereum.FilterQuery, ch chan<- filters.Payload) (ethereum.Subscription, error) { + client.passedStateChangeFilterQuery = q + client.passedStateChangePayloadCh = ch + subscription := rpc.ClientSubscription{} + return &subscription, nil +} + func (client *MockEthClient) TransactionSender(ctx context.Context, tx *types.Transaction, block common.Hash, index uint) (common.Address, error) { return common.HexToAddress("0x123"), client.transactionSenderErr } @@ -187,3 +199,8 @@ func (client *MockEthClient) AssertFilterLogsCalledWith(ctx context.Context, q e func (client *MockEthClient) AssertBatchCalledWith(method string) { Expect(client.passedMethod).To(Equal(method)) } + +func (client *MockEthClient) AssertSubscribeNewStateChangesCalledWith(filterQuery ethereum.FilterQuery, payloadCh chan<- filters.Payload) { + Expect(client.passedStateChangeFilterQuery).To(Equal(filterQuery)) + Expect(client.passedStateChangePayloadCh).To(Equal(payloadCh)) +} diff --git a/pkg/fakes/mock_rpc_client.go b/pkg/fakes/mock_rpc_client.go index f18b4b886..dbe5f710c 100644 --- a/pkg/fakes/mock_rpc_client.go +++ b/pkg/fakes/mock_rpc_client.go @@ -23,9 +23,9 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/rpc" - "github.com/ethereum/go-ethereum/statediff" "github.com/makerdao/vulcanizedb/pkg/core" "github.com/makerdao/vulcanizedb/pkg/eth/client" . "github.com/onsi/gomega" @@ -45,7 +45,7 @@ type MockRpcClient struct { passedResult interface{} passedBatch []core.BatchElem passedNamespace string - passedPayloadChan chan statediff.Payload + passedPayloadChan chan filters.Payload passedSubscribeArgs []interface{} lengthOfBatch int returnPOAHeader core.POAHeader @@ -61,7 +61,7 @@ func NewMockRpcClient() *MockRpcClient { func (c *MockRpcClient) Subscribe(namespace string, payloadChan interface{}, args ...interface{}) (core.Subscription, error) { c.passedNamespace = namespace - passedPayloadChan, ok := payloadChan.(chan statediff.Payload) + passedPayloadChan, ok := payloadChan.(chan filters.Payload) if !ok { return nil, errors.New("passed in channel is not of the correct type") } @@ -75,7 +75,7 @@ func (c *MockRpcClient) Subscribe(namespace string, payloadChan interface{}, arg return client.Subscription{RpcSubscription: &subscription}, nil } -func (c *MockRpcClient) AssertSubscribeCalledWith(namespace string, payloadChan chan statediff.Payload, args []interface{}) { +func (c *MockRpcClient) AssertSubscribeCalledWith(namespace string, payloadChan chan filters.Payload, args []interface{}) { Expect(c.passedNamespace).To(Equal(namespace)) Expect(c.passedPayloadChan).To(Equal(payloadChan)) Expect(c.passedSubscribeArgs).To(Equal(args)) From abb5e964b16cbbf37fc05ca9efea180dea4f0777 Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Tue, 11 Aug 2020 11:28:26 -0500 Subject: [PATCH 08/66] Add eth_node_id to storage_diffs table It's a foreign key on the eth_nodes table, so make sure to give it an index. --- .../00005_create_storage_diffs_table.sql | 5 +- ...reate_create_back_filled_diff_function.sql | 10 ++-- db/schema.sql | 28 ++++++++--- libraries/shared/storage/diff_repository.go | 8 ++-- .../shared/storage/diff_repository_test.go | 47 ++++++++++--------- libraries/shared/storage/types/diff.go | 1 + 6 files changed, 62 insertions(+), 37 deletions(-) diff --git a/db/migrations/00005_create_storage_diffs_table.sql b/db/migrations/00005_create_storage_diffs_table.sql index 2e91e4c86..ab2cb7f50 100644 --- a/db/migrations/00005_create_storage_diffs_table.sql +++ b/db/migrations/00005_create_storage_diffs_table.sql @@ -7,6 +7,7 @@ CREATE TABLE public.storage_diff hashed_address BYTEA, storage_key BYTEA, storage_value BYTEA, + eth_node_id INTEGER NOT NULL REFERENCES public.eth_nodes (id) ON DELETE CASCADE, checked BOOLEAN NOT NULL DEFAULT FALSE, from_backfill BOOLEAN NOT NULL DEFAULT FALSE, UNIQUE (block_height, block_hash, hashed_address, storage_key, storage_value) @@ -14,6 +15,8 @@ CREATE TABLE public.storage_diff CREATE INDEX storage_diff_checked_index ON public.storage_diff (checked) WHERE checked = false; +CREATE INDEX storage_diff_eth_node + ON public.storage_diff (eth_node_id); -- +goose Down -DROP TABLE public.storage_diff; \ No newline at end of file +DROP TABLE public.storage_diff; diff --git a/db/migrations/00011_create_create_back_filled_diff_function.sql b/db/migrations/00011_create_create_back_filled_diff_function.sql index b4e8150da..411b14ab6 100644 --- a/db/migrations/00011_create_create_back_filled_diff_function.sql +++ b/db/migrations/00011_create_create_back_filled_diff_function.sql @@ -1,7 +1,7 @@ -- +goose Up -- +goose StatementBegin CREATE OR REPLACE FUNCTION public.create_back_filled_diff(block_height BIGINT, block_hash BYTEA, hashed_address BYTEA, - storage_key BYTEA, storage_value BYTEA) RETURNS VOID AS + storage_key BYTEA, storage_value BYTEA, eth_node_id INTEGER) RETURNS VOID AS $$ DECLARE last_storage_value BYTEA := ( @@ -26,10 +26,10 @@ BEGIN END IF; INSERT INTO public.storage_diff (block_height, block_hash, hashed_address, storage_key, storage_value, - from_backfill) + eth_node_id, from_backfill) VALUES (create_back_filled_diff.block_height, create_back_filled_diff.block_hash, create_back_filled_diff.hashed_address, create_back_filled_diff.storage_key, - create_back_filled_diff.storage_value, true) + create_back_filled_diff.storage_value, create_back_filled_diff.eth_node_id, true) ON CONFLICT DO NOTHING; RETURN; @@ -38,8 +38,8 @@ $$ LANGUAGE plpgsql; -- +goose StatementEnd -COMMENT ON FUNCTION public.create_back_filled_diff(block_height BIGINT, block_hash BYTEA, hashed_address BYTEA, storage_key BYTEA, storage_value BYTEA) +COMMENT ON FUNCTION public.create_back_filled_diff(block_height BIGINT, block_hash BYTEA, hashed_address BYTEA, storage_key BYTEA, storage_value BYTEA, eth_node_id INTEGER) IS E'@omit'; -- +goose Down -DROP FUNCTION public.create_back_filled_diff(block_height BIGINT, block_hash BYTEA, hashed_address BYTEA, storage_key BYTEA, storage_value BYTEA); +DROP FUNCTION public.create_back_filled_diff(block_height BIGINT, block_hash BYTEA, hashed_address BYTEA, storage_key BYTEA, storage_value BYTEA, eth_node_id INTEGER); diff --git a/db/schema.sql b/db/schema.sql index f0b6363ef..5017b391d 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -17,10 +17,10 @@ SET client_min_messages = warning; SET row_security = off; -- --- Name: create_back_filled_diff(bigint, bytea, bytea, bytea, bytea); Type: FUNCTION; Schema: public; Owner: - +-- Name: create_back_filled_diff(bigint, bytea, bytea, bytea, bytea, integer); Type: FUNCTION; Schema: public; Owner: - -- -CREATE FUNCTION public.create_back_filled_diff(block_height bigint, block_hash bytea, hashed_address bytea, storage_key bytea, storage_value bytea) RETURNS void +CREATE FUNCTION public.create_back_filled_diff(block_height bigint, block_hash bytea, hashed_address bytea, storage_key bytea, storage_value bytea, eth_node_id integer) RETURNS void LANGUAGE plpgsql AS $$ DECLARE @@ -46,10 +46,10 @@ BEGIN END IF; INSERT INTO public.storage_diff (block_height, block_hash, hashed_address, storage_key, storage_value, - from_backfill) + eth_node_id, from_backfill) VALUES (create_back_filled_diff.block_height, create_back_filled_diff.block_hash, create_back_filled_diff.hashed_address, create_back_filled_diff.storage_key, - create_back_filled_diff.storage_value, true) + create_back_filled_diff.storage_value, create_back_filled_diff.eth_node_id, true) ON CONFLICT DO NOTHING; RETURN; @@ -58,10 +58,10 @@ $$; -- --- Name: FUNCTION create_back_filled_diff(block_height bigint, block_hash bytea, hashed_address bytea, storage_key bytea, storage_value bytea); Type: COMMENT; Schema: public; Owner: - +-- Name: FUNCTION create_back_filled_diff(block_height bigint, block_hash bytea, hashed_address bytea, storage_key bytea, storage_value bytea, eth_node_id integer); Type: COMMENT; Schema: public; Owner: - -- -COMMENT ON FUNCTION public.create_back_filled_diff(block_height bigint, block_hash bytea, hashed_address bytea, storage_key bytea, storage_value bytea) IS '@omit'; +COMMENT ON FUNCTION public.create_back_filled_diff(block_height bigint, block_hash bytea, hashed_address bytea, storage_key bytea, storage_value bytea, eth_node_id integer) IS '@omit'; -- @@ -389,6 +389,7 @@ CREATE TABLE public.storage_diff ( hashed_address bytea, storage_key bytea, storage_value bytea, + eth_node_id integer NOT NULL, checked boolean DEFAULT false NOT NULL, from_backfill boolean DEFAULT false NOT NULL ); @@ -768,6 +769,13 @@ CREATE INDEX receipts_transaction ON public.receipts USING btree (transaction_id CREATE INDEX storage_diff_checked_index ON public.storage_diff USING btree (checked) WHERE (checked = false); +-- +-- Name: storage_diff_eth_node; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX storage_diff_eth_node ON public.storage_diff USING btree (eth_node_id); + + -- -- Name: transactions_header; Type: INDEX; Schema: public; Owner: - -- @@ -846,6 +854,14 @@ ALTER TABLE ONLY public.receipts ADD CONSTRAINT receipts_transaction_id_fkey FOREIGN KEY (transaction_id) REFERENCES public.transactions(id) ON DELETE CASCADE; +-- +-- Name: storage_diff storage_diff_eth_node_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.storage_diff + ADD CONSTRAINT storage_diff_eth_node_id_fkey FOREIGN KEY (eth_node_id) REFERENCES public.eth_nodes(id) ON DELETE CASCADE; + + -- -- Name: transactions transactions_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- diff --git a/libraries/shared/storage/diff_repository.go b/libraries/shared/storage/diff_repository.go index 648bf7037..f82b5ebda 100644 --- a/libraries/shared/storage/diff_repository.go +++ b/libraries/shared/storage/diff_repository.go @@ -43,9 +43,9 @@ func NewDiffRepository(db *postgres.DB) diffRepository { func (repository diffRepository) CreateStorageDiff(rawDiff types.RawDiff) (int64, error) { var storageDiffID int64 row := repository.db.QueryRowx(`INSERT INTO public.storage_diff - (hashed_address, block_height, block_hash, storage_key, storage_value) VALUES ($1, $2, $3, $4, $5) + (hashed_address, block_height, block_hash, storage_key, storage_value, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT DO NOTHING RETURNING id`, rawDiff.HashedAddress.Bytes(), rawDiff.BlockHeight, rawDiff.BlockHash.Bytes(), - rawDiff.StorageKey.Bytes(), rawDiff.StorageValue.Bytes()) + rawDiff.StorageKey.Bytes(), rawDiff.StorageValue.Bytes(), repository.db.NodeID) err := row.Scan(&storageDiffID) if err != nil { return 0, fmt.Errorf("error creating storage diff: %w", err) @@ -54,9 +54,9 @@ func (repository diffRepository) CreateStorageDiff(rawDiff types.RawDiff) (int64 } func (repository diffRepository) CreateBackFilledStorageValue(rawDiff types.RawDiff) error { - _, err := repository.db.Exec(`SELECT * FROM public.create_back_filled_diff($1, $2, $3, $4, $5)`, + _, err := repository.db.Exec(`SELECT * FROM public.create_back_filled_diff($1, $2, $3, $4, $5, $6)`, rawDiff.BlockHeight, rawDiff.BlockHash.Bytes(), rawDiff.HashedAddress.Bytes(), - rawDiff.StorageKey.Bytes(), rawDiff.StorageValue.Bytes()) + rawDiff.StorageKey.Bytes(), rawDiff.StorageValue.Bytes(), repository.db.NodeID) if err != nil { return fmt.Errorf("error creating back filled storage value: %w", err) } diff --git a/libraries/shared/storage/diff_repository_test.go b/libraries/shared/storage/diff_repository_test.go index 570db99f4..17b59132c 100644 --- a/libraries/shared/storage/diff_repository_test.go +++ b/libraries/shared/storage/diff_repository_test.go @@ -200,13 +200,14 @@ var _ = Describe("Storage diffs repository", func() { StorageValue: test_data.FakeHash(), } fakePersistedDiff := types.PersistedDiff{ - RawDiff: fakeRawDiff, - ID: rand.Int63(), + RawDiff: fakeRawDiff, + ID: rand.Int63(), + EthNodeID: db.NodeID, } _, insertErr := db.Exec(`INSERT INTO public.storage_diff (id, block_height, block_hash, - hashed_address, storage_key, storage_value) VALUES ($1, $2, $3, $4, $5, $6)`, fakePersistedDiff.ID, + hashed_address, storage_key, storage_value, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6, $7)`, fakePersistedDiff.ID, fakeRawDiff.BlockHeight, fakeRawDiff.BlockHash.Bytes(), fakeRawDiff.HashedAddress.Bytes(), - fakeRawDiff.StorageKey.Bytes(), fakeRawDiff.StorageValue.Bytes()) + fakeRawDiff.StorageKey.Bytes(), fakeRawDiff.StorageValue.Bytes(), fakePersistedDiff.EthNodeID) Expect(insertErr).NotTo(HaveOccurred()) diffs, err := repo.GetNewDiffs(0, 1) @@ -224,15 +225,16 @@ var _ = Describe("Storage diffs repository", func() { StorageValue: test_data.FakeHash(), } fakePersistedDiff := types.PersistedDiff{ - RawDiff: fakeRawDiff, - ID: rand.Int63(), - Checked: true, + RawDiff: fakeRawDiff, + ID: rand.Int63(), + Checked: true, + EthNodeID: db.NodeID, } _, insertErr := db.Exec(`INSERT INTO public.storage_diff (id, block_height, block_hash, - hashed_address, storage_key, storage_value, checked) VALUES ($1, $2, $3, $4, $5, $6, $7)`, + hashed_address, storage_key, storage_value, checked, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, fakePersistedDiff.ID, fakeRawDiff.BlockHeight, fakeRawDiff.BlockHash.Bytes(), fakeRawDiff.HashedAddress.Bytes(), fakeRawDiff.StorageKey.Bytes(), fakeRawDiff.StorageValue.Bytes(), - fakePersistedDiff.Checked) + fakePersistedDiff.Checked, fakePersistedDiff.EthNodeID) Expect(insertErr).NotTo(HaveOccurred()) diffs, err := repo.GetNewDiffs(0, 1) @@ -243,6 +245,7 @@ var _ = Describe("Storage diffs repository", func() { It("enables seeking diffs with greater ID", func() { blockZero := rand.Int() + nodeID := db.NodeID for i := 0; i < 2; i++ { fakeRawDiff := types.RawDiff{ HashedAddress: test_data.FakeHash(), @@ -252,9 +255,9 @@ var _ = Describe("Storage diffs repository", func() { StorageValue: test_data.FakeHash(), } _, insertErr := db.Exec(`INSERT INTO public.storage_diff (block_height, block_hash, - hashed_address, storage_key, storage_value) VALUES ($1, $2, $3, $4, $5)`, fakeRawDiff.BlockHeight, + hashed_address, storage_key, storage_value, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6)`, fakeRawDiff.BlockHeight, fakeRawDiff.BlockHash.Bytes(), fakeRawDiff.HashedAddress.Bytes(), fakeRawDiff.StorageKey.Bytes(), - fakeRawDiff.StorageValue.Bytes()) + fakeRawDiff.StorageValue.Bytes(), nodeID) Expect(insertErr).NotTo(HaveOccurred()) } @@ -281,15 +284,16 @@ var _ = Describe("Storage diffs repository", func() { StorageValue: test_data.FakeHash(), } fakePersistedDiff := types.PersistedDiff{ - RawDiff: fakeRawDiff, - ID: rand.Int63(), - Checked: false, + RawDiff: fakeRawDiff, + ID: rand.Int63(), + Checked: false, + EthNodeID: db.NodeID, } _, insertErr := db.Exec(`INSERT INTO public.storage_diff (id, block_height, block_hash, - hashed_address, storage_key, storage_value, checked) VALUES ($1, $2, $3, $4, $5, $6, $7)`, + hashed_address, storage_key, storage_value, checked, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, fakePersistedDiff.ID, fakeRawDiff.BlockHeight, fakeRawDiff.BlockHash.Bytes(), fakeRawDiff.HashedAddress.Bytes(), fakeRawDiff.StorageKey.Bytes(), fakeRawDiff.StorageValue.Bytes(), - fakePersistedDiff.Checked) + fakePersistedDiff.Checked, fakePersistedDiff.EthNodeID) Expect(insertErr).NotTo(HaveOccurred()) err := repo.MarkChecked(fakePersistedDiff.ID) @@ -353,15 +357,16 @@ var _ = Describe("Storage diffs repository", func() { StorageValue: test_data.FakeHash(), } fakePersistedDiff := types.PersistedDiff{ - RawDiff: fakeRawDiff, - ID: rand.Int63(), - Checked: true, + RawDiff: fakeRawDiff, + ID: rand.Int63(), + Checked: true, + EthNodeID: db.NodeID, } _, insertErr := db.Exec(`INSERT INTO public.storage_diff (id, block_height, block_hash, - hashed_address, storage_key, storage_value, checked) VALUES ($1, $2, $3, $4, $5, $6, $7)`, + hashed_address, storage_key, storage_value, checked, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, fakePersistedDiff.ID, fakeRawDiff.BlockHeight, fakeRawDiff.BlockHash.Bytes(), fakeRawDiff.HashedAddress.Bytes(), fakeRawDiff.StorageKey.Bytes(), fakeRawDiff.StorageValue.Bytes(), - fakePersistedDiff.Checked) + fakePersistedDiff.Checked, fakePersistedDiff.EthNodeID) Expect(insertErr).NotTo(HaveOccurred()) var insertedDiffID int64 diff --git a/libraries/shared/storage/types/diff.go b/libraries/shared/storage/types/diff.go index 92f00fffa..8bd87b249 100644 --- a/libraries/shared/storage/types/diff.go +++ b/libraries/shared/storage/types/diff.go @@ -41,6 +41,7 @@ type PersistedDiff struct { FromBackfill bool `db:"from_backfill"` ID int64 HeaderID int64 `db:"header_id"` + EthNodeID int64 `db:"eth_node_id"` } func FromParityCsvRow(csvRow []string) (RawDiff, error) { From bb4e1ba04769913ece03bc13dbeecf6788271c07 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Mon, 17 Aug 2020 14:29:33 -0500 Subject: [PATCH 09/66] Change storge_diff checked column to an enum of allowed statuses new transformed unrecognized noncanonical unwatched --- .../00005_create_storage_diffs_table.sql | 14 +- db/schema.sql | 17 ++- libraries/shared/storage/diff_repository.go | 4 +- .../shared/storage/diff_repository_test.go | 135 +++++++++--------- libraries/shared/storage/types/diff.go | 2 +- 5 files changed, 97 insertions(+), 75 deletions(-) diff --git a/db/migrations/00005_create_storage_diffs_table.sql b/db/migrations/00005_create_storage_diffs_table.sql index ab2cb7f50..db4037f92 100644 --- a/db/migrations/00005_create_storage_diffs_table.sql +++ b/db/migrations/00005_create_storage_diffs_table.sql @@ -1,4 +1,12 @@ -- +goose Up +CREATE TYPE public.diff_status AS ENUM ( + 'new', + 'transformed', + 'unrecognized', + 'noncanonical', + 'unwatched' + ); + CREATE TABLE public.storage_diff ( id BIGSERIAL PRIMARY KEY, @@ -8,15 +16,17 @@ CREATE TABLE public.storage_diff storage_key BYTEA, storage_value BYTEA, eth_node_id INTEGER NOT NULL REFERENCES public.eth_nodes (id) ON DELETE CASCADE, - checked BOOLEAN NOT NULL DEFAULT FALSE, + --to do: update checked to status + checked diff_status NOT NULL DEFAULT 'new', from_backfill BOOLEAN NOT NULL DEFAULT FALSE, UNIQUE (block_height, block_hash, hashed_address, storage_key, storage_value) ); CREATE INDEX storage_diff_checked_index - ON public.storage_diff (checked) WHERE checked = false; + ON public.storage_diff (checked) WHERE checked = 'new'; CREATE INDEX storage_diff_eth_node ON public.storage_diff (eth_node_id); -- +goose Down +DROP TYPE public.diff_status CASCADE; DROP TABLE public.storage_diff; diff --git a/db/schema.sql b/db/schema.sql index 5017b391d..d616ab269 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -16,6 +16,19 @@ SET xmloption = content; SET client_min_messages = warning; SET row_security = off; +-- +-- Name: diff_status; Type: TYPE; Schema: public; Owner: - +-- + +CREATE TYPE public.diff_status AS ENUM ( + 'new', + 'transformed', + 'unrecognized', + 'noncanonical', + 'unwatched' +); + + -- -- Name: create_back_filled_diff(bigint, bytea, bytea, bytea, bytea, integer); Type: FUNCTION; Schema: public; Owner: - -- @@ -390,7 +403,7 @@ CREATE TABLE public.storage_diff ( storage_key bytea, storage_value bytea, eth_node_id integer NOT NULL, - checked boolean DEFAULT false NOT NULL, + checked public.diff_status DEFAULT 'new'::public.diff_status NOT NULL, from_backfill boolean DEFAULT false NOT NULL ); @@ -766,7 +779,7 @@ CREATE INDEX receipts_transaction ON public.receipts USING btree (transaction_id -- Name: storage_diff_checked_index; Type: INDEX; Schema: public; Owner: - -- -CREATE INDEX storage_diff_checked_index ON public.storage_diff USING btree (checked) WHERE (checked = false); +CREATE INDEX storage_diff_checked_index ON public.storage_diff USING btree (checked) WHERE (checked = 'new'::public.diff_status); -- diff --git a/libraries/shared/storage/diff_repository.go b/libraries/shared/storage/diff_repository.go index f82b5ebda..04829bf23 100644 --- a/libraries/shared/storage/diff_repository.go +++ b/libraries/shared/storage/diff_repository.go @@ -65,7 +65,7 @@ func (repository diffRepository) CreateBackFilledStorageValue(rawDiff types.RawD func (repository diffRepository) GetNewDiffs(minID, limit int) ([]types.PersistedDiff, error) { var result []types.PersistedDiff - query := fmt.Sprintf("SELECT * FROM public.storage_diff WHERE checked = false and id > %d ORDER BY id ASC LIMIT %d", minID, limit) + query := fmt.Sprintf("SELECT * FROM public.storage_diff WHERE checked = 'new' and id > %d ORDER BY id ASC LIMIT %d", minID, limit) err := repository.db.Select(&result, query) if err != nil { return nil, fmt.Errorf("error getting unchecked storage diffs with id greater than %d: %w", minID, err) @@ -74,7 +74,7 @@ func (repository diffRepository) GetNewDiffs(minID, limit int) ([]types.Persiste } func (repository diffRepository) MarkChecked(id int64) error { - _, err := repository.db.Exec(`UPDATE public.storage_diff SET checked = true WHERE id = $1`, id) + _, err := repository.db.Exec(`UPDATE public.storage_diff SET checked = 'transformed' WHERE id = $1`, id) if err != nil { return fmt.Errorf("error marking diff %d checked: %w", id, err) } diff --git a/libraries/shared/storage/diff_repository_test.go b/libraries/shared/storage/diff_repository_test.go index 17b59132c..9cfd8a2df 100644 --- a/libraries/shared/storage/diff_repository_test.go +++ b/libraries/shared/storage/diff_repository_test.go @@ -24,6 +24,7 @@ import ( "github.com/makerdao/vulcanizedb/libraries/shared/storage" "github.com/makerdao/vulcanizedb/libraries/shared/storage/types" "github.com/makerdao/vulcanizedb/libraries/shared/test_data" + "github.com/makerdao/vulcanizedb/pkg/datastore/postgres" "github.com/makerdao/vulcanizedb/test_config" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -63,6 +64,7 @@ var _ = Describe("Storage diffs repository", func() { Expect(persisted.BlockHeight).To(Equal(fakeStorageDiff.BlockHeight)) Expect(persisted.StorageKey).To(Equal(fakeStorageDiff.StorageKey)) Expect(persisted.StorageValue).To(Equal(fakeStorageDiff.StorageValue)) + Expect(persisted.Checked).To(Equal("new")) }) It("does not duplicate storage diffs", func() { @@ -93,6 +95,7 @@ var _ = Describe("Storage diffs repository", func() { Expect(persisted.BlockHeight).To(Equal(fakeStorageDiff.BlockHeight)) Expect(persisted.StorageKey).To(Equal(fakeStorageDiff.StorageKey)) Expect(persisted.StorageValue).To(Equal(fakeStorageDiff.StorageValue)) + Expect(persisted.Checked).To(Equal("new")) }) It("marks diff as back-filled", func() { @@ -192,23 +195,13 @@ var _ = Describe("Storage diffs repository", func() { Describe("GetNewDiffs", func() { It("sends diffs that are not marked as checked", func() { - fakeRawDiff := types.RawDiff{ - HashedAddress: test_data.FakeHash(), - BlockHash: test_data.FakeHash(), - BlockHeight: rand.Int(), - StorageKey: test_data.FakeHash(), - StorageValue: test_data.FakeHash(), - } fakePersistedDiff := types.PersistedDiff{ - RawDiff: fakeRawDiff, + RawDiff: fakeStorageDiff, ID: rand.Int63(), EthNodeID: db.NodeID, + Checked: "new", } - _, insertErr := db.Exec(`INSERT INTO public.storage_diff (id, block_height, block_hash, - hashed_address, storage_key, storage_value, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6, $7)`, fakePersistedDiff.ID, - fakeRawDiff.BlockHeight, fakeRawDiff.BlockHash.Bytes(), fakeRawDiff.HashedAddress.Bytes(), - fakeRawDiff.StorageKey.Bytes(), fakeRawDiff.StorageValue.Bytes(), fakePersistedDiff.EthNodeID) - Expect(insertErr).NotTo(HaveOccurred()) + insertTestDiff(fakePersistedDiff, db) diffs, err := repo.GetNewDiffs(0, 1) @@ -216,26 +209,44 @@ var _ = Describe("Storage diffs repository", func() { Expect(diffs).To(ConsistOf(fakePersistedDiff)) }) - It("does not send diff that's marked as checked", func() { - fakeRawDiff := types.RawDiff{ - HashedAddress: test_data.FakeHash(), - BlockHash: test_data.FakeHash(), - BlockHeight: rand.Int(), - StorageKey: test_data.FakeHash(), - StorageValue: test_data.FakeHash(), + It("does not send diff that are marked as transformed", func() { + transformedPersistedDiff := types.PersistedDiff{ + RawDiff: fakeStorageDiff, + ID: rand.Int63(), + Checked: "transformed", + EthNodeID: db.NodeID, } - fakePersistedDiff := types.PersistedDiff{ - RawDiff: fakeRawDiff, + insertTestDiff(transformedPersistedDiff, db) + + diffs, err := repo.GetNewDiffs(0, 1) + + Expect(err).NotTo(HaveOccurred()) + Expect(diffs).To(BeEmpty()) + }) + + It("does not send diff that are marked as noncanonical", func() { + noncanonicalPersistedDiff := types.PersistedDiff{ + RawDiff: fakeStorageDiff, ID: rand.Int63(), - Checked: true, + Checked: "noncanonical", EthNodeID: db.NodeID, } - _, insertErr := db.Exec(`INSERT INTO public.storage_diff (id, block_height, block_hash, - hashed_address, storage_key, storage_value, checked, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, - fakePersistedDiff.ID, fakeRawDiff.BlockHeight, fakeRawDiff.BlockHash.Bytes(), - fakeRawDiff.HashedAddress.Bytes(), fakeRawDiff.StorageKey.Bytes(), fakeRawDiff.StorageValue.Bytes(), - fakePersistedDiff.Checked, fakePersistedDiff.EthNodeID) - Expect(insertErr).NotTo(HaveOccurred()) + insertTestDiff(noncanonicalPersistedDiff, db) + + diffs, err := repo.GetNewDiffs(0, 1) + + Expect(err).NotTo(HaveOccurred()) + Expect(diffs).To(BeEmpty()) + }) + + It("does not send diff that are marked as unrecognized", func() { + unrecognizedPersistedDiff := types.PersistedDiff{ + RawDiff: fakeStorageDiff, + ID: rand.Int63(), + Checked: "unrecognized", + EthNodeID: db.NodeID, + } + insertTestDiff(unrecognizedPersistedDiff, db) diffs, err := repo.GetNewDiffs(0, 1) @@ -245,7 +256,6 @@ var _ = Describe("Storage diffs repository", func() { It("enables seeking diffs with greater ID", func() { blockZero := rand.Int() - nodeID := db.NodeID for i := 0; i < 2; i++ { fakeRawDiff := types.RawDiff{ HashedAddress: test_data.FakeHash(), @@ -254,11 +264,13 @@ var _ = Describe("Storage diffs repository", func() { StorageKey: test_data.FakeHash(), StorageValue: test_data.FakeHash(), } - _, insertErr := db.Exec(`INSERT INTO public.storage_diff (block_height, block_hash, - hashed_address, storage_key, storage_value, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6)`, fakeRawDiff.BlockHeight, - fakeRawDiff.BlockHash.Bytes(), fakeRawDiff.HashedAddress.Bytes(), fakeRawDiff.StorageKey.Bytes(), - fakeRawDiff.StorageValue.Bytes(), nodeID) - Expect(insertErr).NotTo(HaveOccurred()) + persistedDiff := types.PersistedDiff{ + RawDiff: fakeRawDiff, + ID: rand.Int63(), + Checked: "new", + EthNodeID: db.NodeID, + } + insertTestDiff(persistedDiff, db) } minID := 0 @@ -276,33 +288,21 @@ var _ = Describe("Storage diffs repository", func() { Describe("MarkChecked", func() { It("marks a diff as checked", func() { - fakeRawDiff := types.RawDiff{ - HashedAddress: test_data.FakeHash(), - BlockHash: test_data.FakeHash(), - BlockHeight: rand.Int(), - StorageKey: test_data.FakeHash(), - StorageValue: test_data.FakeHash(), - } fakePersistedDiff := types.PersistedDiff{ - RawDiff: fakeRawDiff, + RawDiff: fakeStorageDiff, ID: rand.Int63(), - Checked: false, + Checked: "new", EthNodeID: db.NodeID, } - _, insertErr := db.Exec(`INSERT INTO public.storage_diff (id, block_height, block_hash, - hashed_address, storage_key, storage_value, checked, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, - fakePersistedDiff.ID, fakeRawDiff.BlockHeight, fakeRawDiff.BlockHash.Bytes(), - fakeRawDiff.HashedAddress.Bytes(), fakeRawDiff.StorageKey.Bytes(), fakeRawDiff.StorageValue.Bytes(), - fakePersistedDiff.Checked, fakePersistedDiff.EthNodeID) - Expect(insertErr).NotTo(HaveOccurred()) + insertTestDiff(fakePersistedDiff, db) err := repo.MarkChecked(fakePersistedDiff.ID) Expect(err).NotTo(HaveOccurred()) - var checked bool + var checked string checkedErr := db.Get(&checked, `SELECT checked FROM public.storage_diff WHERE id = $1`, fakePersistedDiff.ID) Expect(checkedErr).NotTo(HaveOccurred()) - Expect(checked).To(BeTrue()) + Expect(checked).To(Equal("transformed")) }) }) @@ -349,31 +349,20 @@ var _ = Describe("Storage diffs repository", func() { }) It("won't fail if all of the diffs within the id range are already checked", func() { - fakeRawDiff := types.RawDiff{ - HashedAddress: test_data.FakeHash(), - BlockHash: test_data.FakeHash(), - BlockHeight: rand.Int(), - StorageKey: test_data.FakeHash(), - StorageValue: test_data.FakeHash(), - } fakePersistedDiff := types.PersistedDiff{ - RawDiff: fakeRawDiff, + RawDiff: fakeStorageDiff, ID: rand.Int63(), - Checked: true, + Checked: "transformed", EthNodeID: db.NodeID, } - _, insertErr := db.Exec(`INSERT INTO public.storage_diff (id, block_height, block_hash, - hashed_address, storage_key, storage_value, checked, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, - fakePersistedDiff.ID, fakeRawDiff.BlockHeight, fakeRawDiff.BlockHash.Bytes(), - fakeRawDiff.HashedAddress.Bytes(), fakeRawDiff.StorageKey.Bytes(), fakeRawDiff.StorageValue.Bytes(), - fakePersistedDiff.Checked, fakePersistedDiff.EthNodeID) - Expect(insertErr).NotTo(HaveOccurred()) + + insertTestDiff(fakePersistedDiff, db) var insertedDiffID int64 getInsertedDiffIDErr := db.Get(&insertedDiffID, `SELECT id FROM storage_diff LIMIT 1`) Expect(getInsertedDiffIDErr).NotTo(HaveOccurred()) - blockBeforeDiffBlockHeight := int64(fakeRawDiff.BlockHeight - 1) + blockBeforeDiffBlockHeight := int64(fakeStorageDiff.BlockHeight - 1) diffID, diffErr := repo.GetFirstDiffIDForBlockHeight(blockBeforeDiffBlockHeight) Expect(diffErr).NotTo(HaveOccurred()) Expect(diffID).To(Equal(insertedDiffID)) @@ -386,3 +375,13 @@ var _ = Describe("Storage diffs repository", func() { }) }) }) + +func insertTestDiff(persistedDiff types.PersistedDiff, db *postgres.DB) { + rawDiff := persistedDiff.RawDiff + _, insertErr := db.Exec(`INSERT INTO public.storage_diff (id, block_height, block_hash, + hashed_address, storage_key, storage_value, checked, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, + persistedDiff.ID, rawDiff.BlockHeight, rawDiff.BlockHash.Bytes(), + rawDiff.HashedAddress.Bytes(), rawDiff.StorageKey.Bytes(), rawDiff.StorageValue.Bytes(), + persistedDiff.Checked, persistedDiff.EthNodeID) + Expect(insertErr).NotTo(HaveOccurred()) +} diff --git a/libraries/shared/storage/types/diff.go b/libraries/shared/storage/types/diff.go index 8bd87b249..c5f9c9f21 100644 --- a/libraries/shared/storage/types/diff.go +++ b/libraries/shared/storage/types/diff.go @@ -37,7 +37,7 @@ type RawDiff struct { type PersistedDiff struct { RawDiff - Checked bool + Checked string FromBackfill bool `db:"from_backfill"` ID int64 HeaderID int64 `db:"header_id"` From c0f53fd40b61b3fd992b52f089d762cd807d8897 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Mon, 17 Aug 2020 15:56:10 -0500 Subject: [PATCH 10/66] Change storage_diff.checked -> storage_diff.status --- .../00005_create_storage_diffs_table.sql | 11 ++++----- db/schema.sql | 10 ++++---- libraries/shared/storage/diff_repository.go | 4 ++-- .../shared/storage/diff_repository_test.go | 24 +++++++++---------- libraries/shared/storage/types/diff.go | 2 +- 5 files changed, 25 insertions(+), 26 deletions(-) diff --git a/db/migrations/00005_create_storage_diffs_table.sql b/db/migrations/00005_create_storage_diffs_table.sql index db4037f92..906c44c90 100644 --- a/db/migrations/00005_create_storage_diffs_table.sql +++ b/db/migrations/00005_create_storage_diffs_table.sql @@ -15,15 +15,14 @@ CREATE TABLE public.storage_diff hashed_address BYTEA, storage_key BYTEA, storage_value BYTEA, - eth_node_id INTEGER NOT NULL REFERENCES public.eth_nodes (id) ON DELETE CASCADE, - --to do: update checked to status - checked diff_status NOT NULL DEFAULT 'new', - from_backfill BOOLEAN NOT NULL DEFAULT FALSE, + eth_node_id INTEGER NOT NULL REFERENCES public.eth_nodes (id) ON DELETE CASCADE, + status diff_status NOT NULL DEFAULT 'new', + from_backfill BOOLEAN NOT NULL DEFAULT FALSE, UNIQUE (block_height, block_hash, hashed_address, storage_key, storage_value) ); -CREATE INDEX storage_diff_checked_index - ON public.storage_diff (checked) WHERE checked = 'new'; +CREATE INDEX storage_diff_new_status_index + ON public.storage_diff (status) WHERE status = 'new'; CREATE INDEX storage_diff_eth_node ON public.storage_diff (eth_node_id); diff --git a/db/schema.sql b/db/schema.sql index d616ab269..42a985bbb 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -403,7 +403,7 @@ CREATE TABLE public.storage_diff ( storage_key bytea, storage_value bytea, eth_node_id integer NOT NULL, - checked public.diff_status DEFAULT 'new'::public.diff_status NOT NULL, + status public.diff_status DEFAULT 'new'::public.diff_status NOT NULL, from_backfill boolean DEFAULT false NOT NULL ); @@ -776,17 +776,17 @@ CREATE INDEX receipts_transaction ON public.receipts USING btree (transaction_id -- --- Name: storage_diff_checked_index; Type: INDEX; Schema: public; Owner: - +-- Name: storage_diff_eth_node; Type: INDEX; Schema: public; Owner: - -- -CREATE INDEX storage_diff_checked_index ON public.storage_diff USING btree (checked) WHERE (checked = 'new'::public.diff_status); +CREATE INDEX storage_diff_eth_node ON public.storage_diff USING btree (eth_node_id); -- --- Name: storage_diff_eth_node; Type: INDEX; Schema: public; Owner: - +-- Name: storage_diff_new_status_index; Type: INDEX; Schema: public; Owner: - -- -CREATE INDEX storage_diff_eth_node ON public.storage_diff USING btree (eth_node_id); +CREATE INDEX storage_diff_new_status_index ON public.storage_diff USING btree (status) WHERE (status = 'new'::public.diff_status); -- diff --git a/libraries/shared/storage/diff_repository.go b/libraries/shared/storage/diff_repository.go index 04829bf23..225af2482 100644 --- a/libraries/shared/storage/diff_repository.go +++ b/libraries/shared/storage/diff_repository.go @@ -65,7 +65,7 @@ func (repository diffRepository) CreateBackFilledStorageValue(rawDiff types.RawD func (repository diffRepository) GetNewDiffs(minID, limit int) ([]types.PersistedDiff, error) { var result []types.PersistedDiff - query := fmt.Sprintf("SELECT * FROM public.storage_diff WHERE checked = 'new' and id > %d ORDER BY id ASC LIMIT %d", minID, limit) + query := fmt.Sprintf("SELECT * FROM public.storage_diff WHERE status = 'new' and id > %d ORDER BY id ASC LIMIT %d", minID, limit) err := repository.db.Select(&result, query) if err != nil { return nil, fmt.Errorf("error getting unchecked storage diffs with id greater than %d: %w", minID, err) @@ -74,7 +74,7 @@ func (repository diffRepository) GetNewDiffs(minID, limit int) ([]types.Persiste } func (repository diffRepository) MarkChecked(id int64) error { - _, err := repository.db.Exec(`UPDATE public.storage_diff SET checked = 'transformed' WHERE id = $1`, id) + _, err := repository.db.Exec(`UPDATE public.storage_diff SET status = 'transformed' WHERE id = $1`, id) if err != nil { return fmt.Errorf("error marking diff %d checked: %w", id, err) } diff --git a/libraries/shared/storage/diff_repository_test.go b/libraries/shared/storage/diff_repository_test.go index 9cfd8a2df..daaf3e46b 100644 --- a/libraries/shared/storage/diff_repository_test.go +++ b/libraries/shared/storage/diff_repository_test.go @@ -64,7 +64,7 @@ var _ = Describe("Storage diffs repository", func() { Expect(persisted.BlockHeight).To(Equal(fakeStorageDiff.BlockHeight)) Expect(persisted.StorageKey).To(Equal(fakeStorageDiff.StorageKey)) Expect(persisted.StorageValue).To(Equal(fakeStorageDiff.StorageValue)) - Expect(persisted.Checked).To(Equal("new")) + Expect(persisted.Status).To(Equal("new")) }) It("does not duplicate storage diffs", func() { @@ -95,7 +95,7 @@ var _ = Describe("Storage diffs repository", func() { Expect(persisted.BlockHeight).To(Equal(fakeStorageDiff.BlockHeight)) Expect(persisted.StorageKey).To(Equal(fakeStorageDiff.StorageKey)) Expect(persisted.StorageValue).To(Equal(fakeStorageDiff.StorageValue)) - Expect(persisted.Checked).To(Equal("new")) + Expect(persisted.Status).To(Equal("new")) }) It("marks diff as back-filled", func() { @@ -199,7 +199,7 @@ var _ = Describe("Storage diffs repository", func() { RawDiff: fakeStorageDiff, ID: rand.Int63(), EthNodeID: db.NodeID, - Checked: "new", + Status: "new", } insertTestDiff(fakePersistedDiff, db) @@ -213,7 +213,7 @@ var _ = Describe("Storage diffs repository", func() { transformedPersistedDiff := types.PersistedDiff{ RawDiff: fakeStorageDiff, ID: rand.Int63(), - Checked: "transformed", + Status: "transformed", EthNodeID: db.NodeID, } insertTestDiff(transformedPersistedDiff, db) @@ -228,7 +228,7 @@ var _ = Describe("Storage diffs repository", func() { noncanonicalPersistedDiff := types.PersistedDiff{ RawDiff: fakeStorageDiff, ID: rand.Int63(), - Checked: "noncanonical", + Status: "noncanonical", EthNodeID: db.NodeID, } insertTestDiff(noncanonicalPersistedDiff, db) @@ -243,7 +243,7 @@ var _ = Describe("Storage diffs repository", func() { unrecognizedPersistedDiff := types.PersistedDiff{ RawDiff: fakeStorageDiff, ID: rand.Int63(), - Checked: "unrecognized", + Status: "unrecognized", EthNodeID: db.NodeID, } insertTestDiff(unrecognizedPersistedDiff, db) @@ -267,7 +267,7 @@ var _ = Describe("Storage diffs repository", func() { persistedDiff := types.PersistedDiff{ RawDiff: fakeRawDiff, ID: rand.Int63(), - Checked: "new", + Status: "new", EthNodeID: db.NodeID, } insertTestDiff(persistedDiff, db) @@ -291,7 +291,7 @@ var _ = Describe("Storage diffs repository", func() { fakePersistedDiff := types.PersistedDiff{ RawDiff: fakeStorageDiff, ID: rand.Int63(), - Checked: "new", + Status: "new", EthNodeID: db.NodeID, } insertTestDiff(fakePersistedDiff, db) @@ -300,7 +300,7 @@ var _ = Describe("Storage diffs repository", func() { Expect(err).NotTo(HaveOccurred()) var checked string - checkedErr := db.Get(&checked, `SELECT checked FROM public.storage_diff WHERE id = $1`, fakePersistedDiff.ID) + checkedErr := db.Get(&checked, `SELECT status FROM public.storage_diff WHERE id = $1`, fakePersistedDiff.ID) Expect(checkedErr).NotTo(HaveOccurred()) Expect(checked).To(Equal("transformed")) }) @@ -352,7 +352,7 @@ var _ = Describe("Storage diffs repository", func() { fakePersistedDiff := types.PersistedDiff{ RawDiff: fakeStorageDiff, ID: rand.Int63(), - Checked: "transformed", + Status: "transformed", EthNodeID: db.NodeID, } @@ -379,9 +379,9 @@ var _ = Describe("Storage diffs repository", func() { func insertTestDiff(persistedDiff types.PersistedDiff, db *postgres.DB) { rawDiff := persistedDiff.RawDiff _, insertErr := db.Exec(`INSERT INTO public.storage_diff (id, block_height, block_hash, - hashed_address, storage_key, storage_value, checked, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, + hashed_address, storage_key, storage_value, status, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, persistedDiff.ID, rawDiff.BlockHeight, rawDiff.BlockHash.Bytes(), rawDiff.HashedAddress.Bytes(), rawDiff.StorageKey.Bytes(), rawDiff.StorageValue.Bytes(), - persistedDiff.Checked, persistedDiff.EthNodeID) + persistedDiff.Status, persistedDiff.EthNodeID) Expect(insertErr).NotTo(HaveOccurred()) } diff --git a/libraries/shared/storage/types/diff.go b/libraries/shared/storage/types/diff.go index c5f9c9f21..4e3542248 100644 --- a/libraries/shared/storage/types/diff.go +++ b/libraries/shared/storage/types/diff.go @@ -37,7 +37,7 @@ type RawDiff struct { type PersistedDiff struct { RawDiff - Checked string + Status string FromBackfill bool `db:"from_backfill"` ID int64 HeaderID int64 `db:"header_id"` From bb680ec3b7ab73000effd8ce83abca2931556239 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Mon, 17 Aug 2020 16:24:21 -0500 Subject: [PATCH 11/66] Allow storage_diffs to be marked transformed, unrecognized, noncanonical, unwatched --- .../shared/mocks/storage_diff_repository.go | 15 +++++- libraries/shared/storage/diff_repository.go | 31 +++++++++++- .../shared/storage/diff_repository_test.go | 49 ++++++++++++++++--- libraries/shared/watcher/storage_watcher.go | 10 ++-- 4 files changed, 89 insertions(+), 16 deletions(-) diff --git a/libraries/shared/mocks/storage_diff_repository.go b/libraries/shared/mocks/storage_diff_repository.go index 2c1f65eb0..4256d2a9d 100644 --- a/libraries/shared/mocks/storage_diff_repository.go +++ b/libraries/shared/mocks/storage_diff_repository.go @@ -54,11 +54,24 @@ func (repository *MockStorageDiffRepository) GetNewDiffs(minID, limit int) ([]ty return repository.GetNewDiffsDiffs, err } -func (repository *MockStorageDiffRepository) MarkChecked(id int64) error { +func (repository *MockStorageDiffRepository) MarkTransformed(id int64) error { repository.MarkCheckedPassedID = id return nil } +func (repository *MockStorageDiffRepository) MarkNoncanonical(id int64) error { + panic("implement me") +} + +func (repository *MockStorageDiffRepository) MarkUnrecognized(id int64) error { + repository.MarkUnrecognizedPassedID = id + return nil +} + +func (repository *MockStorageDiffRepository) MarkUnwatched(id int64) error { + panic("implement me") +} + func (repository *MockStorageDiffRepository) GetFirstDiffIDForBlockHeight(blockHeight int64) (int64, error) { repository.GetFirstDiffBlockHeightPassed = blockHeight return repository.GetFirstDiffIDToReturn, repository.GetFirstDiffIDErr diff --git a/libraries/shared/storage/diff_repository.go b/libraries/shared/storage/diff_repository.go index 225af2482..a1599a203 100644 --- a/libraries/shared/storage/diff_repository.go +++ b/libraries/shared/storage/diff_repository.go @@ -27,7 +27,10 @@ type DiffRepository interface { CreateStorageDiff(rawDiff types.RawDiff) (int64, error) CreateBackFilledStorageValue(rawDiff types.RawDiff) error GetNewDiffs(minID, limit int) ([]types.PersistedDiff, error) - MarkChecked(id int64) error + MarkTransformed(id int64) error + MarkNoncanonical(id int64) error + MarkUnrecognized(id int64) error + MarkUnwatched(id int64) error GetFirstDiffIDForBlockHeight(blockHeight int64) (int64, error) } @@ -73,8 +76,32 @@ func (repository diffRepository) GetNewDiffs(minID, limit int) ([]types.Persiste return result, nil } -func (repository diffRepository) MarkChecked(id int64) error { +func (repository diffRepository) MarkTransformed(id int64) error { _, err := repository.db.Exec(`UPDATE public.storage_diff SET status = 'transformed' WHERE id = $1`, id) + if err != nil { + return fmt.Errorf("error marking diff %d transformed: %w", id, err) + } + return nil +} + +func (repository diffRepository) MarkUnrecognized(id int64) error { + _, err := repository.db.Exec(`UPDATE public.storage_diff SET status = 'unrecognized' WHERE id = $1`, id) + if err != nil { + return fmt.Errorf("error marking diff %d checked: %w", id, err) + } + return nil +} + +func (repository diffRepository) MarkNoncanonical(id int64) error { + _, err := repository.db.Exec(`UPDATE public.storage_diff SET status = 'noncanonical' WHERE id = $1`, id) + if err != nil { + return fmt.Errorf("error marking diff %d checked: %w", id, err) + } + return nil +} + +func (repository diffRepository) MarkUnwatched(id int64) error { + _, err := repository.db.Exec(`UPDATE public.storage_diff SET status = 'unwatched' WHERE id = $1`, id) if err != nil { return fmt.Errorf("error marking diff %d checked: %w", id, err) } diff --git a/libraries/shared/storage/diff_repository_test.go b/libraries/shared/storage/diff_repository_test.go index daaf3e46b..f945a8d1a 100644 --- a/libraries/shared/storage/diff_repository_test.go +++ b/libraries/shared/storage/diff_repository_test.go @@ -286,23 +286,56 @@ var _ = Describe("Storage diffs repository", func() { }) }) - Describe("MarkChecked", func() { - It("marks a diff as checked", func() { - fakePersistedDiff := types.PersistedDiff{ + Describe("Changing the diff status", func() { + var fakePersistedDiff types.PersistedDiff + BeforeEach(func() { + fakePersistedDiff = types.PersistedDiff{ RawDiff: fakeStorageDiff, ID: rand.Int63(), Status: "new", EthNodeID: db.NodeID, } insertTestDiff(fakePersistedDiff, db) + }) - err := repo.MarkChecked(fakePersistedDiff.ID) + It("marks a diff as transformed", func() { + err := repo.MarkTransformed(fakePersistedDiff.ID) Expect(err).NotTo(HaveOccurred()) - var checked string - checkedErr := db.Get(&checked, `SELECT status FROM public.storage_diff WHERE id = $1`, fakePersistedDiff.ID) - Expect(checkedErr).NotTo(HaveOccurred()) - Expect(checked).To(Equal("transformed")) + var status string + getStatusErr := db.Get(&status, `SELECT status FROM public.storage_diff WHERE id = $1`, fakePersistedDiff.ID) + Expect(getStatusErr).NotTo(HaveOccurred()) + Expect(status).To(Equal("transformed")) + }) + + It("marks a diff as unrecognized", func() { + err := repo.MarkUnrecognized(fakePersistedDiff.ID) + + Expect(err).NotTo(HaveOccurred()) + var status string + getStatusErr := db.Get(&status, `SELECT status FROM public.storage_diff WHERE id = $1`, fakePersistedDiff.ID) + Expect(getStatusErr).NotTo(HaveOccurred()) + Expect(status).To(Equal("unrecognized")) + }) + + It("marks a diff as noncanonical", func() { + err := repo.MarkNoncanonical(fakePersistedDiff.ID) + + Expect(err).NotTo(HaveOccurred()) + var status string + getStatusErr := db.Get(&status, `SELECT status FROM public.storage_diff WHERE id = $1`, fakePersistedDiff.ID) + Expect(getStatusErr).NotTo(HaveOccurred()) + Expect(status).To(Equal("noncanonical")) + }) + + It("marks a diff as unwatched", func() { + err := repo.MarkUnwatched(fakePersistedDiff.ID) + + Expect(err).NotTo(HaveOccurred()) + var status string + getStatusErr := db.Get(&status, `SELECT status FROM public.storage_diff WHERE id = $1`, fakePersistedDiff.ID) + Expect(getStatusErr).NotTo(HaveOccurred()) + Expect(status).To(Equal("unwatched")) }) }) diff --git a/libraries/shared/watcher/storage_watcher.go b/libraries/shared/watcher/storage_watcher.go index f30406f6b..b6b0625a1 100644 --- a/libraries/shared/watcher/storage_watcher.go +++ b/libraries/shared/watcher/storage_watcher.go @@ -144,9 +144,9 @@ func (watcher StorageWatcher) transformDiffs() error { func (watcher StorageWatcher) transformDiff(diff types.PersistedDiff) error { t, watching := watcher.getTransformer(diff) if !watching { - markCheckedErr := watcher.StorageDiffRepository.MarkChecked(diff.ID) - if markCheckedErr != nil { - return fmt.Errorf("error marking diff checked: %w", markCheckedErr) + markTransformedErr := watcher.StorageDiffRepository.MarkTransformed(diff.ID) + if markTransformedErr != nil { + return fmt.Errorf("error marking diff transformed: %w", markTransformedErr) } return nil } @@ -165,7 +165,7 @@ func (watcher StorageWatcher) transformDiff(diff types.PersistedDiff) error { return fmt.Errorf("error executing storage transformer: %w", executeErr) } - markCheckedErr := watcher.StorageDiffRepository.MarkChecked(diff.ID) + markCheckedErr := watcher.StorageDiffRepository.MarkTransformed(diff.ID) if markCheckedErr != nil { return fmt.Errorf("error marking diff checked: %w", markCheckedErr) } @@ -198,7 +198,7 @@ func (watcher StorageWatcher) handleDiffWithInvalidHeaderHash(diff types.Persist return fmt.Errorf(msg, diff.ID, maxBlockErr) } if diff.BlockHeight < int(maxBlock)-ReorgWindow { - return watcher.StorageDiffRepository.MarkChecked(diff.ID) + return watcher.StorageDiffRepository.MarkTransformed(diff.ID) } return nil } From 40808f4bda91b95ef050072f3febda64022b2afc Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Tue, 18 Aug 2020 10:26:39 -0500 Subject: [PATCH 12/66] Mark a diff as unrecognized if the storage key is not found --- .../shared/mocks/storage_diff_repository.go | 1 + libraries/shared/watcher/storage_watcher.go | 26 ++++++++++++++----- .../shared/watcher/storage_watcher_test.go | 11 ++++++++ 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/libraries/shared/mocks/storage_diff_repository.go b/libraries/shared/mocks/storage_diff_repository.go index 4256d2a9d..f96c4ecb2 100644 --- a/libraries/shared/mocks/storage_diff_repository.go +++ b/libraries/shared/mocks/storage_diff_repository.go @@ -29,6 +29,7 @@ type MockStorageDiffRepository struct { GetNewDiffsPassedMinIDs []int GetNewDiffsPassedLimits []int MarkCheckedPassedID int64 + MarkUnrecognizedPassedID int64 GetFirstDiffIDToReturn int64 GetFirstDiffIDErr error GetFirstDiffBlockHeightPassed int64 diff --git a/libraries/shared/watcher/storage_watcher.go b/libraries/shared/watcher/storage_watcher.go index b6b0625a1..e86feed55 100644 --- a/libraries/shared/watcher/storage_watcher.go +++ b/libraries/shared/watcher/storage_watcher.go @@ -123,12 +123,8 @@ func (watcher StorageWatcher) transformDiffs() error { } for _, diff := range diffs { transformErr := watcher.transformDiff(diff) - if transformErr != nil { - if isCommonTransformError(transformErr) { - logrus.Tracef("error transforming diff: %s", transformErr.Error()) - } else { - logrus.Infof("error transforming diff: %s", transformErr.Error()) - } + if handleErr := watcher.handleTransformError(transformErr, diff); handleErr != nil { + return handleErr } } lenDiffs := len(diffs) @@ -198,11 +194,29 @@ func (watcher StorageWatcher) handleDiffWithInvalidHeaderHash(diff types.Persist return fmt.Errorf(msg, diff.ID, maxBlockErr) } if diff.BlockHeight < int(maxBlock)-ReorgWindow { + //TODO: is this the noncanonical case? return watcher.StorageDiffRepository.MarkTransformed(diff.ID) } return nil } +func (watcher StorageWatcher) handleTransformError(transformErr error, diff types.PersistedDiff) error { + if transformErr != nil { + if errors.Is(transformErr, types.ErrKeyNotFound) { + markUnrecognizedErr := watcher.StorageDiffRepository.MarkUnrecognized(diff.ID) + if markUnrecognizedErr != nil { + return markUnrecognizedErr + } + } + if isCommonTransformError(transformErr) { + logrus.Tracef("error transforming diff: %s", transformErr.Error()) + } else { + logrus.Infof("error transforming diff: %s", transformErr.Error()) + } + } + return nil +} + func isCommonTransformError(err error) bool { return errors.Is(err, sql.ErrNoRows) || errors.Is(err, types.ErrKeyNotFound) } diff --git a/libraries/shared/watcher/storage_watcher_test.go b/libraries/shared/watcher/storage_watcher_test.go index 7a6d10929..914158cb9 100644 --- a/libraries/shared/watcher/storage_watcher_test.go +++ b/libraries/shared/watcher/storage_watcher_test.go @@ -367,6 +367,17 @@ var _ = Describe("Storage Watcher", func() { Expect(mockDiffsRepository.MarkCheckedPassedID).NotTo(Equal(fakePersistedDiff.ID)) }) + It("marks diff as 'unrecognized' when transforming the diff returns a ErrKeyNotFound error", func() { + mockTransformer.ExecuteErr = types.ErrKeyNotFound + mockDiffsRepository.GetNewDiffsErrors = []error{nil, types.ErrKeyNotFound} + + err := storageWatcher.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(types.ErrKeyNotFound)) + Expect(mockDiffsRepository.MarkUnrecognizedPassedID).To(Equal(fakePersistedDiff.ID)) + }) + It("marks diff checked if transformer execution doesn't fail", func() { mockDiffsRepository.GetNewDiffsDiffs = []types.PersistedDiff{fakePersistedDiff} mockDiffsRepository.GetNewDiffsErrors = []error{nil, fakes.FakeError} From d18176ad2bca2b025ee4fd86c560ce35f727088a Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Tue, 18 Aug 2020 11:08:32 -0500 Subject: [PATCH 13/66] Mark diffs as unwatched if no transformers are watching the contract --- libraries/shared/mocks/storage_diff_repository.go | 4 +++- libraries/shared/watcher/storage_watcher.go | 6 +++--- libraries/shared/watcher/storage_watcher_test.go | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/libraries/shared/mocks/storage_diff_repository.go b/libraries/shared/mocks/storage_diff_repository.go index f96c4ecb2..fa869b89f 100644 --- a/libraries/shared/mocks/storage_diff_repository.go +++ b/libraries/shared/mocks/storage_diff_repository.go @@ -30,6 +30,7 @@ type MockStorageDiffRepository struct { GetNewDiffsPassedLimits []int MarkCheckedPassedID int64 MarkUnrecognizedPassedID int64 + MarkUnwatchedPassedID int64 GetFirstDiffIDToReturn int64 GetFirstDiffIDErr error GetFirstDiffBlockHeightPassed int64 @@ -70,7 +71,8 @@ func (repository *MockStorageDiffRepository) MarkUnrecognized(id int64) error { } func (repository *MockStorageDiffRepository) MarkUnwatched(id int64) error { - panic("implement me") + repository.MarkUnwatchedPassedID = id + return nil } func (repository *MockStorageDiffRepository) GetFirstDiffIDForBlockHeight(blockHeight int64) (int64, error) { diff --git a/libraries/shared/watcher/storage_watcher.go b/libraries/shared/watcher/storage_watcher.go index e86feed55..657102291 100644 --- a/libraries/shared/watcher/storage_watcher.go +++ b/libraries/shared/watcher/storage_watcher.go @@ -140,9 +140,9 @@ func (watcher StorageWatcher) transformDiffs() error { func (watcher StorageWatcher) transformDiff(diff types.PersistedDiff) error { t, watching := watcher.getTransformer(diff) if !watching { - markTransformedErr := watcher.StorageDiffRepository.MarkTransformed(diff.ID) - if markTransformedErr != nil { - return fmt.Errorf("error marking diff transformed: %w", markTransformedErr) + markUnwatchedErr := watcher.StorageDiffRepository.MarkUnwatched(diff.ID) + if markUnwatchedErr != nil { + return fmt.Errorf("error marking diff unwatched: %w", markUnwatchedErr) } return nil } diff --git a/libraries/shared/watcher/storage_watcher_test.go b/libraries/shared/watcher/storage_watcher_test.go index 914158cb9..3e787795c 100644 --- a/libraries/shared/watcher/storage_watcher_test.go +++ b/libraries/shared/watcher/storage_watcher_test.go @@ -124,7 +124,7 @@ var _ = Describe("Storage Watcher", func() { Expect(mockDiffsRepository.GetNewDiffsPassedMinIDs).To(ConsistOf(0, 0)) }) - It("marks diff as checked if no transformer is watching its address", func() { + It("marks diff as unwatched if no transformer is watching its address", func() { unwatchedDiff := types.PersistedDiff{ RawDiff: types.RawDiff{ HashedAddress: test_data.FakeHash(), @@ -138,7 +138,7 @@ var _ = Describe("Storage Watcher", func() { Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) - Expect(mockDiffsRepository.MarkCheckedPassedID).To(Equal(unwatchedDiff.ID)) + Expect(mockDiffsRepository.MarkUnwatchedPassedID).To(Equal(unwatchedDiff.ID)) }) Describe("When the watcher is configured to skip old diffs", func() { From 7aecbbb222076ec87be24a7350e52e1cc60a8173 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Tue, 18 Aug 2020 11:58:14 -0500 Subject: [PATCH 14/66] Mark diffs as noncanonical if header hashes don't match and block number is outside the reorg window --- libraries/shared/mocks/storage_diff_repository.go | 6 ++++-- libraries/shared/watcher/storage_watcher.go | 9 ++++----- libraries/shared/watcher/storage_watcher_test.go | 4 ++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/libraries/shared/mocks/storage_diff_repository.go b/libraries/shared/mocks/storage_diff_repository.go index fa869b89f..8afd2bac8 100644 --- a/libraries/shared/mocks/storage_diff_repository.go +++ b/libraries/shared/mocks/storage_diff_repository.go @@ -30,7 +30,8 @@ type MockStorageDiffRepository struct { GetNewDiffsPassedLimits []int MarkCheckedPassedID int64 MarkUnrecognizedPassedID int64 - MarkUnwatchedPassedID int64 + MarkNoncanonicalPassedID int64 + MarkUnwatchedPassedID int64 GetFirstDiffIDToReturn int64 GetFirstDiffIDErr error GetFirstDiffBlockHeightPassed int64 @@ -62,7 +63,8 @@ func (repository *MockStorageDiffRepository) MarkTransformed(id int64) error { } func (repository *MockStorageDiffRepository) MarkNoncanonical(id int64) error { - panic("implement me") + repository.MarkNoncanonicalPassedID = id + return nil } func (repository *MockStorageDiffRepository) MarkUnrecognized(id int64) error { diff --git a/libraries/shared/watcher/storage_watcher.go b/libraries/shared/watcher/storage_watcher.go index 657102291..73f3c2ea6 100644 --- a/libraries/shared/watcher/storage_watcher.go +++ b/libraries/shared/watcher/storage_watcher.go @@ -161,9 +161,9 @@ func (watcher StorageWatcher) transformDiff(diff types.PersistedDiff) error { return fmt.Errorf("error executing storage transformer: %w", executeErr) } - markCheckedErr := watcher.StorageDiffRepository.MarkTransformed(diff.ID) - if markCheckedErr != nil { - return fmt.Errorf("error marking diff checked: %w", markCheckedErr) + markTransformedErr := watcher.StorageDiffRepository.MarkTransformed(diff.ID) + if markTransformedErr != nil { + return fmt.Errorf("error marking diff transformed: %w", markTransformedErr) } return nil @@ -194,8 +194,7 @@ func (watcher StorageWatcher) handleDiffWithInvalidHeaderHash(diff types.Persist return fmt.Errorf(msg, diff.ID, maxBlockErr) } if diff.BlockHeight < int(maxBlock)-ReorgWindow { - //TODO: is this the noncanonical case? - return watcher.StorageDiffRepository.MarkTransformed(diff.ID) + return watcher.StorageDiffRepository.MarkNoncanonical(diff.ID) } return nil } diff --git a/libraries/shared/watcher/storage_watcher_test.go b/libraries/shared/watcher/storage_watcher_test.go index 3e787795c..286ea6741 100644 --- a/libraries/shared/watcher/storage_watcher_test.go +++ b/libraries/shared/watcher/storage_watcher_test.go @@ -313,7 +313,7 @@ var _ = Describe("Storage Watcher", func() { Expect(mockDiffsRepository.MarkCheckedPassedID).NotTo(Equal(fakePersistedDiff.ID)) }) - It("marks diff checked if block height less than max known block height minus reorg window", func() { + It("marks diff noncanonical if block height less than max known block height minus reorg window", func() { mockHeaderRepository.MostRecentHeaderBlockNumber = int64(blockNumber + watcher.ReorgWindow + 1) mockDiffsRepository.GetNewDiffsErrors = []error{nil, fakes.FakeError} @@ -321,7 +321,7 @@ var _ = Describe("Storage Watcher", func() { Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) - Expect(mockDiffsRepository.MarkCheckedPassedID).To(Equal(fakePersistedDiff.ID)) + Expect(mockDiffsRepository.MarkNoncanonicalPassedID).To(Equal(fakePersistedDiff.ID)) }) It("does not mark diff checked if block height is within reorg window", func() { From 1215dd9a17d6cf02c8c3f803f20b1183e9781010 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Tue, 18 Aug 2020 15:40:46 -0500 Subject: [PATCH 15/66] Send unrecognized diffs in GetNewDiffs --- libraries/shared/storage/diff_repository.go | 5 +++- .../shared/storage/diff_repository_test.go | 28 +++++++++---------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/libraries/shared/storage/diff_repository.go b/libraries/shared/storage/diff_repository.go index a1599a203..752aaaf89 100644 --- a/libraries/shared/storage/diff_repository.go +++ b/libraries/shared/storage/diff_repository.go @@ -68,7 +68,10 @@ func (repository diffRepository) CreateBackFilledStorageValue(rawDiff types.RawD func (repository diffRepository) GetNewDiffs(minID, limit int) ([]types.PersistedDiff, error) { var result []types.PersistedDiff - query := fmt.Sprintf("SELECT * FROM public.storage_diff WHERE status = 'new' and id > %d ORDER BY id ASC LIMIT %d", minID, limit) + query := fmt.Sprintf( + "SELECT * FROM public.storage_diff WHERE (status = 'new' OR status = 'unrecognized') AND id > %d ORDER BY id ASC LIMIT %d", + minID, limit, + ) err := repository.db.Select(&result, query) if err != nil { return nil, fmt.Errorf("error getting unchecked storage diffs with id greater than %d: %w", minID, err) diff --git a/libraries/shared/storage/diff_repository_test.go b/libraries/shared/storage/diff_repository_test.go index f945a8d1a..2d31b99d0 100644 --- a/libraries/shared/storage/diff_repository_test.go +++ b/libraries/shared/storage/diff_repository_test.go @@ -194,7 +194,7 @@ var _ = Describe("Storage diffs repository", func() { }) Describe("GetNewDiffs", func() { - It("sends diffs that are not marked as checked", func() { + It("sends diffs that are marked as 'new'", func() { fakePersistedDiff := types.PersistedDiff{ RawDiff: fakeStorageDiff, ID: rand.Int63(), @@ -209,29 +209,29 @@ var _ = Describe("Storage diffs repository", func() { Expect(diffs).To(ConsistOf(fakePersistedDiff)) }) - It("does not send diff that are marked as transformed", func() { - transformedPersistedDiff := types.PersistedDiff{ + It("sends diffs that are marked as 'unrecognized'", func() { + unrecognizedPersistedDiff := types.PersistedDiff{ RawDiff: fakeStorageDiff, ID: rand.Int63(), - Status: "transformed", + Status: "unrecognized", EthNodeID: db.NodeID, } - insertTestDiff(transformedPersistedDiff, db) + insertTestDiff(unrecognizedPersistedDiff, db) diffs, err := repo.GetNewDiffs(0, 1) Expect(err).NotTo(HaveOccurred()) - Expect(diffs).To(BeEmpty()) + Expect(diffs).To(ConsistOf(unrecognizedPersistedDiff)) }) - It("does not send diff that are marked as noncanonical", func() { - noncanonicalPersistedDiff := types.PersistedDiff{ + It("does not send diffs that are marked as transformed", func() { + transformedPersistedDiff := types.PersistedDiff{ RawDiff: fakeStorageDiff, ID: rand.Int63(), - Status: "noncanonical", + Status: "transformed", EthNodeID: db.NodeID, } - insertTestDiff(noncanonicalPersistedDiff, db) + insertTestDiff(transformedPersistedDiff, db) diffs, err := repo.GetNewDiffs(0, 1) @@ -239,14 +239,14 @@ var _ = Describe("Storage diffs repository", func() { Expect(diffs).To(BeEmpty()) }) - It("does not send diff that are marked as unrecognized", func() { - unrecognizedPersistedDiff := types.PersistedDiff{ + It("does not send diffs that are marked as noncanonical", func() { + noncanonicalPersistedDiff := types.PersistedDiff{ RawDiff: fakeStorageDiff, ID: rand.Int63(), - Status: "unrecognized", + Status: "noncanonical", EthNodeID: db.NodeID, } - insertTestDiff(unrecognizedPersistedDiff, db) + insertTestDiff(noncanonicalPersistedDiff, db) diffs, err := repo.GetNewDiffs(0, 1) From 4fe778b52f9e03da6f0e300f97638f4d9b36c984 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Tue, 18 Aug 2020 15:42:01 -0500 Subject: [PATCH 16/66] Add a partial index for unrecognized diffs --- db/migrations/00005_create_storage_diffs_table.sql | 2 ++ db/schema.sql | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/db/migrations/00005_create_storage_diffs_table.sql b/db/migrations/00005_create_storage_diffs_table.sql index 906c44c90..b702dc150 100644 --- a/db/migrations/00005_create_storage_diffs_table.sql +++ b/db/migrations/00005_create_storage_diffs_table.sql @@ -23,6 +23,8 @@ CREATE TABLE public.storage_diff CREATE INDEX storage_diff_new_status_index ON public.storage_diff (status) WHERE status = 'new'; +CREATE INDEX storage_diff_unrecognized_status_index + ON public.storage_diff (status) WHERE status = 'unrecognized'; CREATE INDEX storage_diff_eth_node ON public.storage_diff (eth_node_id); diff --git a/db/schema.sql b/db/schema.sql index 42a985bbb..4b546058d 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -789,6 +789,13 @@ CREATE INDEX storage_diff_eth_node ON public.storage_diff USING btree (eth_node_ CREATE INDEX storage_diff_new_status_index ON public.storage_diff USING btree (status) WHERE (status = 'new'::public.diff_status); +-- +-- Name: storage_diff_unrecognized_status_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX storage_diff_unrecognized_status_index ON public.storage_diff USING btree (status) WHERE (status = 'unrecognized'::public.diff_status); + + -- -- Name: transactions_header; Type: INDEX; Schema: public; Owner: - -- From 25fe903f85b17eed14e71313afb0190d2c160054 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Tue, 25 Aug 2020 10:29:58 -0500 Subject: [PATCH 17/66] Add PR feedback --- libraries/shared/storage/diff_repository.go | 24 ++++++++++++------- .../shared/storage/diff_repository_test.go | 24 +++++++++---------- libraries/shared/watcher/storage_watcher.go | 8 +++---- 3 files changed, 32 insertions(+), 24 deletions(-) diff --git a/libraries/shared/storage/diff_repository.go b/libraries/shared/storage/diff_repository.go index 752aaaf89..9dea9fc4a 100644 --- a/libraries/shared/storage/diff_repository.go +++ b/libraries/shared/storage/diff_repository.go @@ -34,6 +34,14 @@ type DiffRepository interface { GetFirstDiffIDForBlockHeight(blockHeight int64) (int64, error) } +var ( + New = `new` + Noncanonical = `noncanonical` + Transformed = `transformed` + Unrecognized = `unrecognized` + Unwatched = `unwatched` +) + type diffRepository struct { db *postgres.DB } @@ -68,11 +76,11 @@ func (repository diffRepository) CreateBackFilledStorageValue(rawDiff types.RawD func (repository diffRepository) GetNewDiffs(minID, limit int) ([]types.PersistedDiff, error) { var result []types.PersistedDiff - query := fmt.Sprintf( - "SELECT * FROM public.storage_diff WHERE (status = 'new' OR status = 'unrecognized') AND id > %d ORDER BY id ASC LIMIT %d", - minID, limit, + err := repository.db.Select( + &result, + `SELECT * FROM public.storage_diff WHERE (status = $1 OR status = $2) AND id > $3 ORDER BY id ASC LIMIT $4`, + New, Unrecognized, minID, limit, ) - err := repository.db.Select(&result, query) if err != nil { return nil, fmt.Errorf("error getting unchecked storage diffs with id greater than %d: %w", minID, err) } @@ -80,7 +88,7 @@ func (repository diffRepository) GetNewDiffs(minID, limit int) ([]types.Persiste } func (repository diffRepository) MarkTransformed(id int64) error { - _, err := repository.db.Exec(`UPDATE public.storage_diff SET status = 'transformed' WHERE id = $1`, id) + _, err := repository.db.Exec(`UPDATE public.storage_diff SET status = $1 WHERE id = $2`, Transformed, id) if err != nil { return fmt.Errorf("error marking diff %d transformed: %w", id, err) } @@ -88,7 +96,7 @@ func (repository diffRepository) MarkTransformed(id int64) error { } func (repository diffRepository) MarkUnrecognized(id int64) error { - _, err := repository.db.Exec(`UPDATE public.storage_diff SET status = 'unrecognized' WHERE id = $1`, id) + _, err := repository.db.Exec(`UPDATE public.storage_diff SET status = $1 WHERE id = $2`, Unrecognized, id) if err != nil { return fmt.Errorf("error marking diff %d checked: %w", id, err) } @@ -96,7 +104,7 @@ func (repository diffRepository) MarkUnrecognized(id int64) error { } func (repository diffRepository) MarkNoncanonical(id int64) error { - _, err := repository.db.Exec(`UPDATE public.storage_diff SET status = 'noncanonical' WHERE id = $1`, id) + _, err := repository.db.Exec(`UPDATE public.storage_diff SET status = $1 WHERE id = $2`, Noncanonical, id) if err != nil { return fmt.Errorf("error marking diff %d checked: %w", id, err) } @@ -104,7 +112,7 @@ func (repository diffRepository) MarkNoncanonical(id int64) error { } func (repository diffRepository) MarkUnwatched(id int64) error { - _, err := repository.db.Exec(`UPDATE public.storage_diff SET status = 'unwatched' WHERE id = $1`, id) + _, err := repository.db.Exec(`UPDATE public.storage_diff SET status = $1 WHERE id = $2`, Unwatched, id) if err != nil { return fmt.Errorf("error marking diff %d checked: %w", id, err) } diff --git a/libraries/shared/storage/diff_repository_test.go b/libraries/shared/storage/diff_repository_test.go index 2d31b99d0..6889230af 100644 --- a/libraries/shared/storage/diff_repository_test.go +++ b/libraries/shared/storage/diff_repository_test.go @@ -64,7 +64,7 @@ var _ = Describe("Storage diffs repository", func() { Expect(persisted.BlockHeight).To(Equal(fakeStorageDiff.BlockHeight)) Expect(persisted.StorageKey).To(Equal(fakeStorageDiff.StorageKey)) Expect(persisted.StorageValue).To(Equal(fakeStorageDiff.StorageValue)) - Expect(persisted.Status).To(Equal("new")) + Expect(persisted.Status).To(Equal(storage.New)) }) It("does not duplicate storage diffs", func() { @@ -199,7 +199,7 @@ var _ = Describe("Storage diffs repository", func() { RawDiff: fakeStorageDiff, ID: rand.Int63(), EthNodeID: db.NodeID, - Status: "new", + Status: storage.New, } insertTestDiff(fakePersistedDiff, db) @@ -213,7 +213,7 @@ var _ = Describe("Storage diffs repository", func() { unrecognizedPersistedDiff := types.PersistedDiff{ RawDiff: fakeStorageDiff, ID: rand.Int63(), - Status: "unrecognized", + Status: storage.Unrecognized, EthNodeID: db.NodeID, } insertTestDiff(unrecognizedPersistedDiff, db) @@ -228,7 +228,7 @@ var _ = Describe("Storage diffs repository", func() { transformedPersistedDiff := types.PersistedDiff{ RawDiff: fakeStorageDiff, ID: rand.Int63(), - Status: "transformed", + Status: storage.Transformed, EthNodeID: db.NodeID, } insertTestDiff(transformedPersistedDiff, db) @@ -243,7 +243,7 @@ var _ = Describe("Storage diffs repository", func() { noncanonicalPersistedDiff := types.PersistedDiff{ RawDiff: fakeStorageDiff, ID: rand.Int63(), - Status: "noncanonical", + Status: storage.Noncanonical, EthNodeID: db.NodeID, } insertTestDiff(noncanonicalPersistedDiff, db) @@ -267,7 +267,7 @@ var _ = Describe("Storage diffs repository", func() { persistedDiff := types.PersistedDiff{ RawDiff: fakeRawDiff, ID: rand.Int63(), - Status: "new", + Status: storage.New, EthNodeID: db.NodeID, } insertTestDiff(persistedDiff, db) @@ -292,7 +292,7 @@ var _ = Describe("Storage diffs repository", func() { fakePersistedDiff = types.PersistedDiff{ RawDiff: fakeStorageDiff, ID: rand.Int63(), - Status: "new", + Status: storage.New, EthNodeID: db.NodeID, } insertTestDiff(fakePersistedDiff, db) @@ -305,7 +305,7 @@ var _ = Describe("Storage diffs repository", func() { var status string getStatusErr := db.Get(&status, `SELECT status FROM public.storage_diff WHERE id = $1`, fakePersistedDiff.ID) Expect(getStatusErr).NotTo(HaveOccurred()) - Expect(status).To(Equal("transformed")) + Expect(status).To(Equal(storage.Transformed)) }) It("marks a diff as unrecognized", func() { @@ -315,7 +315,7 @@ var _ = Describe("Storage diffs repository", func() { var status string getStatusErr := db.Get(&status, `SELECT status FROM public.storage_diff WHERE id = $1`, fakePersistedDiff.ID) Expect(getStatusErr).NotTo(HaveOccurred()) - Expect(status).To(Equal("unrecognized")) + Expect(status).To(Equal(storage.Unrecognized)) }) It("marks a diff as noncanonical", func() { @@ -325,7 +325,7 @@ var _ = Describe("Storage diffs repository", func() { var status string getStatusErr := db.Get(&status, `SELECT status FROM public.storage_diff WHERE id = $1`, fakePersistedDiff.ID) Expect(getStatusErr).NotTo(HaveOccurred()) - Expect(status).To(Equal("noncanonical")) + Expect(status).To(Equal(storage.Noncanonical)) }) It("marks a diff as unwatched", func() { @@ -335,7 +335,7 @@ var _ = Describe("Storage diffs repository", func() { var status string getStatusErr := db.Get(&status, `SELECT status FROM public.storage_diff WHERE id = $1`, fakePersistedDiff.ID) Expect(getStatusErr).NotTo(HaveOccurred()) - Expect(status).To(Equal("unwatched")) + Expect(status).To(Equal(storage.Unwatched)) }) }) @@ -385,7 +385,7 @@ var _ = Describe("Storage diffs repository", func() { fakePersistedDiff := types.PersistedDiff{ RawDiff: fakeStorageDiff, ID: rand.Int63(), - Status: "transformed", + Status: storage.Transformed, EthNodeID: db.NodeID, } diff --git a/libraries/shared/watcher/storage_watcher.go b/libraries/shared/watcher/storage_watcher.go index 73f3c2ea6..dac9dcf20 100644 --- a/libraries/shared/watcher/storage_watcher.go +++ b/libraries/shared/watcher/storage_watcher.go @@ -119,12 +119,12 @@ func (watcher StorageWatcher) transformDiffs() error { for { diffs, extractErr := watcher.StorageDiffRepository.GetNewDiffs(minID, ResultsLimit) if extractErr != nil { - return fmt.Errorf("error getting unchecked diffs: %w", extractErr) + return fmt.Errorf("error getting new diffs: %w", extractErr) } for _, diff := range diffs { transformErr := watcher.transformDiff(diff) if handleErr := watcher.handleTransformError(transformErr, diff); handleErr != nil { - return handleErr + return fmt.Errorf("error transforming diff: %w", handleErr) } } lenDiffs := len(diffs) @@ -142,7 +142,7 @@ func (watcher StorageWatcher) transformDiff(diff types.PersistedDiff) error { if !watching { markUnwatchedErr := watcher.StorageDiffRepository.MarkUnwatched(diff.ID) if markUnwatchedErr != nil { - return fmt.Errorf("error marking diff unwatched: %w", markUnwatchedErr) + return fmt.Errorf("error marking diff %s: %w", storage.Unwatched, markUnwatchedErr) } return nil } @@ -163,7 +163,7 @@ func (watcher StorageWatcher) transformDiff(diff types.PersistedDiff) error { markTransformedErr := watcher.StorageDiffRepository.MarkTransformed(diff.ID) if markTransformedErr != nil { - return fmt.Errorf("error marking diff transformed: %w", markTransformedErr) + return fmt.Errorf("error marking diff %s: %w", storage.Transformed, markTransformedErr) } return nil From 728a6405a368cbe90be0c7091ce147704940ab65 Mon Sep 17 00:00:00 2001 From: Rob Mulholand Date: Mon, 17 Aug 2020 10:39:28 -0500 Subject: [PATCH 18/66] Update primary key for addresses to bigserial - enable adding more addresses --- db/migrations/00002_add_address_table.sql | 2 +- db/migrations/00007_create_receipts_table.sql | 2 +- db/migrations/00008_create_event_logs_table.sql | 2 +- .../00011_create_create_back_filled_diff_function.sql | 3 ++- db/schema.sql | 7 +++---- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/db/migrations/00002_add_address_table.sql b/db/migrations/00002_add_address_table.sql index 6866f6e1d..642533c1f 100644 --- a/db/migrations/00002_add_address_table.sql +++ b/db/migrations/00002_add_address_table.sql @@ -1,7 +1,7 @@ -- +goose Up CREATE TABLE public.addresses ( - id SERIAL PRIMARY KEY, + id BIGSERIAL PRIMARY KEY, address character varying(42), hashed_address character varying(66), UNIQUE (address) diff --git a/db/migrations/00007_create_receipts_table.sql b/db/migrations/00007_create_receipts_table.sql index bc57f10f9..200ef2398 100644 --- a/db/migrations/00007_create_receipts_table.sql +++ b/db/migrations/00007_create_receipts_table.sql @@ -4,7 +4,7 @@ CREATE TABLE public.receipts id SERIAL PRIMARY KEY, transaction_id INTEGER NOT NULL REFERENCES public.transactions (id) ON DELETE CASCADE, header_id INTEGER NOT NULL REFERENCES public.headers (id) ON DELETE CASCADE, - contract_address_id INTEGER NOT NULL REFERENCES public.addresses (id) ON DELETE CASCADE, + contract_address_id BIGINT NOT NULL REFERENCES public.addresses (id) ON DELETE CASCADE, cumulative_gas_used NUMERIC, gas_used NUMERIC, state_root VARCHAR(66), diff --git a/db/migrations/00008_create_event_logs_table.sql b/db/migrations/00008_create_event_logs_table.sql index 9cf50b9c7..7185b27bc 100644 --- a/db/migrations/00008_create_event_logs_table.sql +++ b/db/migrations/00008_create_event_logs_table.sql @@ -3,7 +3,7 @@ CREATE TABLE public.event_logs ( id SERIAL PRIMARY KEY, header_id INTEGER NOT NULL REFERENCES public.headers (id) ON DELETE CASCADE, - address INTEGER NOT NULL REFERENCES public.addresses (id) ON DELETE CASCADE, + address BIGINT NOT NULL REFERENCES public.addresses (id) ON DELETE CASCADE, topics BYTEA[], data BYTEA, block_number BIGINT, diff --git a/db/migrations/00011_create_create_back_filled_diff_function.sql b/db/migrations/00011_create_create_back_filled_diff_function.sql index 411b14ab6..98855b221 100644 --- a/db/migrations/00011_create_create_back_filled_diff_function.sql +++ b/db/migrations/00011_create_create_back_filled_diff_function.sql @@ -1,7 +1,8 @@ -- +goose Up -- +goose StatementBegin CREATE OR REPLACE FUNCTION public.create_back_filled_diff(block_height BIGINT, block_hash BYTEA, hashed_address BYTEA, - storage_key BYTEA, storage_value BYTEA, eth_node_id INTEGER) RETURNS VOID AS + storage_key BYTEA, storage_value BYTEA, + eth_node_id INTEGER) RETURNS VOID AS $$ DECLARE last_storage_value BYTEA := ( diff --git a/db/schema.sql b/db/schema.sql index 4b546058d..234726ffc 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -155,7 +155,7 @@ SET default_with_oids = false; -- CREATE TABLE public.addresses ( - id integer NOT NULL, + id bigint NOT NULL, address character varying(42), hashed_address character varying(66) ); @@ -166,7 +166,6 @@ CREATE TABLE public.addresses ( -- CREATE SEQUENCE public.addresses_id_seq - AS integer START WITH 1 INCREMENT BY 1 NO MINVALUE @@ -251,7 +250,7 @@ ALTER SEQUENCE public.eth_nodes_id_seq OWNED BY public.eth_nodes.id; CREATE TABLE public.event_logs ( id integer NOT NULL, header_id integer NOT NULL, - address integer NOT NULL, + address bigint NOT NULL, topics bytea[], data bytea, block_number bigint, @@ -361,7 +360,7 @@ CREATE TABLE public.receipts ( id integer NOT NULL, transaction_id integer NOT NULL, header_id integer NOT NULL, - contract_address_id integer NOT NULL, + contract_address_id bigint NOT NULL, cumulative_gas_used numeric, gas_used numeric, state_root character varying(66), From 806fda8253712f8881e46028f203816ccc2c6968 Mon Sep 17 00:00:00 2001 From: Rob Mulholand Date: Tue, 18 Aug 2020 13:42:41 -0500 Subject: [PATCH 19/66] Update primary key for events to bigserial - enable adding more events --- db/migrations/00008_create_event_logs_table.sql | 2 +- db/schema.sql | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/db/migrations/00008_create_event_logs_table.sql b/db/migrations/00008_create_event_logs_table.sql index 7185b27bc..ad5c77253 100644 --- a/db/migrations/00008_create_event_logs_table.sql +++ b/db/migrations/00008_create_event_logs_table.sql @@ -1,7 +1,7 @@ -- +goose Up CREATE TABLE public.event_logs ( - id SERIAL PRIMARY KEY, + id BIGSERIAL PRIMARY KEY, header_id INTEGER NOT NULL REFERENCES public.headers (id) ON DELETE CASCADE, address BIGINT NOT NULL REFERENCES public.addresses (id) ON DELETE CASCADE, topics BYTEA[], diff --git a/db/schema.sql b/db/schema.sql index 234726ffc..e60a0736d 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -248,7 +248,7 @@ ALTER SEQUENCE public.eth_nodes_id_seq OWNED BY public.eth_nodes.id; -- CREATE TABLE public.event_logs ( - id integer NOT NULL, + id bigint NOT NULL, header_id integer NOT NULL, address bigint NOT NULL, topics bytea[], @@ -268,7 +268,6 @@ CREATE TABLE public.event_logs ( -- CREATE SEQUENCE public.event_logs_id_seq - AS integer START WITH 1 INCREMENT BY 1 NO MINVALUE From a9c90183699b0eaf4ad7aeca8082c80b78a85945 Mon Sep 17 00:00:00 2001 From: Rob Mulholand Date: Fri, 14 Aug 2020 14:26:09 -0500 Subject: [PATCH 20/66] Chunk headers in backfill events process - Don't load all headers in range into memory; chunk into manageable quantity (currently 1000 headers per lookup) --- libraries/shared/logs/extractor.go | 60 ++++++++++-- libraries/shared/logs/extractor_test.go | 98 ++++++++++++++----- .../backfill/storage_value_loader_test.go | 4 +- pkg/fakes/mock_header_repository.go | 8 +- 4 files changed, 127 insertions(+), 43 deletions(-) diff --git a/libraries/shared/logs/extractor.go b/libraries/shared/logs/extractor.go index 9c8e75ca9..ec9d9227a 100644 --- a/libraries/shared/logs/extractor.go +++ b/libraries/shared/logs/extractor.go @@ -32,9 +32,14 @@ import ( "github.com/sirupsen/logrus" ) +type BlockIdentifier string + var ( - ErrNoUncheckedHeaders = errors.New("no unchecked headers available for log fetching") - ErrNoWatchedAddresses = errors.New("no watched addresses configured in the log extractor") + StartInterval BlockIdentifier = "start" + EndInterval BlockIdentifier = "end" + ErrNoUncheckedHeaders = errors.New("no unchecked headers available for log fetching") + ErrNoWatchedAddresses = errors.New("no watched addresses configured in the log extractor") + HeaderChunkSize int64 = 1000 ) type ILogExtractor interface { @@ -152,21 +157,56 @@ func (extractor LogExtractor) BackFillLogs(endingBlock int64) error { return fmt.Errorf("error extracting logs: %w", ErrNoWatchedAddresses) } - headers, headersErr := extractor.HeaderRepository.GetHeadersInRange(*extractor.StartingBlock, endingBlock) - if headersErr != nil { - logrus.Errorf("error fetching missing headers: %s", headersErr) - return fmt.Errorf("error getting unchecked headers to check for logs: %w", headersErr) + ranges, chunkErr := ChunkRanges(*extractor.StartingBlock, endingBlock, HeaderChunkSize) + if chunkErr != nil { + return fmt.Errorf("error chunking headers to lookup in logs backfill: %w", chunkErr) } - for _, header := range headers { - err := extractor.fetchAndPersistLogsForHeader(header) - if err != nil { - return fmt.Errorf("error fetching and persisting logs for header with id %d: %w", header.Id, err) + for _, r := range ranges { + headers, headersErr := extractor.HeaderRepository.GetHeadersInRange(r[StartInterval], r[EndInterval]) + if headersErr != nil { + logrus.Errorf("error fetching missing headers: %s", headersErr) + return fmt.Errorf("error getting unchecked headers to check for logs: %w", headersErr) + } + + for _, header := range headers { + err := extractor.fetchAndPersistLogsForHeader(header) + if err != nil { + return fmt.Errorf("error fetching and persisting logs for header with id %d: %w", header.Id, err) + } } } + return nil } +func ChunkRanges(startingBlock, endingBlock, interval int64) ([]map[BlockIdentifier]int64, error) { + if endingBlock <= startingBlock { + return nil, errors.New("ending block for backfill not > starting block") + } + + totalLength := endingBlock - startingBlock + numIntervals := totalLength / interval + if totalLength%interval != 0 { + numIntervals++ + } + + results := make([]map[BlockIdentifier]int64, numIntervals) + for i := int64(0); i < numIntervals; i++ { + nextStartBlock := startingBlock + i*interval + nextEndingBlock := nextStartBlock + interval - 1 + if nextEndingBlock > endingBlock { + nextEndingBlock = endingBlock + } + nextInterval := map[BlockIdentifier]int64{ + StartInterval: nextStartBlock, + EndInterval: nextEndingBlock, + } + results[i] = nextInterval + } + return results, nil +} + func logError(description string, err error, header core.Header) { logrus.WithFields(logrus.Fields{ "headerId": header.Id, diff --git a/libraries/shared/logs/extractor_test.go b/libraries/shared/logs/extractor_test.go index 9bfb6ad5d..b63d9e07a 100644 --- a/libraries/shared/logs/extractor_test.go +++ b/libraries/shared/logs/extractor_test.go @@ -393,29 +393,43 @@ var _ = Describe("Log extractor", func() { }) It("gets headers from transformer's starting block through passed ending block", func() { - fakeConfig := event.TransformerConfig{ - ContractAddresses: []string{fakes.FakeAddress.Hex()}, - Topic: fakes.FakeHash.Hex(), - StartingBlockNumber: rand.Int63(), - } - extractor.AddTransformerConfig(fakeConfig) + startingBlock := addTransformerConfig(extractor) mockHeaderRepository := &fakes.MockHeaderRepository{} extractor.HeaderRepository = mockHeaderRepository - endingBlock := rand.Int63() + endingBlock := startingBlock + 1 _ = extractor.BackFillLogs(endingBlock) - Expect(mockHeaderRepository.GetHeadersInRangeStartingBlock).To(Equal(fakeConfig.StartingBlockNumber)) - Expect(mockHeaderRepository.GetHeadersInRangeEndingBlock).To(Equal(endingBlock)) + Expect(mockHeaderRepository.GetHeadersInRangeStartingBlocks).To(ContainElement(startingBlock)) + Expect(mockHeaderRepository.GetHeadersInRangeEndingBlocks).To(ContainElement(endingBlock)) + }) + + It("chunks headers if range greater than interval defined by HeaderChunkSize", func() { + startingBlock := addTransformerConfig(extractor) + mockHeaderRepository := &fakes.MockHeaderRepository{} + extractor.HeaderRepository = mockHeaderRepository + endingBlock := startingBlock + logs.HeaderChunkSize*2 + + err := extractor.BackFillLogs(endingBlock) + + Expect(err).NotTo(HaveOccurred()) + Expect(mockHeaderRepository.GetHeadersInRangeStartingBlocks).To(ConsistOf([]int64{ + startingBlock, + startingBlock + logs.HeaderChunkSize, + })) + Expect(mockHeaderRepository.GetHeadersInRangeEndingBlocks).To(ConsistOf([]int64{ + startingBlock + logs.HeaderChunkSize - 1, + startingBlock + logs.HeaderChunkSize*2 - 1, + })) }) It("returns error if getting headers in range returns error", func() { mockHeaderRepository := &fakes.MockHeaderRepository{} mockHeaderRepository.GetHeadersInRangeError = fakes.FakeError extractor.HeaderRepository = mockHeaderRepository - addTransformerConfig(extractor) + startingBlock := addTransformerConfig(extractor) - err := extractor.BackFillLogs(0) + err := extractor.BackFillLogs(startingBlock + 1) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) @@ -424,11 +438,11 @@ var _ = Describe("Log extractor", func() { It("does nothing if no headers found", func() { mockHeaderRepository := &fakes.MockHeaderRepository{} extractor.HeaderRepository = mockHeaderRepository - addTransformerConfig(extractor) + startingBlock := addTransformerConfig(extractor) mockLogFetcher := &mocks.MockLogFetcher{} extractor.Fetcher = mockLogFetcher - _ = extractor.BackFillLogs(0) + _ = extractor.BackFillLogs(startingBlock + 1) Expect(mockLogFetcher.FetchCalled).To(BeFalse()) }) @@ -445,7 +459,7 @@ var _ = Describe("Log extractor", func() { mockLogFetcher := &mocks.MockLogFetcher{} extractor.Fetcher = mockLogFetcher - err := extractor.BackFillLogs(0) + err := extractor.BackFillLogs(config.StartingBlockNumber + 1) Expect(err).NotTo(HaveOccurred()) Expect(mockLogFetcher.FetchCalled).To(BeTrue()) @@ -457,12 +471,12 @@ var _ = Describe("Log extractor", func() { It("returns error if fetching logs fails", func() { addHeaderInRange(extractor) - addTransformerConfig(extractor) + startingBlock := addTransformerConfig(extractor) mockLogFetcher := &mocks.MockLogFetcher{} mockLogFetcher.ReturnError = fakes.FakeError extractor.Fetcher = mockLogFetcher - err := extractor.BackFillLogs(0) + err := extractor.BackFillLogs(startingBlock + 1) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) @@ -470,11 +484,11 @@ var _ = Describe("Log extractor", func() { It("does not sync transactions when no logs", func() { addHeaderInRange(extractor) - addTransformerConfig(extractor) + startingBlock := addTransformerConfig(extractor) mockTransactionSyncer := &fakes.MockTransactionSyncer{} extractor.Syncer = mockTransactionSyncer - err := extractor.BackFillLogs(0) + err := extractor.BackFillLogs(startingBlock + 1) Expect(err).NotTo(HaveOccurred()) Expect(mockTransactionSyncer.SyncTransactionsCalled).To(BeFalse()) @@ -484,11 +498,11 @@ var _ = Describe("Log extractor", func() { It("syncs transactions", func() { addHeaderInRange(extractor) addFetchedLog(extractor) - addTransformerConfig(extractor) + startingBlock := addTransformerConfig(extractor) mockTransactionSyncer := &fakes.MockTransactionSyncer{} extractor.Syncer = mockTransactionSyncer - err := extractor.BackFillLogs(0) + err := extractor.BackFillLogs(startingBlock + 1) Expect(err).NotTo(HaveOccurred()) Expect(mockTransactionSyncer.SyncTransactionsCalled).To(BeTrue()) @@ -497,12 +511,12 @@ var _ = Describe("Log extractor", func() { It("returns error if syncing transactions fails", func() { addHeaderInRange(extractor) addFetchedLog(extractor) - addTransformerConfig(extractor) + startingBlock := addTransformerConfig(extractor) mockTransactionSyncer := &fakes.MockTransactionSyncer{} mockTransactionSyncer.SyncTransactionsError = fakes.FakeError extractor.Syncer = mockTransactionSyncer - err := extractor.BackFillLogs(0) + err := extractor.BackFillLogs(startingBlock + 1) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) @@ -510,7 +524,7 @@ var _ = Describe("Log extractor", func() { It("persists fetched logs", func() { addHeaderInRange(extractor) - addTransformerConfig(extractor) + startingBlock := addTransformerConfig(extractor) fakeLogs := []types.Log{{ Address: common.HexToAddress("0xA"), Topics: []common.Hash{common.HexToHash("0xA")}, @@ -522,7 +536,7 @@ var _ = Describe("Log extractor", func() { mockLogRepository := &fakes.MockEventLogRepository{} extractor.LogRepository = mockLogRepository - err := extractor.BackFillLogs(0) + err := extractor.BackFillLogs(startingBlock + 1) Expect(err).NotTo(HaveOccurred()) Expect(mockLogRepository.PassedLogs).To(Equal(fakeLogs)) @@ -531,27 +545,57 @@ var _ = Describe("Log extractor", func() { It("returns error if persisting logs fails", func() { addHeaderInRange(extractor) addFetchedLog(extractor) - addTransformerConfig(extractor) + startingBlock := addTransformerConfig(extractor) mockLogRepository := &fakes.MockEventLogRepository{} mockLogRepository.CreateError = fakes.FakeError extractor.LogRepository = mockLogRepository - err := extractor.BackFillLogs(0) + err := extractor.BackFillLogs(startingBlock + 1) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) }) }) }) + + Describe("ChunkRanges", func() { + It("returns error if upper bound <= lower bound", func() { + _, err := logs.ChunkRanges(10, 10, 1) + + Expect(err).To(HaveOccurred()) + }) + + It("chunks items in range by interval", func() { + res, err := logs.ChunkRanges(1, 20, 10) + + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(ConsistOf([]map[logs.BlockIdentifier]int64{ + {logs.StartInterval: 1, logs.EndInterval: 10}, + {logs.StartInterval: 11, logs.EndInterval: 20}, + })) + }) + + It("truncates final chunk to end at upper bound", func() { + res, err := logs.ChunkRanges(1, 25, 10) + + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(ConsistOf([]map[logs.BlockIdentifier]int64{ + {logs.StartInterval: 1, logs.EndInterval: 10}, + {logs.StartInterval: 11, logs.EndInterval: 20}, + {logs.StartInterval: 21, logs.EndInterval: 25}, + })) + }) + }) }) -func addTransformerConfig(extractor *logs.LogExtractor) { +func addTransformerConfig(extractor *logs.LogExtractor) int64 { fakeConfig := event.TransformerConfig{ ContractAddresses: []string{fakes.FakeAddress.Hex()}, Topic: fakes.FakeHash.Hex(), StartingBlockNumber: rand.Int63(), } extractor.AddTransformerConfig(fakeConfig) + return fakeConfig.StartingBlockNumber } func addUncheckedHeader(extractor *logs.LogExtractor) { diff --git a/libraries/shared/storage/backfill/storage_value_loader_test.go b/libraries/shared/storage/backfill/storage_value_loader_test.go index c8699c731..f91b4f51d 100644 --- a/libraries/shared/storage/backfill/storage_value_loader_test.go +++ b/libraries/shared/storage/backfill/storage_value_loader_test.go @@ -123,8 +123,8 @@ var _ = Describe("StorageValueLoader", func() { It("fetches headers in the given block range", func() { runnerErr := runner.Run() Expect(runnerErr).NotTo(HaveOccurred()) - Expect(headerRepo.GetHeadersInRangeStartingBlock).To(Equal(blockOne)) - Expect(headerRepo.GetHeadersInRangeEndingBlock).To(Equal(blockTwo)) + Expect(headerRepo.GetHeadersInRangeStartingBlocks).To(ConsistOf(blockOne)) + Expect(headerRepo.GetHeadersInRangeEndingBlocks).To(ConsistOf(blockTwo)) }) It("returns an error if a header for the given block cannot be retrieved", func() { diff --git a/pkg/fakes/mock_header_repository.go b/pkg/fakes/mock_header_repository.go index 50f9e6443..0d934e4ab 100644 --- a/pkg/fakes/mock_header_repository.go +++ b/pkg/fakes/mock_header_repository.go @@ -32,9 +32,9 @@ type MockHeaderRepository struct { GetHeaderByIDError error GetHeaderByIDHeaderToReturn core.Header GetHeaderPassedBlockNumber int64 - GetHeadersInRangeEndingBlock int64 + GetHeadersInRangeEndingBlocks []int64 GetHeadersInRangeError error - GetHeadersInRangeStartingBlock int64 + GetHeadersInRangeStartingBlocks []int64 MostRecentHeaderBlockNumber int64 MostRecentHeaderBlockNumberErr error createOrUpdateHeaderCallCount int @@ -90,8 +90,8 @@ func (mock *MockHeaderRepository) GetHeaderByID(id int64) (core.Header, error) { } func (mock *MockHeaderRepository) GetHeadersInRange(startingBlock, endingBlock int64) ([]core.Header, error) { - mock.GetHeadersInRangeStartingBlock = startingBlock - mock.GetHeadersInRangeEndingBlock = endingBlock + mock.GetHeadersInRangeStartingBlocks = append(mock.GetHeadersInRangeStartingBlocks, startingBlock) + mock.GetHeadersInRangeEndingBlocks = append(mock.GetHeadersInRangeEndingBlocks, endingBlock) return mock.AllHeaders, mock.GetHeadersInRangeError } From 9a0dc6298e898744d062856c6ea4de7bf53b9ffc Mon Sep 17 00:00:00 2001 From: Andrew J Yao <1547809+yaoandrew@users.noreply.github.com> Date: Tue, 4 Aug 2020 16:55:18 -0700 Subject: [PATCH 21/66] Remove hashed address --- db/migrations/00002_add_address_table.sql | 3 +- .../00005_create_storage_diffs_table.sql | 1 - ...reate_create_back_filled_diff_function.sql | 1 - db/schema.sql | 6 +- go.mod | 31 +------ go.sum | 2 + .../shared/factories/storage/transformer.go | 11 --- .../factories/storage/transformer_test.go | 28 +++---- libraries/shared/mocks/storage_transformer.go | 5 -- .../shared/repository/address_repository.go | 9 +-- .../repository/address_repository_test.go | 16 ++-- .../storage/backfill/storage_value_loader.go | 16 ++-- .../backfill/storage_value_loader_test.go | 45 +++++++---- .../shared/storage/diff_repository_test.go | 80 ++++++++++++++----- libraries/shared/storage/extractor.go | 4 +- libraries/shared/storage/extractor_test.go | 9 +-- .../fetcher/geth_rpc_storage_fetcher_test.go | 27 +++---- libraries/shared/storage/types/diff.go | 29 ++++--- libraries/shared/storage/types/diff_test.go | 5 +- libraries/shared/watcher/storage_watcher.go | 8 +- .../shared/watcher/storage_watcher_test.go | 44 +++++----- 21 files changed, 180 insertions(+), 200 deletions(-) diff --git a/db/migrations/00002_add_address_table.sql b/db/migrations/00002_add_address_table.sql index 642533c1f..d644b95e7 100644 --- a/db/migrations/00002_add_address_table.sql +++ b/db/migrations/00002_add_address_table.sql @@ -3,9 +3,8 @@ CREATE TABLE public.addresses ( id BIGSERIAL PRIMARY KEY, address character varying(42), - hashed_address character varying(66), UNIQUE (address) ); -- +goose Down -DROP TABLE public.addresses; \ No newline at end of file +DROP TABLE public.addresses; diff --git a/db/migrations/00005_create_storage_diffs_table.sql b/db/migrations/00005_create_storage_diffs_table.sql index b702dc150..0480bd697 100644 --- a/db/migrations/00005_create_storage_diffs_table.sql +++ b/db/migrations/00005_create_storage_diffs_table.sql @@ -12,7 +12,6 @@ CREATE TABLE public.storage_diff id BIGSERIAL PRIMARY KEY, block_height BIGINT, block_hash BYTEA, - hashed_address BYTEA, storage_key BYTEA, storage_value BYTEA, eth_node_id INTEGER NOT NULL REFERENCES public.eth_nodes (id) ON DELETE CASCADE, diff --git a/db/migrations/00011_create_create_back_filled_diff_function.sql b/db/migrations/00011_create_create_back_filled_diff_function.sql index 98855b221..d2162e318 100644 --- a/db/migrations/00011_create_create_back_filled_diff_function.sql +++ b/db/migrations/00011_create_create_back_filled_diff_function.sql @@ -9,7 +9,6 @@ DECLARE SELECT storage_diff.storage_value FROM public.storage_diff WHERE storage_diff.block_height <= create_back_filled_diff.block_height - AND storage_diff.hashed_address = create_back_filled_diff.hashed_address AND storage_diff.storage_key = create_back_filled_diff.storage_key ORDER BY storage_diff.block_height DESC LIMIT 1 diff --git a/db/schema.sql b/db/schema.sql index e60a0736d..bf7188504 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -41,7 +41,6 @@ DECLARE SELECT storage_diff.storage_value FROM public.storage_diff WHERE storage_diff.block_height <= create_back_filled_diff.block_height - AND storage_diff.hashed_address = create_back_filled_diff.hashed_address AND storage_diff.storage_key = create_back_filled_diff.storage_key ORDER BY storage_diff.block_height DESC LIMIT 1 @@ -397,7 +396,6 @@ CREATE TABLE public.storage_diff ( id bigint NOT NULL, block_height bigint, block_hash bytea, - hashed_address bytea, storage_key bytea, storage_value bytea, eth_node_id integer NOT NULL, @@ -671,11 +669,11 @@ ALTER TABLE ONLY public.receipts -- --- Name: storage_diff storage_diff_block_height_block_hash_hashed_address_storage_key; Type: CONSTRAINT; Schema: public; Owner: - +-- Name: storage_diff storage_diff_block_height_block_hash_storage_key_storage_va_key; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY public.storage_diff - ADD CONSTRAINT storage_diff_block_height_block_hash_hashed_address_storage_key UNIQUE (block_height, block_hash, hashed_address, storage_key, storage_value); + ADD CONSTRAINT storage_diff_block_height_block_hash_storage_key_storage_va_key UNIQUE (block_height, block_hash, storage_key, storage_value); -- diff --git a/go.mod b/go.mod index 17417fa18..a87ff2cc8 100644 --- a/go.mod +++ b/go.mod @@ -3,49 +3,22 @@ module github.com/makerdao/vulcanizedb go 1.12 require ( - github.com/allegro/bigcache v1.2.1 // indirect - github.com/apilayer/freegeoip v3.5.0+incompatible // indirect - github.com/aristanetworks/goarista v0.0.0-20191106175434-873d404c7f40 // indirect - github.com/btcsuite/btcd v0.20.1-beta // indirect - github.com/cespare/cp v1.1.1 // indirect github.com/dave/jennifer v1.3.0 - github.com/deckarep/golang-set v1.7.1 // indirect - github.com/docker/docker v1.13.1 // indirect - github.com/edsrzf/mmap-go v1.0.0 // indirect - github.com/elastic/gosigar v0.10.5 // indirect - github.com/ethereum/go-ethereum v1.9.6 - github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect - github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 // indirect - github.com/gorilla/websocket v1.4.1 // indirect + github.com/ethereum/go-ethereum v1.9.8 github.com/hashicorp/golang-lru v0.5.4 github.com/hpcloud/tail v1.0.0 - github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/influxdata/influxdb v1.7.9 // indirect - github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jmoiron/sqlx v0.0.0-20181024163419-82935fac6c1a - github.com/karalabe/usb v0.0.0-20191104083709-911d15fe12a9 // indirect github.com/lib/pq v1.0.0 - github.com/mattn/go-colorable v0.1.4 // indirect - github.com/mattn/go-isatty v0.0.10 // indirect + github.com/makerdao/vdb-mcd-transformers v0.2.17 github.com/mitchellh/go-homedir v1.1.0 - github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect - github.com/olekukonko/tablewriter v0.0.3 // indirect github.com/onsi/ginkgo v1.10.1 github.com/onsi/gomega v1.10.0 - github.com/oschwald/maxminddb-golang v1.5.0 // indirect - github.com/pborman/uuid v1.2.0 // indirect github.com/pressly/goose v2.6.0+incompatible - github.com/prometheus/tsdb v0.10.0 // indirect - github.com/rjeczalik/notify v0.9.2 // indirect - github.com/rs/cors v1.7.0 // indirect github.com/sirupsen/logrus v1.2.0 github.com/spf13/cobra v0.0.3 github.com/spf13/viper v1.3.2 - github.com/status-im/keycard-go v0.0.0-20191119114148-6dd40a46baa0 // indirect - github.com/tyler-smith/go-bip39 v1.0.2 // indirect golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0 golang.org/x/sync v0.0.0-20190423024810-112230192c58 - google.golang.org/appengine v1.6.5 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 ) diff --git a/go.sum b/go.sum index 709b5d2a9..02e64238b 100644 --- a/go.sum +++ b/go.sum @@ -223,6 +223,8 @@ github.com/makerdao/go-ethereum v1.9.11-rc2 h1:ZjxmFdkar8P43EkK17ni8y3vXxHHFZjOd github.com/makerdao/go-ethereum v1.9.11-rc2/go.mod h1:7oC0Ni6dosMv5pxMigm6s0hN8g4haJMBnqmmo0D9YfQ= github.com/makerdao/go-ethereum v1.9.15-statechange-filter h1:jLX9CleT+i/gUn5LLTUoor1NXYLHSTnZM1pOFKrg2IA= github.com/makerdao/go-ethereum v1.9.15-statechange-filter/go.mod h1:slT8bPPRhXsyNTwHQxrOnjuTZ1sDXRajW11EkJ84QJ0= +github.com/makerdao/vdb-mcd-transformers v0.2.17/go.mod h1:lon896AMm/ZzNGu4n1Cfa1XW2V3C1jtHs13VRvTHKrg= +github.com/makerdao/vulcanizedb v0.0.11-rc.1/go.mod h1:FZZ/SKy+7Ejdc+4Wa5oTGrr2cFuaCgGBxc2xoifaEoY= github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= diff --git a/libraries/shared/factories/storage/transformer.go b/libraries/shared/factories/storage/transformer.go index 59088e424..79ff8295c 100644 --- a/libraries/shared/factories/storage/transformer.go +++ b/libraries/shared/factories/storage/transformer.go @@ -27,7 +27,6 @@ import ( type ITransformer interface { Execute(diff types.PersistedDiff) error - KeccakContractAddress() common.Hash GetStorageKeysLookup() KeysLookup GetContractAddress() common.Address } @@ -55,16 +54,6 @@ func (transformer Transformer) NewTransformer(db *postgres.DB) ITransformer { return &transformer } -func (transformer *Transformer) KeccakContractAddress() common.Hash { - emptyHash := common.Hash{} - if transformer.hashedAddress == emptyHash { - transformer.hashedAddress = types.HexToKeccak256Hash(transformer.Address.Hex()) - return transformer.hashedAddress - } - - return transformer.hashedAddress -} - func (transformer Transformer) Execute(diff types.PersistedDiff) error { metadata, lookupErr := transformer.StorageKeysLookup.Lookup(diff.StorageKey) if lookupErr != nil { diff --git a/libraries/shared/factories/storage/transformer_test.go b/libraries/shared/factories/storage/transformer_test.go index 8b857e263..b10939915 100644 --- a/libraries/shared/factories/storage/transformer_test.go +++ b/libraries/shared/factories/storage/transformer_test.go @@ -45,14 +45,6 @@ var _ = Describe("Storage transformer", func() { } }) - It("returns the keccaked contract address being watched", func() { - fakeAddress := fakes.FakeAddress - keccakOfAddress := types.HexToKeccak256Hash(fakeAddress.Hex()) - t.Address = fakeAddress - - Expect(t.KeccakContractAddress()).To(Equal(keccakOfAddress)) - }) - It("returns the contract address being watched", func() { fakeAddress := fakes.FakeAddress t.Address = fakeAddress @@ -86,11 +78,11 @@ var _ = Describe("Storage transformer", func() { ID: rand.Int63(), HeaderID: fakeHeaderID, RawDiff: types.RawDiff{ - HashedAddress: common.Hash{}, - BlockHash: common.HexToHash(fakeBlockHash), - BlockHeight: fakeBlockNumber, - StorageKey: common.Hash{}, - StorageValue: rawValue.Hash(), + Address: fakes.FakeAddress, + BlockHash: common.HexToHash(fakeBlockHash), + BlockHeight: fakeBlockNumber, + StorageKey: common.Hash{}, + StorageValue: rawValue.Hash(), }, } @@ -140,11 +132,11 @@ var _ = Describe("Storage transformer", func() { ID: rand.Int63(), HeaderID: fakeHeaderID, RawDiff: types.RawDiff{ - HashedAddress: common.Hash{}, - BlockHash: common.HexToHash(fakeBlockHash), - BlockHeight: fakeBlockNumber, - StorageKey: common.Hash{}, - StorageValue: rawValue.Hash(), + Address: fakes.FakeAddress, + BlockHash: common.HexToHash(fakeBlockHash), + BlockHeight: fakeBlockNumber, + StorageKey: common.Hash{}, + StorageValue: rawValue.Hash(), }, } diff --git a/libraries/shared/mocks/storage_transformer.go b/libraries/shared/mocks/storage_transformer.go index d2db218a7..fe698bef9 100644 --- a/libraries/shared/mocks/storage_transformer.go +++ b/libraries/shared/mocks/storage_transformer.go @@ -26,7 +26,6 @@ import ( type MockStorageTransformer struct { Address common.Address StorageKeysLookup storage.KeysLookup - KeccakOfAddress common.Hash ExecuteErr error PassedDiff types.PersistedDiff } @@ -36,10 +35,6 @@ func (transformer *MockStorageTransformer) Execute(diff types.PersistedDiff) err return transformer.ExecuteErr } -func (transformer *MockStorageTransformer) KeccakContractAddress() common.Hash { - return transformer.KeccakOfAddress -} - func (transformer *MockStorageTransformer) GetContractAddress() common.Address { return transformer.Address } diff --git a/libraries/shared/repository/address_repository.go b/libraries/shared/repository/address_repository.go index af630c393..d94cc15dd 100644 --- a/libraries/shared/repository/address_repository.go +++ b/libraries/shared/repository/address_repository.go @@ -32,12 +32,11 @@ package repository import ( "github.com/ethereum/go-ethereum/common" "github.com/jmoiron/sqlx" - "github.com/makerdao/vulcanizedb/libraries/shared/storage/types" "github.com/makerdao/vulcanizedb/pkg/datastore/postgres" ) const getOrCreateAddressQuery = `WITH addressId AS ( - INSERT INTO addresses (address, hashed_address) VALUES ($1, $2) ON CONFLICT DO NOTHING RETURNING id + INSERT INTO addresses (address) VALUES ($1) ON CONFLICT DO NOTHING RETURNING id ) SELECT id FROM addresses WHERE address = $1 UNION @@ -45,20 +44,18 @@ const getOrCreateAddressQuery = `WITH addressId AS ( func GetOrCreateAddress(db *postgres.DB, address string) (int64, error) { checksumAddress := getChecksumAddress(address) - hashedAddress := types.HexToKeccak256Hash(checksumAddress).Hex() var addressId int64 - getOrCreateErr := db.Get(&addressId, getOrCreateAddressQuery, checksumAddress, hashedAddress) + getOrCreateErr := db.Get(&addressId, getOrCreateAddressQuery, checksumAddress) return addressId, getOrCreateErr } func GetOrCreateAddressInTransaction(tx *sqlx.Tx, address string) (int64, error) { checksumAddress := getChecksumAddress(address) - hashedAddress := types.HexToKeccak256Hash(checksumAddress).Hex() var addressId int64 - getOrCreateErr := tx.Get(&addressId, getOrCreateAddressQuery, checksumAddress, hashedAddress) + getOrCreateErr := tx.Get(&addressId, getOrCreateAddressQuery, checksumAddress) return addressId, getOrCreateErr } diff --git a/libraries/shared/repository/address_repository_test.go b/libraries/shared/repository/address_repository_test.go index 91d0426ba..ca68b105f 100644 --- a/libraries/shared/repository/address_repository_test.go +++ b/libraries/shared/repository/address_repository_test.go @@ -21,7 +21,6 @@ import ( "github.com/jmoiron/sqlx" "github.com/makerdao/vulcanizedb/libraries/shared/repository" - "github.com/makerdao/vulcanizedb/libraries/shared/storage/types" "github.com/makerdao/vulcanizedb/pkg/datastore/postgres" "github.com/makerdao/vulcanizedb/pkg/fakes" "github.com/makerdao/vulcanizedb/test_config" @@ -44,9 +43,8 @@ var _ = Describe("address lookup", func() { }) type dbAddress struct { - Id int64 - Address string - HashedAddress string `db:"hashed_address"` + Id int64 + Address string } Describe("GetOrCreateAddress", func() { @@ -55,10 +53,9 @@ var _ = Describe("address lookup", func() { Expect(createErr).NotTo(HaveOccurred()) var actualAddress dbAddress - getErr := db.Get(&actualAddress, `SELECT id, address, hashed_address FROM public.addresses LIMIT 1`) + getErr := db.Get(&actualAddress, `SELECT id, address FROM public.addresses LIMIT 1`) Expect(getErr).NotTo(HaveOccurred()) - hashedAddress := types.HexToKeccak256Hash(address).Hex() - expectedAddress := dbAddress{Id: addressId, Address: address, HashedAddress: hashedAddress} + expectedAddress := dbAddress{Id: addressId, Address: address} Expect(actualAddress).To(Equal(expectedAddress)) }) @@ -118,10 +115,9 @@ var _ = Describe("address lookup", func() { Expect(commitErr).NotTo(HaveOccurred()) var actualAddress dbAddress - getErr := db.Get(&actualAddress, `SELECT id, address, hashed_address FROM public.addresses LIMIT 1`) + getErr := db.Get(&actualAddress, `SELECT id, address FROM public.addresses LIMIT 1`) Expect(getErr).NotTo(HaveOccurred()) - hashedAddress := types.HexToKeccak256Hash(address).Hex() - expectedAddress := dbAddress{Id: addressId, Address: address, HashedAddress: hashedAddress} + expectedAddress := dbAddress{Id: addressId, Address: address} Expect(actualAddress).To(Equal(expectedAddress)) }) diff --git a/libraries/shared/storage/backfill/storage_value_loader.go b/libraries/shared/storage/backfill/storage_value_loader.go index 62477cbe5..89fb278d4 100644 --- a/libraries/shared/storage/backfill/storage_value_loader.go +++ b/libraries/shared/storage/backfill/storage_value_loader.go @@ -104,16 +104,14 @@ func (r *StorageValueLoader) getAndPersistStorageValues(blockNumber int64, heade blockHash := common.HexToHash(headerHashStr) for address, chunkedKeysToValues := range r.storageByAddress { - keccakOfAddress := crypto.Keccak256Hash(address[:]) for chunkIndex, currentKeysToValues := range chunkedKeysToValues { var keys []storageKey for key, _ := range currentKeysToValues { keys = append(keys, key) } logrus.WithFields(logrus.Fields{ - "Address": address.Hex(), - "HashedAddress": keccakOfAddress.Hex(), - "BlockNumber": blockNumber, + "Address": address.Hex(), + "BlockNumber": blockNumber, }).Infof("Getting and persisting %v storage values", len(keys)) newKeysToValues, getStorageValuesErr := r.bc.BatchGetStorageAt(address, keys, blockNumberBigInt) if getStorageValuesErr != nil { @@ -125,11 +123,11 @@ func (r *StorageValueLoader) getAndPersistStorageValues(blockNumber int64, heade if newValueHash != currentKeysToValues[key] { // update last known value to new value if changed diff := types.RawDiff{ - HashedAddress: keccakOfAddress, - BlockHash: blockHash, - BlockHeight: int(blockNumber), - StorageKey: key, - StorageValue: newValueHash, + Address: address, + BlockHash: blockHash, + BlockHeight: int(blockNumber), + StorageKey: crypto.Keccak256Hash(key.Bytes()), + StorageValue: newValueHash, } createDiffErr := r.StorageDiffRepo.CreateBackFilledStorageValue(diff) if createDiffErr != nil { diff --git a/libraries/shared/storage/backfill/storage_value_loader_test.go b/libraries/shared/storage/backfill/storage_value_loader_test.go index f91b4f51d..55f258c52 100644 --- a/libraries/shared/storage/backfill/storage_value_loader_test.go +++ b/libraries/shared/storage/backfill/storage_value_loader_test.go @@ -220,6 +220,7 @@ var _ = Describe("StorageValueLoader", func() { headerHashBytes := common.HexToHash(blockOneHeader.Hash) expectedDiffOne := types.RawDiff{ +<<<<<<< HEAD BlockHeight: int(blockOne), BlockHash: headerHashBytes, HashedAddress: crypto.Keccak256Hash(addressOne[:]), @@ -232,6 +233,20 @@ var _ = Describe("StorageValueLoader", func() { HashedAddress: crypto.Keccak256Hash(addressTwo[:]), StorageKey: keyTwo, StorageValue: valueTwo, +======= + Address: addressOne, + BlockHeight: int(blockOne), + BlockHash: headerHashBytes, + StorageKey: crypto.Keccak256Hash(keyOne.Bytes()), + StorageValue: valueOne, + } + expectedDiffTwo := types.RawDiff{ + Address: addressTwo, + BlockHeight: int(blockOne), + BlockHash: headerHashBytes, + StorageKey: crypto.Keccak256Hash(keyTwo.Bytes()), + StorageValue: valueTwo, +>>>>>>> 041bc44e... Remove hashed address } Expect(diffRepo.CreateBackFilledStorageValuePassedRawDiffs).To(ConsistOf(expectedDiffOne, expectedDiffTwo)) @@ -253,25 +268,25 @@ var _ = Describe("StorageValueLoader", func() { headerHashBytes := common.HexToHash(blockOneHeader.Hash) expectedDiffOne := types.RawDiff{ - BlockHeight: int(blockOne), - BlockHash: headerHashBytes, - HashedAddress: crypto.Keccak256Hash(addressOne[:]), - StorageKey: keyOne, - StorageValue: valueOne, + Address: addressOne, + BlockHeight: int(blockOne), + BlockHash: headerHashBytes, + StorageKey: crypto.Keccak256Hash(keyOne.Bytes()), + StorageValue: valueOne, } expectedDiffTwo := types.RawDiff{ - BlockHeight: int(blockOne), - BlockHash: headerHashBytes, - HashedAddress: crypto.Keccak256Hash(addressTwo[:]), - StorageKey: keyTwo, - StorageValue: valueTwo, + Address: addressTwo, + BlockHeight: int(blockOne), + BlockHash: headerHashBytes, + StorageKey: crypto.Keccak256Hash(keyTwo.Bytes()), + StorageValue: valueTwo, } expectedDiffThree := types.RawDiff{ - BlockHeight: int(blockTwo), - BlockHash: common.HexToHash(blockTwoHeader.Hash), - HashedAddress: crypto.Keccak256Hash(addressOne[:]), - StorageKey: keyOne, - StorageValue: valueTwo, + Address: addressOne, + BlockHeight: int(blockTwo), + BlockHash: common.HexToHash(blockTwoHeader.Hash), + StorageKey: crypto.Keccak256Hash(keyOne.Bytes()), + StorageValue: valueTwo, } Expect(diffRepo.CreateBackFilledStorageValuePassedRawDiffs).To(ConsistOf(expectedDiffOne, expectedDiffTwo, expectedDiffThree)) diff --git a/libraries/shared/storage/diff_repository_test.go b/libraries/shared/storage/diff_repository_test.go index 6889230af..eb75b744f 100644 --- a/libraries/shared/storage/diff_repository_test.go +++ b/libraries/shared/storage/diff_repository_test.go @@ -41,11 +41,11 @@ var _ = Describe("Storage diffs repository", func() { test_config.CleanTestDB(db) repo = storage.NewDiffRepository(db) fakeStorageDiff = types.RawDiff{ - HashedAddress: test_data.FakeHash(), - BlockHash: test_data.FakeHash(), - BlockHeight: rand.Int(), - StorageKey: test_data.FakeHash(), - StorageValue: test_data.FakeHash(), + Address: test_data.FakeAddress(), + BlockHash: test_data.FakeHash(), + BlockHeight: rand.Int(), + StorageKey: test_data.FakeHash(), + StorageValue: test_data.FakeHash(), } }) @@ -59,7 +59,6 @@ var _ = Describe("Storage diffs repository", func() { getErr := db.Get(&persisted, `SELECT * FROM public.storage_diff`) Expect(getErr).NotTo(HaveOccurred()) Expect(persisted.ID).To(Equal(id)) - Expect(persisted.HashedAddress).To(Equal(fakeStorageDiff.HashedAddress)) Expect(persisted.BlockHash).To(Equal(fakeStorageDiff.BlockHash)) Expect(persisted.BlockHeight).To(Equal(fakeStorageDiff.BlockHeight)) Expect(persisted.StorageKey).To(Equal(fakeStorageDiff.StorageKey)) @@ -90,7 +89,6 @@ var _ = Describe("Storage diffs repository", func() { var persisted types.PersistedDiff getErr := db.Get(&persisted, `SELECT * FROM public.storage_diff`) Expect(getErr).NotTo(HaveOccurred()) - Expect(persisted.HashedAddress).To(Equal(fakeStorageDiff.HashedAddress)) Expect(persisted.BlockHash).To(Equal(fakeStorageDiff.BlockHash)) Expect(persisted.BlockHeight).To(Equal(fakeStorageDiff.BlockHeight)) Expect(persisted.StorageKey).To(Equal(fakeStorageDiff.StorageKey)) @@ -194,7 +192,17 @@ var _ = Describe("Storage diffs repository", func() { }) Describe("GetNewDiffs", func() { +<<<<<<< HEAD It("sends diffs that are marked as 'new'", func() { +======= + It("sends diffs that are not marked as checked", func() { + fakeRawDiff := types.RawDiff{ + BlockHash: test_data.FakeHash(), + BlockHeight: rand.Int(), + StorageKey: test_data.FakeHash(), + StorageValue: test_data.FakeHash(), + } +>>>>>>> 1999ea8c... Remove hashed address fakePersistedDiff := types.PersistedDiff{ RawDiff: fakeStorageDiff, ID: rand.Int63(), @@ -209,12 +217,21 @@ var _ = Describe("Storage diffs repository", func() { Expect(diffs).To(ConsistOf(fakePersistedDiff)) }) +<<<<<<< HEAD It("sends diffs that are marked as 'unrecognized'", func() { unrecognizedPersistedDiff := types.PersistedDiff{ RawDiff: fakeStorageDiff, ID: rand.Int63(), Status: storage.Unrecognized, EthNodeID: db.NodeID, +======= + It("does not send diff that's marked as checked", func() { + fakeRawDiff := types.RawDiff{ + BlockHash: test_data.FakeHash(), + BlockHeight: rand.Int(), + StorageKey: test_data.FakeHash(), + StorageValue: test_data.FakeHash(), +>>>>>>> 1999ea8c... Remove hashed address } insertTestDiff(unrecognizedPersistedDiff, db) @@ -258,11 +275,10 @@ var _ = Describe("Storage diffs repository", func() { blockZero := rand.Int() for i := 0; i < 2; i++ { fakeRawDiff := types.RawDiff{ - HashedAddress: test_data.FakeHash(), - BlockHash: test_data.FakeHash(), - BlockHeight: blockZero + i, - StorageKey: test_data.FakeHash(), - StorageValue: test_data.FakeHash(), + BlockHash: test_data.FakeHash(), + BlockHeight: blockZero + i, + StorageKey: test_data.FakeHash(), + StorageValue: test_data.FakeHash(), } persistedDiff := types.PersistedDiff{ RawDiff: fakeRawDiff, @@ -286,11 +302,24 @@ var _ = Describe("Storage diffs repository", func() { }) }) +<<<<<<< HEAD Describe("Changing the diff status", func() { var fakePersistedDiff types.PersistedDiff BeforeEach(func() { fakePersistedDiff = types.PersistedDiff{ RawDiff: fakeStorageDiff, +======= + Describe("MarkChecked", func() { + It("marks a diff as checked", func() { + fakeRawDiff := types.RawDiff{ + BlockHash: test_data.FakeHash(), + BlockHeight: rand.Int(), + StorageKey: test_data.FakeHash(), + StorageValue: test_data.FakeHash(), + } + fakePersistedDiff := types.PersistedDiff{ + RawDiff: fakeRawDiff, +>>>>>>> 1999ea8c... Remove hashed address ID: rand.Int63(), Status: storage.New, EthNodeID: db.NodeID, @@ -343,11 +372,10 @@ var _ = Describe("Storage diffs repository", func() { It("sends first diff for a given block height", func() { blockHeight := fakeStorageDiff.BlockHeight fakeStorageDiff2 := types.RawDiff{ - HashedAddress: test_data.FakeHash(), - BlockHash: test_data.FakeHash(), - BlockHeight: blockHeight, - StorageKey: test_data.FakeHash(), - StorageValue: test_data.FakeHash(), + BlockHash: test_data.FakeHash(), + BlockHeight: blockHeight, + StorageKey: test_data.FakeHash(), + StorageValue: test_data.FakeHash(), } id1, create1Err := repo.CreateStorageDiff(fakeStorageDiff) @@ -363,11 +391,10 @@ var _ = Describe("Storage diffs repository", func() { It("sends a diff for the next block height if one doesn't exist for the block passed in", func() { blockHeight := fakeStorageDiff.BlockHeight fakeStorageDiff2 := types.RawDiff{ - HashedAddress: test_data.FakeHash(), - BlockHash: test_data.FakeHash(), - BlockHeight: blockHeight, - StorageKey: test_data.FakeHash(), - StorageValue: test_data.FakeHash(), + BlockHash: test_data.FakeHash(), + BlockHeight: blockHeight, + StorageKey: test_data.FakeHash(), + StorageValue: test_data.FakeHash(), } id1, create1Err := repo.CreateStorageDiff(fakeStorageDiff) @@ -382,6 +409,15 @@ var _ = Describe("Storage diffs repository", func() { }) It("won't fail if all of the diffs within the id range are already checked", func() { +<<<<<<< HEAD +======= + fakeRawDiff := types.RawDiff{ + BlockHash: test_data.FakeHash(), + BlockHeight: rand.Int(), + StorageKey: test_data.FakeHash(), + StorageValue: test_data.FakeHash(), + } +>>>>>>> 1999ea8c... Remove hashed address fakePersistedDiff := types.PersistedDiff{ RawDiff: fakeStorageDiff, ID: rand.Int63(), diff --git a/libraries/shared/storage/extractor.go b/libraries/shared/storage/extractor.go index e8c708f31..5ea786e4a 100644 --- a/libraries/shared/storage/extractor.go +++ b/libraries/shared/storage/extractor.go @@ -48,8 +48,8 @@ func (extractor DiffExtractor) persistDiff(rawDiff types.RawDiff) { _, err := extractor.StorageDiffRepository.CreateStorageDiff(rawDiff) if err != nil { if errors.Is(err, sql.ErrNoRows) { - logrus.Tracef("ignoring duplicate diff. Block number: %v, blockHash: %v, hashedAddress: %v, storageKey: %v, storageValue: %v", - rawDiff.BlockHeight, rawDiff.BlockHash.Hex(), rawDiff.HashedAddress, rawDiff.StorageKey, rawDiff.StorageValue) + logrus.Tracef("ignoring duplicate diff. Block number: %v, blockHash: %v, storageKey: %v, storageValue: %v", + rawDiff.BlockHeight, rawDiff.BlockHash.Hex(), rawDiff.StorageKey, rawDiff.StorageValue) return } logrus.Warnf("failed to persist storage diff: %s", err.Error()) diff --git a/libraries/shared/storage/extractor_test.go b/libraries/shared/storage/extractor_test.go index ad78ee5be..32cd15ff1 100644 --- a/libraries/shared/storage/extractor_test.go +++ b/libraries/shared/storage/extractor_test.go @@ -48,11 +48,10 @@ var _ = Describe("Storage diff extractor", func() { It("persists fetched storage diff", func() { fakeDiff := types.RawDiff{ - HashedAddress: test_data.FakeHash(), - BlockHash: test_data.FakeHash(), - BlockHeight: rand.Int(), - StorageKey: test_data.FakeHash(), - StorageValue: test_data.FakeHash(), + BlockHash: test_data.FakeHash(), + BlockHeight: rand.Int(), + StorageKey: test_data.FakeHash(), + StorageValue: test_data.FakeHash(), } mockFetcher.DiffsToReturn = []types.RawDiff{fakeDiff} mockFetcher.ErrsToReturn = []error{fakes.FakeError} diff --git a/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher_test.go b/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher_test.go index 2eb049084..37d5a33b0 100644 --- a/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher_test.go +++ b/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher_test.go @@ -120,25 +120,22 @@ var _ = Describe("Geth RPC Storage Fetcher", func() { intHeight := int(height.Int64()) expectedDiff1 := types.RawDiff{ - HashedAddress: crypto.Keccak256Hash(test_data.ContractLeafKey[:]), - BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), - BlockHeight: intHeight, - StorageKey: common.BytesToHash(test_data.StorageKey), - StorageValue: common.BytesToHash(test_data.SmallStorageValue), + BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), + BlockHeight: intHeight, + StorageKey: crypto.Keccak256Hash(test_data.StorageKey), + StorageValue: common.BytesToHash(test_data.SmallStorageValue), } expectedDiff2 := types.RawDiff{ - HashedAddress: crypto.Keccak256Hash(test_data.AnotherContractLeafKey[:]), - BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), - BlockHeight: intHeight, - StorageKey: common.BytesToHash(test_data.StorageKey), - StorageValue: common.BytesToHash(test_data.LargeStorageValue), + BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), + BlockHeight: intHeight, + StorageKey: crypto.Keccak256Hash(test_data.StorageKey), + StorageValue: common.BytesToHash(test_data.LargeStorageValue), } expectedDiff3 := types.RawDiff{ - HashedAddress: crypto.Keccak256Hash(test_data.AnotherContractLeafKey[:]), - BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), - BlockHeight: intHeight, - StorageKey: common.BytesToHash(test_data.StorageKey), - StorageValue: common.BytesToHash(test_data.SmallStorageValue), + BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), + BlockHeight: intHeight, + StorageKey: crypto.Keccak256Hash(test_data.StorageKey), + StorageValue: common.BytesToHash(test_data.SmallStorageValue), } diff1 := <-storagediffChan diff --git a/libraries/shared/storage/types/diff.go b/libraries/shared/storage/types/diff.go index 4e3542248..a4c6b794b 100644 --- a/libraries/shared/storage/types/diff.go +++ b/libraries/shared/storage/types/diff.go @@ -28,11 +28,11 @@ import ( const ExpectedRowLength = 5 type RawDiff struct { - HashedAddress common.Hash `db:"hashed_address"` - BlockHash common.Hash `db:"block_hash"` - BlockHeight int `db:"block_height"` - StorageKey common.Hash `db:"storage_key"` - StorageValue common.Hash `db:"storage_value"` + Address common.Address `db:"address"` + BlockHash common.Hash `db:"block_hash"` + BlockHeight int `db:"block_height"` + StorageKey common.Hash `db:"storage_key"` + StorageValue common.Hash `db:"storage_value"` } type PersistedDiff struct { @@ -53,11 +53,11 @@ func FromParityCsvRow(csvRow []string) (RawDiff, error) { return RawDiff{}, err } return RawDiff{ - HashedAddress: HexToKeccak256Hash(csvRow[0]), - BlockHash: common.HexToHash(csvRow[1]), - BlockHeight: height, - StorageKey: common.HexToHash(csvRow[3]), - StorageValue: common.HexToHash(csvRow[4]), + Address: common.HexToAddress(csvRow[0]), + BlockHash: common.HexToHash(csvRow[1]), + BlockHeight: height, + StorageKey: common.HexToHash(csvRow[3]), + StorageValue: common.HexToHash(csvRow[4]), }, nil } @@ -69,11 +69,10 @@ func FromGethStateDiff(account filters.AccountDiff, stateDiff *filters.StateDiff } return RawDiff{ - HashedAddress: crypto.Keccak256Hash(account.Key), - BlockHash: stateDiff.BlockHash, - BlockHeight: int(stateDiff.BlockNumber.Int64()), - StorageKey: common.BytesToHash(storage.Key), - StorageValue: common.BytesToHash(decodedRLPStorageValue), + BlockHash: stateDiff.BlockHash, + BlockHeight: int(stateDiff.BlockNumber.Int64()), + StorageKey: crypto.Keccak256Hash(storage.Key), + StorageValue: common.BytesToHash(decodedRLPStorageValue), }, nil } diff --git a/libraries/shared/storage/types/diff_test.go b/libraries/shared/storage/types/diff_test.go index 194500643..8116d8ea9 100644 --- a/libraries/shared/storage/types/diff_test.go +++ b/libraries/shared/storage/types/diff_test.go @@ -44,8 +44,7 @@ var _ = Describe("Storage row parsing", func() { result, err := types.FromParityCsvRow(data) Expect(err).NotTo(HaveOccurred()) - expectedKeccakOfContractAddress := types.HexToKeccak256Hash(contract) - Expect(result.HashedAddress).To(Equal(expectedKeccakOfContractAddress)) + Expect(result.Address).To(Equal(common.HexToAddress(contract))) Expect(result.BlockHash).To(Equal(common.HexToHash(blockHash))) Expect(result.BlockHeight).To(Equal(789)) Expect(result.StorageKey).To(Equal(common.HexToHash(storageKey))) @@ -88,8 +87,6 @@ var _ = Describe("Storage row parsing", func() { result, err := types.FromGethStateDiff(accountDiff, stateDiff, storageDiff) Expect(err).NotTo(HaveOccurred()) - expectedHashedAddress := crypto.Keccak256Hash(accountDiff.Key) - Expect(result.HashedAddress).To(Equal(expectedHashedAddress)) Expect(result.BlockHash).To(Equal(fakes.FakeHash)) expectedBlockHeight := int(stateDiff.BlockNumber.Int64()) Expect(result.BlockHeight).To(Equal(expectedBlockHeight)) diff --git a/libraries/shared/watcher/storage_watcher.go b/libraries/shared/watcher/storage_watcher.go index dac9dcf20..49e05e2e0 100644 --- a/libraries/shared/watcher/storage_watcher.go +++ b/libraries/shared/watcher/storage_watcher.go @@ -46,7 +46,7 @@ type IStorageWatcher interface { type StorageWatcher struct { db *postgres.DB HeaderRepository datastore.HeaderRepository - KeccakAddressTransformers map[common.Hash]storage2.ITransformer // keccak hash of an address => transformer + KeccakAddressTransformers map[common.Address]storage2.ITransformer // keccak hash of an address => transformer StorageDiffRepository storage.DiffRepository DiffBlocksFromHeadOfChain int64 // the number of blocks from the head of the chain where diffs should be processed StatusWriter fs.StatusWriter @@ -55,7 +55,7 @@ type StorageWatcher struct { func NewStorageWatcher(db *postgres.DB, backFromHeadOfChain int64, statusWriter fs.StatusWriter) StorageWatcher { headerRepository := repositories.NewHeaderRepository(db) storageDiffRepository := storage.NewDiffRepository(db) - transformers := make(map[common.Hash]storage2.ITransformer) + transformers := make(map[common.Address]storage2.ITransformer) return StorageWatcher{ db: db, HeaderRepository: headerRepository, @@ -69,7 +69,7 @@ func NewStorageWatcher(db *postgres.DB, backFromHeadOfChain int64, statusWriter func (watcher StorageWatcher) AddTransformers(initializers []storage2.TransformerInitializer) { for _, initializer := range initializers { storageTransformer := initializer(watcher.db) - watcher.KeccakAddressTransformers[storageTransformer.KeccakContractAddress()] = storageTransformer + watcher.KeccakAddressTransformers[storageTransformer.GetContractAddress()] = storageTransformer } } @@ -170,7 +170,7 @@ func (watcher StorageWatcher) transformDiff(diff types.PersistedDiff) error { } func (watcher StorageWatcher) getTransformer(diff types.PersistedDiff) (storage2.ITransformer, bool) { - storageTransformer, ok := watcher.KeccakAddressTransformers[diff.HashedAddress] + storageTransformer, ok := watcher.KeccakAddressTransformers[diff.Address] return storageTransformer, ok } diff --git a/libraries/shared/watcher/storage_watcher_test.go b/libraries/shared/watcher/storage_watcher_test.go index 286ea6741..b69c7055a 100644 --- a/libraries/shared/watcher/storage_watcher_test.go +++ b/libraries/shared/watcher/storage_watcher_test.go @@ -37,13 +37,13 @@ var _ = Describe("Storage Watcher", func() { var statusWriter fakes.MockStatusWriter Describe("AddTransformer", func() { It("adds transformers", func() { - fakeHashedAddress := types.HexToKeccak256Hash("0x12345") - fakeTransformer := &mocks.MockStorageTransformer{KeccakOfAddress: fakeHashedAddress} + fakeAddress := fakes.FakeAddress + fakeTransformer := &mocks.MockStorageTransformer{Address: fakeAddress} w := watcher.NewStorageWatcher(test_config.NewTestDB(test_config.NewTestNode()), -1, &statusWriter) w.AddTransformers([]storage.TransformerInitializer{fakeTransformer.FakeTransformerInitializer}) - Expect(w.KeccakAddressTransformers[fakeHashedAddress]).To(Equal(fakeTransformer)) + Expect(w.KeccakAddressTransformers[fakeAddress]).To(Equal(fakeTransformer)) }) }) @@ -87,7 +87,7 @@ var _ = Describe("Storage Watcher", func() { diffID = diffID + i diff := types.PersistedDiff{ RawDiff: types.RawDiff{ - HashedAddress: test_data.FakeHash(), + Address: test_data.FakeAddress(), }, ID: int64(diffID), } @@ -109,7 +109,7 @@ var _ = Describe("Storage Watcher", func() { diffID = diffID + i diff := types.PersistedDiff{ RawDiff: types.RawDiff{ - HashedAddress: test_data.FakeHash(), + Address: test_data.FakeAddress(), }, ID: int64(diffID), } @@ -127,7 +127,7 @@ var _ = Describe("Storage Watcher", func() { It("marks diff as unwatched if no transformer is watching its address", func() { unwatchedDiff := types.PersistedDiff{ RawDiff: types.RawDiff{ - HashedAddress: test_data.FakeHash(), + Address: test_data.FakeAddress(), }, ID: rand.Int63(), } @@ -149,7 +149,7 @@ var _ = Describe("Storage Watcher", func() { storageWatcher = watcher.StorageWatcher{ HeaderRepository: mockHeaderRepository, StorageDiffRepository: mockDiffsRepository, - KeccakAddressTransformers: map[common.Hash]storage.ITransformer{}, + KeccakAddressTransformers: map[common.Address]storage.ITransformer{}, DiffBlocksFromHeadOfChain: numberOfBlocksFromHeadOfChain, StatusWriter: &statusWriter, } @@ -158,7 +158,7 @@ var _ = Describe("Storage Watcher", func() { diffID = diffID + i diff := types.PersistedDiff{ RawDiff: types.RawDiff{ - HashedAddress: test_data.FakeHash(), + Address: test_data.FakeAddress(), }, ID: int64(diffID), } @@ -192,7 +192,7 @@ var _ = Describe("Storage Watcher", func() { diffID = diffID + i diff := types.PersistedDiff{ RawDiff: types.RawDiff{ - HashedAddress: test_data.FakeHash(), + Address: test_data.FakeAddress(), }, ID: int64(diffID), } @@ -247,22 +247,22 @@ var _ = Describe("Storage Watcher", func() { Describe("when diff's address is watched", func() { var ( - hashedAddress common.Hash + contractAddress common.Address mockTransformer *mocks.MockStorageTransformer ) BeforeEach(func() { - hashedAddress = types.HexToKeccak256Hash("0x" + fakes.RandomString(20)) - mockTransformer = &mocks.MockStorageTransformer{KeccakOfAddress: hashedAddress} + contractAddress = test_data.FakeAddress() + mockTransformer = &mocks.MockStorageTransformer{Address: contractAddress} storageWatcher.AddTransformers([]storage.TransformerInitializer{mockTransformer.FakeTransformerInitializer}) }) It("does not mark diff checked if no matching header", func() { diffWithoutHeader := types.PersistedDiff{ RawDiff: types.RawDiff{ - HashedAddress: hashedAddress, - BlockHash: test_data.FakeHash(), - BlockHeight: rand.Int(), + Address: contractAddress, + BlockHash: test_data.FakeHash(), + BlockHeight: rand.Int(), }, ID: rand.Int63(), } @@ -286,11 +286,11 @@ var _ = Describe("Storage Watcher", func() { BeforeEach(func() { blockNumber = rand.Int() fakeRawDiff := types.RawDiff{ - HashedAddress: hashedAddress, - BlockHash: test_data.FakeHash(), - BlockHeight: blockNumber, - StorageKey: test_data.FakeHash(), - StorageValue: test_data.FakeHash(), + Address: contractAddress, + BlockHash: test_data.FakeHash(), + BlockHeight: blockNumber, + StorageKey: test_data.FakeHash(), + StorageValue: test_data.FakeHash(), } mockHeaderRepository.GetHeaderByBlockNumberReturnID = int64(blockNumber) mockHeaderRepository.GetHeaderByBlockNumberReturnHash = test_data.FakeHash().Hex() @@ -342,8 +342,8 @@ var _ = Describe("Storage Watcher", func() { BeforeEach(func() { fakeBlockHash := test_data.FakeHash() fakeRawDiff := types.RawDiff{ - HashedAddress: hashedAddress, - BlockHash: fakeBlockHash, + Address: contractAddress, + BlockHash: fakeBlockHash, } mockHeaderRepository.GetHeaderByBlockNumberReturnID = rand.Int63() From ca1206fc34e4574f8d6f5a5d99332e98d16f9649 Mon Sep 17 00:00:00 2001 From: Andrew J Yao <1547809+yaoandrew@users.noreply.github.com> Date: Tue, 11 Aug 2020 01:21:26 -0700 Subject: [PATCH 22/66] Fixes diff repository and migrations --- .../00005_create_storage_diffs_table.sql | 8 +++ ...reate_create_back_filled_diff_function.sql | 1 + db/schema.sql | 6 +- .../shared/storage/diff_repository_test.go | 69 +++++++++++++++++++ libraries/shared/storage/extractor.go | 4 +- libraries/shared/storage/extractor_test.go | 1 + libraries/shared/watcher/storage_watcher.go | 8 +-- .../shared/watcher/storage_watcher_test.go | 4 +- 8 files changed, 91 insertions(+), 10 deletions(-) diff --git a/db/migrations/00005_create_storage_diffs_table.sql b/db/migrations/00005_create_storage_diffs_table.sql index 0480bd697..0330f2a28 100644 --- a/db/migrations/00005_create_storage_diffs_table.sql +++ b/db/migrations/00005_create_storage_diffs_table.sql @@ -10,14 +10,22 @@ CREATE TYPE public.diff_status AS ENUM ( CREATE TABLE public.storage_diff ( id BIGSERIAL PRIMARY KEY, + address BYTEA, block_height BIGINT, block_hash BYTEA, storage_key BYTEA, storage_value BYTEA, +<<<<<<< HEAD eth_node_id INTEGER NOT NULL REFERENCES public.eth_nodes (id) ON DELETE CASCADE, status diff_status NOT NULL DEFAULT 'new', from_backfill BOOLEAN NOT NULL DEFAULT FALSE, UNIQUE (block_height, block_hash, hashed_address, storage_key, storage_value) +======= + eth_node_id INTEGER NOT NULL REFERENCES public.eth_nodes (id) ON DELETE CASCADE, + checked BOOLEAN NOT NULL DEFAULT FALSE, + from_backfill BOOLEAN NOT NULL DEFAULT FALSE, + UNIQUE (address, block_height, block_hash, storage_key, storage_value) +>>>>>>> fa657d32... Fixes diff repository and migrations ); CREATE INDEX storage_diff_new_status_index diff --git a/db/migrations/00011_create_create_back_filled_diff_function.sql b/db/migrations/00011_create_create_back_filled_diff_function.sql index d2162e318..f56921dd6 100644 --- a/db/migrations/00011_create_create_back_filled_diff_function.sql +++ b/db/migrations/00011_create_create_back_filled_diff_function.sql @@ -9,6 +9,7 @@ DECLARE SELECT storage_diff.storage_value FROM public.storage_diff WHERE storage_diff.block_height <= create_back_filled_diff.block_height + AND storage_diff.address = create_back_filled_diff.address AND storage_diff.storage_key = create_back_filled_diff.storage_key ORDER BY storage_diff.block_height DESC LIMIT 1 diff --git a/db/schema.sql b/db/schema.sql index bf7188504..3609f5cbd 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -41,6 +41,7 @@ DECLARE SELECT storage_diff.storage_value FROM public.storage_diff WHERE storage_diff.block_height <= create_back_filled_diff.block_height + AND storage_diff.address = create_back_filled_diff.address AND storage_diff.storage_key = create_back_filled_diff.storage_key ORDER BY storage_diff.block_height DESC LIMIT 1 @@ -394,6 +395,7 @@ ALTER SEQUENCE public.receipts_id_seq OWNED BY public.receipts.id; CREATE TABLE public.storage_diff ( id bigint NOT NULL, + address bytea, block_height bigint, block_hash bytea, storage_key bytea, @@ -669,11 +671,11 @@ ALTER TABLE ONLY public.receipts -- --- Name: storage_diff storage_diff_block_height_block_hash_storage_key_storage_va_key; Type: CONSTRAINT; Schema: public; Owner: - +-- Name: storage_diff storage_diff_address_block_height_block_hash_storage_key_st_key; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY public.storage_diff - ADD CONSTRAINT storage_diff_block_height_block_hash_storage_key_storage_va_key UNIQUE (block_height, block_hash, storage_key, storage_value); + ADD CONSTRAINT storage_diff_address_block_height_block_hash_storage_key_st_key UNIQUE (address, block_height, block_hash, storage_key, storage_value); -- diff --git a/libraries/shared/storage/diff_repository_test.go b/libraries/shared/storage/diff_repository_test.go index eb75b744f..ee1ebb972 100644 --- a/libraries/shared/storage/diff_repository_test.go +++ b/libraries/shared/storage/diff_repository_test.go @@ -59,6 +59,7 @@ var _ = Describe("Storage diffs repository", func() { getErr := db.Get(&persisted, `SELECT * FROM public.storage_diff`) Expect(getErr).NotTo(HaveOccurred()) Expect(persisted.ID).To(Equal(id)) + Expect(persisted.Address).To(Equal(fakeStorageDiff.Address)) Expect(persisted.BlockHash).To(Equal(fakeStorageDiff.BlockHash)) Expect(persisted.BlockHeight).To(Equal(fakeStorageDiff.BlockHeight)) Expect(persisted.StorageKey).To(Equal(fakeStorageDiff.StorageKey)) @@ -89,6 +90,7 @@ var _ = Describe("Storage diffs repository", func() { var persisted types.PersistedDiff getErr := db.Get(&persisted, `SELECT * FROM public.storage_diff`) Expect(getErr).NotTo(HaveOccurred()) + Expect(persisted.Address).To(Equal(fakeStorageDiff.Address)) Expect(persisted.BlockHash).To(Equal(fakeStorageDiff.BlockHash)) Expect(persisted.BlockHeight).To(Equal(fakeStorageDiff.BlockHeight)) Expect(persisted.StorageKey).To(Equal(fakeStorageDiff.StorageKey)) @@ -197,6 +199,7 @@ var _ = Describe("Storage diffs repository", func() { ======= It("sends diffs that are not marked as checked", func() { fakeRawDiff := types.RawDiff{ + Address: test_data.FakeAddress(), BlockHash: test_data.FakeHash(), BlockHeight: rand.Int(), StorageKey: test_data.FakeHash(), @@ -209,7 +212,21 @@ var _ = Describe("Storage diffs repository", func() { EthNodeID: db.NodeID, Status: storage.New, } +<<<<<<< HEAD insertTestDiff(fakePersistedDiff, db) +======= + _, insertErr := db.Exec(`INSERT INTO public.storage_diff (id, block_height, block_hash, +<<<<<<< HEAD + hashed_address, storage_key, storage_value, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6, $7)`, fakePersistedDiff.ID, + fakeRawDiff.BlockHeight, fakeRawDiff.BlockHash.Bytes(), fakeRawDiff.HashedAddress.Bytes(), + fakeRawDiff.StorageKey.Bytes(), fakeRawDiff.StorageValue.Bytes(), fakePersistedDiff.EthNodeID) +======= + address, storage_key, storage_value) VALUES ($1, $2, $3, $4, $5, $6)`, fakePersistedDiff.ID, + fakeRawDiff.BlockHeight, fakeRawDiff.BlockHash.Bytes(), fakeRawDiff.Address.Bytes(), + fakeRawDiff.StorageKey.Bytes(), fakeRawDiff.StorageValue.Bytes()) +>>>>>>> a4eed9d1... Fixes diff repository and migrations + Expect(insertErr).NotTo(HaveOccurred()) +>>>>>>> fa657d32... Fixes diff repository and migrations diffs, err := repo.GetNewDiffs(0, 1) @@ -227,6 +244,7 @@ var _ = Describe("Storage diffs repository", func() { ======= It("does not send diff that's marked as checked", func() { fakeRawDiff := types.RawDiff{ + Address: test_data.FakeAddress(), BlockHash: test_data.FakeHash(), BlockHeight: rand.Int(), StorageKey: test_data.FakeHash(), @@ -248,6 +266,7 @@ var _ = Describe("Storage diffs repository", func() { Status: storage.Transformed, EthNodeID: db.NodeID, } +<<<<<<< HEAD insertTestDiff(transformedPersistedDiff, db) diffs, err := repo.GetNewDiffs(0, 1) @@ -264,6 +283,21 @@ var _ = Describe("Storage diffs repository", func() { EthNodeID: db.NodeID, } insertTestDiff(noncanonicalPersistedDiff, db) +======= + _, insertErr := db.Exec(`INSERT INTO public.storage_diff (id, block_height, block_hash, +<<<<<<< HEAD + hashed_address, storage_key, storage_value, checked, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, + fakePersistedDiff.ID, fakeRawDiff.BlockHeight, fakeRawDiff.BlockHash.Bytes(), + fakeRawDiff.HashedAddress.Bytes(), fakeRawDiff.StorageKey.Bytes(), fakeRawDiff.StorageValue.Bytes(), + fakePersistedDiff.Checked, fakePersistedDiff.EthNodeID) +======= + address, storage_key, storage_value, checked) VALUES ($1, $2, $3, $4, $5, $6, $7)`, + fakePersistedDiff.ID, fakeRawDiff.BlockHeight, fakeRawDiff.BlockHash.Bytes(), + fakeRawDiff.Address.Bytes(), fakeRawDiff.StorageKey.Bytes(), fakeRawDiff.StorageValue.Bytes(), + fakePersistedDiff.Checked) +>>>>>>> a4eed9d1... Fixes diff repository and migrations + Expect(insertErr).NotTo(HaveOccurred()) +>>>>>>> fa657d32... Fixes diff repository and migrations diffs, err := repo.GetNewDiffs(0, 1) @@ -275,11 +309,13 @@ var _ = Describe("Storage diffs repository", func() { blockZero := rand.Int() for i := 0; i < 2; i++ { fakeRawDiff := types.RawDiff{ + Address: test_data.FakeAddress(), BlockHash: test_data.FakeHash(), BlockHeight: blockZero + i, StorageKey: test_data.FakeHash(), StorageValue: test_data.FakeHash(), } +<<<<<<< HEAD persistedDiff := types.PersistedDiff{ RawDiff: fakeRawDiff, ID: rand.Int63(), @@ -287,6 +323,19 @@ var _ = Describe("Storage diffs repository", func() { EthNodeID: db.NodeID, } insertTestDiff(persistedDiff, db) +======= + _, insertErr := db.Exec(`INSERT INTO public.storage_diff (block_height, block_hash, +<<<<<<< HEAD + hashed_address, storage_key, storage_value, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6)`, fakeRawDiff.BlockHeight, + fakeRawDiff.BlockHash.Bytes(), fakeRawDiff.HashedAddress.Bytes(), fakeRawDiff.StorageKey.Bytes(), + fakeRawDiff.StorageValue.Bytes(), nodeID) +======= + address, storage_key, storage_value) VALUES ($1, $2, $3, $4, $5)`, fakeRawDiff.BlockHeight, + fakeRawDiff.BlockHash.Bytes(), fakeRawDiff.Address.Bytes(), fakeRawDiff.StorageKey.Bytes(), + fakeRawDiff.StorageValue.Bytes()) +>>>>>>> a4eed9d1... Fixes diff repository and migrations + Expect(insertErr).NotTo(HaveOccurred()) +>>>>>>> fa657d32... Fixes diff repository and migrations } minID := 0 @@ -312,6 +361,7 @@ var _ = Describe("Storage diffs repository", func() { Describe("MarkChecked", func() { It("marks a diff as checked", func() { fakeRawDiff := types.RawDiff{ + Address: test_data.FakeAddress(), BlockHash: test_data.FakeHash(), BlockHeight: rand.Int(), StorageKey: test_data.FakeHash(), @@ -324,8 +374,24 @@ var _ = Describe("Storage diffs repository", func() { Status: storage.New, EthNodeID: db.NodeID, } +<<<<<<< HEAD insertTestDiff(fakePersistedDiff, db) }) +======= + _, insertErr := db.Exec(`INSERT INTO public.storage_diff (id, block_height, block_hash, +<<<<<<< HEAD + hashed_address, storage_key, storage_value, checked, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, + fakePersistedDiff.ID, fakeRawDiff.BlockHeight, fakeRawDiff.BlockHash.Bytes(), + fakeRawDiff.HashedAddress.Bytes(), fakeRawDiff.StorageKey.Bytes(), fakeRawDiff.StorageValue.Bytes(), + fakePersistedDiff.Checked, fakePersistedDiff.EthNodeID) +======= + address, storage_key, storage_value, checked) VALUES ($1, $2, $3, $4, $5, $6, $7)`, + fakePersistedDiff.ID, fakeRawDiff.BlockHeight, fakeRawDiff.BlockHash.Bytes(), + fakeRawDiff.Address.Bytes(), fakeRawDiff.StorageKey.Bytes(), fakeRawDiff.StorageValue.Bytes(), + fakePersistedDiff.Checked) +>>>>>>> a4eed9d1... Fixes diff repository and migrations + Expect(insertErr).NotTo(HaveOccurred()) +>>>>>>> fa657d32... Fixes diff repository and migrations It("marks a diff as transformed", func() { err := repo.MarkTransformed(fakePersistedDiff.ID) @@ -372,6 +438,7 @@ var _ = Describe("Storage diffs repository", func() { It("sends first diff for a given block height", func() { blockHeight := fakeStorageDiff.BlockHeight fakeStorageDiff2 := types.RawDiff{ + Address: test_data.FakeAddress(), BlockHash: test_data.FakeHash(), BlockHeight: blockHeight, StorageKey: test_data.FakeHash(), @@ -391,6 +458,7 @@ var _ = Describe("Storage diffs repository", func() { It("sends a diff for the next block height if one doesn't exist for the block passed in", func() { blockHeight := fakeStorageDiff.BlockHeight fakeStorageDiff2 := types.RawDiff{ + Address: test_data.FakeAddress(), BlockHash: test_data.FakeHash(), BlockHeight: blockHeight, StorageKey: test_data.FakeHash(), @@ -412,6 +480,7 @@ var _ = Describe("Storage diffs repository", func() { <<<<<<< HEAD ======= fakeRawDiff := types.RawDiff{ + Address: test_data.FakeAddress(), BlockHash: test_data.FakeHash(), BlockHeight: rand.Int(), StorageKey: test_data.FakeHash(), diff --git a/libraries/shared/storage/extractor.go b/libraries/shared/storage/extractor.go index 5ea786e4a..bc9e1d2e9 100644 --- a/libraries/shared/storage/extractor.go +++ b/libraries/shared/storage/extractor.go @@ -48,8 +48,8 @@ func (extractor DiffExtractor) persistDiff(rawDiff types.RawDiff) { _, err := extractor.StorageDiffRepository.CreateStorageDiff(rawDiff) if err != nil { if errors.Is(err, sql.ErrNoRows) { - logrus.Tracef("ignoring duplicate diff. Block number: %v, blockHash: %v, storageKey: %v, storageValue: %v", - rawDiff.BlockHeight, rawDiff.BlockHash.Hex(), rawDiff.StorageKey, rawDiff.StorageValue) + logrus.Tracef("ignoring duplicate diff. Block number: %v, blockHash: %v, address: %v, storageKey: %v, storageValue: %v", + rawDiff.BlockHeight, rawDiff.BlockHash.Hex(), rawDiff.Address, rawDiff.StorageKey, rawDiff.StorageValue) return } logrus.Warnf("failed to persist storage diff: %s", err.Error()) diff --git a/libraries/shared/storage/extractor_test.go b/libraries/shared/storage/extractor_test.go index 32cd15ff1..76d809860 100644 --- a/libraries/shared/storage/extractor_test.go +++ b/libraries/shared/storage/extractor_test.go @@ -48,6 +48,7 @@ var _ = Describe("Storage diff extractor", func() { It("persists fetched storage diff", func() { fakeDiff := types.RawDiff{ + Address: test_data.FakeAddress(), BlockHash: test_data.FakeHash(), BlockHeight: rand.Int(), StorageKey: test_data.FakeHash(), diff --git a/libraries/shared/watcher/storage_watcher.go b/libraries/shared/watcher/storage_watcher.go index 49e05e2e0..5ccd37562 100644 --- a/libraries/shared/watcher/storage_watcher.go +++ b/libraries/shared/watcher/storage_watcher.go @@ -46,7 +46,7 @@ type IStorageWatcher interface { type StorageWatcher struct { db *postgres.DB HeaderRepository datastore.HeaderRepository - KeccakAddressTransformers map[common.Address]storage2.ITransformer // keccak hash of an address => transformer + AddressTransformers map[common.Address]storage2.ITransformer // contract address => transformer StorageDiffRepository storage.DiffRepository DiffBlocksFromHeadOfChain int64 // the number of blocks from the head of the chain where diffs should be processed StatusWriter fs.StatusWriter @@ -59,7 +59,7 @@ func NewStorageWatcher(db *postgres.DB, backFromHeadOfChain int64, statusWriter return StorageWatcher{ db: db, HeaderRepository: headerRepository, - KeccakAddressTransformers: transformers, + AddressTransformers: transformers, StorageDiffRepository: storageDiffRepository, DiffBlocksFromHeadOfChain: backFromHeadOfChain, StatusWriter: statusWriter, @@ -69,7 +69,7 @@ func NewStorageWatcher(db *postgres.DB, backFromHeadOfChain int64, statusWriter func (watcher StorageWatcher) AddTransformers(initializers []storage2.TransformerInitializer) { for _, initializer := range initializers { storageTransformer := initializer(watcher.db) - watcher.KeccakAddressTransformers[storageTransformer.GetContractAddress()] = storageTransformer + watcher.AddressTransformers[storageTransformer.GetContractAddress()] = storageTransformer } } @@ -170,7 +170,7 @@ func (watcher StorageWatcher) transformDiff(diff types.PersistedDiff) error { } func (watcher StorageWatcher) getTransformer(diff types.PersistedDiff) (storage2.ITransformer, bool) { - storageTransformer, ok := watcher.KeccakAddressTransformers[diff.Address] + storageTransformer, ok := watcher.AddressTransformers[diff.Address] return storageTransformer, ok } diff --git a/libraries/shared/watcher/storage_watcher_test.go b/libraries/shared/watcher/storage_watcher_test.go index b69c7055a..3a50f7856 100644 --- a/libraries/shared/watcher/storage_watcher_test.go +++ b/libraries/shared/watcher/storage_watcher_test.go @@ -43,7 +43,7 @@ var _ = Describe("Storage Watcher", func() { w.AddTransformers([]storage.TransformerInitializer{fakeTransformer.FakeTransformerInitializer}) - Expect(w.KeccakAddressTransformers[fakeAddress]).To(Equal(fakeTransformer)) + Expect(w.AddressTransformers[fakeAddress]).To(Equal(fakeTransformer)) }) }) @@ -149,7 +149,7 @@ var _ = Describe("Storage Watcher", func() { storageWatcher = watcher.StorageWatcher{ HeaderRepository: mockHeaderRepository, StorageDiffRepository: mockDiffsRepository, - KeccakAddressTransformers: map[common.Address]storage.ITransformer{}, + AddressTransformers: map[common.Address]storage.ITransformer{}, DiffBlocksFromHeadOfChain: numberOfBlocksFromHeadOfChain, StatusWriter: &statusWriter, } From 79903b329016230e5a0d4b155684e1933ad784f2 Mon Sep 17 00:00:00 2001 From: Andrew J Yao <1547809+yaoandrew@users.noreply.github.com> Date: Tue, 11 Aug 2020 10:42:42 -0700 Subject: [PATCH 23/66] Fixes geth storage fetcher --- libraries/shared/factories/storage/transformer.go | 1 - .../shared/storage/fetcher/geth_rpc_storage_fetcher_test.go | 3 +++ libraries/shared/storage/types/diff.go | 1 + libraries/shared/storage/types/diff_test.go | 3 ++- 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/libraries/shared/factories/storage/transformer.go b/libraries/shared/factories/storage/transformer.go index 79ff8295c..40d65d512 100644 --- a/libraries/shared/factories/storage/transformer.go +++ b/libraries/shared/factories/storage/transformer.go @@ -37,7 +37,6 @@ type Transformer struct { Address common.Address StorageKeysLookup KeysLookup Repository Repository - hashedAddress common.Hash } func (transformer Transformer) GetStorageKeysLookup() KeysLookup { diff --git a/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher_test.go b/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher_test.go index 37d5a33b0..dbe9b1b2c 100644 --- a/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher_test.go +++ b/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher_test.go @@ -120,18 +120,21 @@ var _ = Describe("Geth RPC Storage Fetcher", func() { intHeight := int(height.Int64()) expectedDiff1 := types.RawDiff{ + Address: common.BytesToAddress(test_data.ContractLeafKey[:]), BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), BlockHeight: intHeight, StorageKey: crypto.Keccak256Hash(test_data.StorageKey), StorageValue: common.BytesToHash(test_data.SmallStorageValue), } expectedDiff2 := types.RawDiff{ + Address: common.BytesToAddress(test_data.AnotherContractLeafKey[:]), BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), BlockHeight: intHeight, StorageKey: crypto.Keccak256Hash(test_data.StorageKey), StorageValue: common.BytesToHash(test_data.LargeStorageValue), } expectedDiff3 := types.RawDiff{ + Address: common.BytesToAddress(test_data.AnotherContractLeafKey[:]), BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), BlockHeight: intHeight, StorageKey: crypto.Keccak256Hash(test_data.StorageKey), diff --git a/libraries/shared/storage/types/diff.go b/libraries/shared/storage/types/diff.go index a4c6b794b..6c20f8748 100644 --- a/libraries/shared/storage/types/diff.go +++ b/libraries/shared/storage/types/diff.go @@ -69,6 +69,7 @@ func FromGethStateDiff(account filters.AccountDiff, stateDiff *filters.StateDiff } return RawDiff{ + Address: common.BytesToAddress(account.Key), BlockHash: stateDiff.BlockHash, BlockHeight: int(stateDiff.BlockNumber.Int64()), StorageKey: crypto.Keccak256Hash(storage.Key), diff --git a/libraries/shared/storage/types/diff_test.go b/libraries/shared/storage/types/diff_test.go index 8116d8ea9..10db6af06 100644 --- a/libraries/shared/storage/types/diff_test.go +++ b/libraries/shared/storage/types/diff_test.go @@ -21,7 +21,6 @@ import ( "math/rand" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/rlp" "github.com/makerdao/vulcanizedb/libraries/shared/storage/types" @@ -87,6 +86,8 @@ var _ = Describe("Storage row parsing", func() { result, err := types.FromGethStateDiff(accountDiff, stateDiff, storageDiff) Expect(err).NotTo(HaveOccurred()) + expectedAddress := common.BytesToAddress(accountDiff.Key) + Expect(result.Address).To(Equal(expectedAddress)) Expect(result.BlockHash).To(Equal(fakes.FakeHash)) expectedBlockHeight := int(stateDiff.BlockNumber.Int64()) Expect(result.BlockHeight).To(Equal(expectedBlockHeight)) From 581b6e4ba74698bcb9d77182e5f86c1d2d81dd94 Mon Sep 17 00:00:00 2001 From: Andrew J Yao <1547809+yaoandrew@users.noreply.github.com> Date: Tue, 11 Aug 2020 11:09:05 -0700 Subject: [PATCH 24/66] Clean up after rebase --- ...reate_create_back_filled_diff_function.sql | 8 +-- db/schema.sql | 10 ++-- .../storage/backfill/storage_value_loader.go | 3 +- .../backfill/storage_value_loader_test.go | 26 ++-------- libraries/shared/storage/diff_repository.go | 6 +-- .../shared/storage/diff_repository_test.go | 51 +++++++------------ .../fetcher/geth_rpc_storage_fetcher.go | 2 +- .../fetcher/geth_rpc_storage_fetcher_test.go | 7 ++- libraries/shared/storage/types/diff.go | 2 +- 9 files changed, 40 insertions(+), 75 deletions(-) diff --git a/db/migrations/00011_create_create_back_filled_diff_function.sql b/db/migrations/00011_create_create_back_filled_diff_function.sql index f56921dd6..a2abb2761 100644 --- a/db/migrations/00011_create_create_back_filled_diff_function.sql +++ b/db/migrations/00011_create_create_back_filled_diff_function.sql @@ -26,10 +26,10 @@ BEGIN RETURN; END IF; - INSERT INTO public.storage_diff (block_height, block_hash, hashed_address, storage_key, storage_value, + INSERT INTO public.storage_diff (block_height, block_hash, address, storage_key, storage_value, eth_node_id, from_backfill) VALUES (create_back_filled_diff.block_height, create_back_filled_diff.block_hash, - create_back_filled_diff.hashed_address, create_back_filled_diff.storage_key, + create_back_filled_diff.address, create_back_filled_diff.storage_key, create_back_filled_diff.storage_value, create_back_filled_diff.eth_node_id, true) ON CONFLICT DO NOTHING; @@ -39,8 +39,8 @@ $$ LANGUAGE plpgsql; -- +goose StatementEnd -COMMENT ON FUNCTION public.create_back_filled_diff(block_height BIGINT, block_hash BYTEA, hashed_address BYTEA, storage_key BYTEA, storage_value BYTEA, eth_node_id INTEGER) +COMMENT ON FUNCTION public.create_back_filled_diff(block_height BIGINT, block_hash BYTEA, address BYTEA, storage_key BYTEA, storage_value BYTEA, eth_node_id INTEGER) IS E'@omit'; -- +goose Down -DROP FUNCTION public.create_back_filled_diff(block_height BIGINT, block_hash BYTEA, hashed_address BYTEA, storage_key BYTEA, storage_value BYTEA, eth_node_id INTEGER); +DROP FUNCTION public.create_back_filled_diff(block_height BIGINT, block_hash BYTEA, address BYTEA, storage_key BYTEA, storage_value BYTEA, eth_node_id INTEGER); diff --git a/db/schema.sql b/db/schema.sql index 3609f5cbd..8f55e696e 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -33,7 +33,7 @@ CREATE TYPE public.diff_status AS ENUM ( -- Name: create_back_filled_diff(bigint, bytea, bytea, bytea, bytea, integer); Type: FUNCTION; Schema: public; Owner: - -- -CREATE FUNCTION public.create_back_filled_diff(block_height bigint, block_hash bytea, hashed_address bytea, storage_key bytea, storage_value bytea, eth_node_id integer) RETURNS void +CREATE FUNCTION public.create_back_filled_diff(block_height bigint, block_hash bytea, address bytea, storage_key bytea, storage_value bytea, eth_node_id integer) RETURNS void LANGUAGE plpgsql AS $$ DECLARE @@ -58,10 +58,10 @@ BEGIN RETURN; END IF; - INSERT INTO public.storage_diff (block_height, block_hash, hashed_address, storage_key, storage_value, + INSERT INTO public.storage_diff (block_height, block_hash, address, storage_key, storage_value, eth_node_id, from_backfill) VALUES (create_back_filled_diff.block_height, create_back_filled_diff.block_hash, - create_back_filled_diff.hashed_address, create_back_filled_diff.storage_key, + create_back_filled_diff.address, create_back_filled_diff.storage_key, create_back_filled_diff.storage_value, create_back_filled_diff.eth_node_id, true) ON CONFLICT DO NOTHING; @@ -71,10 +71,10 @@ $$; -- --- Name: FUNCTION create_back_filled_diff(block_height bigint, block_hash bytea, hashed_address bytea, storage_key bytea, storage_value bytea, eth_node_id integer); Type: COMMENT; Schema: public; Owner: - +-- Name: FUNCTION create_back_filled_diff(block_height bigint, block_hash bytea, address bytea, storage_key bytea, storage_value bytea, eth_node_id integer); Type: COMMENT; Schema: public; Owner: - -- -COMMENT ON FUNCTION public.create_back_filled_diff(block_height bigint, block_hash bytea, hashed_address bytea, storage_key bytea, storage_value bytea, eth_node_id integer) IS '@omit'; +COMMENT ON FUNCTION public.create_back_filled_diff(block_height bigint, block_hash bytea, address bytea, storage_key bytea, storage_value bytea, eth_node_id integer) IS '@omit'; -- diff --git a/libraries/shared/storage/backfill/storage_value_loader.go b/libraries/shared/storage/backfill/storage_value_loader.go index 89fb278d4..c8f34d40c 100644 --- a/libraries/shared/storage/backfill/storage_value_loader.go +++ b/libraries/shared/storage/backfill/storage_value_loader.go @@ -5,7 +5,6 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" "github.com/makerdao/vulcanizedb/libraries/shared/factories/storage" storage2 "github.com/makerdao/vulcanizedb/libraries/shared/storage" "github.com/makerdao/vulcanizedb/libraries/shared/storage/types" @@ -126,7 +125,7 @@ func (r *StorageValueLoader) getAndPersistStorageValues(blockNumber int64, heade Address: address, BlockHash: blockHash, BlockHeight: int(blockNumber), - StorageKey: crypto.Keccak256Hash(key.Bytes()), + StorageKey: common.BytesToHash(key.Bytes()), StorageValue: newValueHash, } createDiffErr := r.StorageDiffRepo.CreateBackFilledStorageValue(diff) diff --git a/libraries/shared/storage/backfill/storage_value_loader_test.go b/libraries/shared/storage/backfill/storage_value_loader_test.go index 55f258c52..2d14f2214 100644 --- a/libraries/shared/storage/backfill/storage_value_loader_test.go +++ b/libraries/shared/storage/backfill/storage_value_loader_test.go @@ -5,7 +5,6 @@ import ( "math/rand" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" "github.com/makerdao/vulcanizedb/libraries/shared/factories/storage" "github.com/makerdao/vulcanizedb/libraries/shared/mocks" "github.com/makerdao/vulcanizedb/libraries/shared/storage/backfill" @@ -220,33 +219,18 @@ var _ = Describe("StorageValueLoader", func() { headerHashBytes := common.HexToHash(blockOneHeader.Hash) expectedDiffOne := types.RawDiff{ -<<<<<<< HEAD - BlockHeight: int(blockOne), - BlockHash: headerHashBytes, - HashedAddress: crypto.Keccak256Hash(addressOne[:]), - StorageKey: keyOne, - StorageValue: valueOne, - } - expectedDiffTwo := types.RawDiff{ - BlockHeight: int(blockOne), - BlockHash: headerHashBytes, - HashedAddress: crypto.Keccak256Hash(addressTwo[:]), - StorageKey: keyTwo, - StorageValue: valueTwo, -======= Address: addressOne, BlockHeight: int(blockOne), BlockHash: headerHashBytes, - StorageKey: crypto.Keccak256Hash(keyOne.Bytes()), + StorageKey: keyOne, StorageValue: valueOne, } expectedDiffTwo := types.RawDiff{ Address: addressTwo, BlockHeight: int(blockOne), BlockHash: headerHashBytes, - StorageKey: crypto.Keccak256Hash(keyTwo.Bytes()), + StorageKey: keyTwo, StorageValue: valueTwo, ->>>>>>> 041bc44e... Remove hashed address } Expect(diffRepo.CreateBackFilledStorageValuePassedRawDiffs).To(ConsistOf(expectedDiffOne, expectedDiffTwo)) @@ -271,21 +255,21 @@ var _ = Describe("StorageValueLoader", func() { Address: addressOne, BlockHeight: int(blockOne), BlockHash: headerHashBytes, - StorageKey: crypto.Keccak256Hash(keyOne.Bytes()), + StorageKey: keyOne, StorageValue: valueOne, } expectedDiffTwo := types.RawDiff{ Address: addressTwo, BlockHeight: int(blockOne), BlockHash: headerHashBytes, - StorageKey: crypto.Keccak256Hash(keyTwo.Bytes()), + StorageKey: keyTwo, StorageValue: valueTwo, } expectedDiffThree := types.RawDiff{ Address: addressOne, BlockHeight: int(blockTwo), BlockHash: common.HexToHash(blockTwoHeader.Hash), - StorageKey: crypto.Keccak256Hash(keyOne.Bytes()), + StorageKey: keyOne, StorageValue: valueTwo, } diff --git a/libraries/shared/storage/diff_repository.go b/libraries/shared/storage/diff_repository.go index 9dea9fc4a..ddc92f790 100644 --- a/libraries/shared/storage/diff_repository.go +++ b/libraries/shared/storage/diff_repository.go @@ -54,8 +54,8 @@ func NewDiffRepository(db *postgres.DB) diffRepository { func (repository diffRepository) CreateStorageDiff(rawDiff types.RawDiff) (int64, error) { var storageDiffID int64 row := repository.db.QueryRowx(`INSERT INTO public.storage_diff - (hashed_address, block_height, block_hash, storage_key, storage_value, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6) - ON CONFLICT DO NOTHING RETURNING id`, rawDiff.HashedAddress.Bytes(), rawDiff.BlockHeight, rawDiff.BlockHash.Bytes(), + (address, block_height, block_hash, storage_key, storage_value, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT DO NOTHING RETURNING id`, rawDiff.Address.Bytes(), rawDiff.BlockHeight, rawDiff.BlockHash.Bytes(), rawDiff.StorageKey.Bytes(), rawDiff.StorageValue.Bytes(), repository.db.NodeID) err := row.Scan(&storageDiffID) if err != nil { @@ -66,7 +66,7 @@ func (repository diffRepository) CreateStorageDiff(rawDiff types.RawDiff) (int64 func (repository diffRepository) CreateBackFilledStorageValue(rawDiff types.RawDiff) error { _, err := repository.db.Exec(`SELECT * FROM public.create_back_filled_diff($1, $2, $3, $4, $5, $6)`, - rawDiff.BlockHeight, rawDiff.BlockHash.Bytes(), rawDiff.HashedAddress.Bytes(), + rawDiff.BlockHeight, rawDiff.BlockHash.Bytes(), rawDiff.Address.Bytes(), rawDiff.StorageKey.Bytes(), rawDiff.StorageValue.Bytes(), repository.db.NodeID) if err != nil { return fmt.Errorf("error creating back filled storage value: %w", err) diff --git a/libraries/shared/storage/diff_repository_test.go b/libraries/shared/storage/diff_repository_test.go index ee1ebb972..d64350e00 100644 --- a/libraries/shared/storage/diff_repository_test.go +++ b/libraries/shared/storage/diff_repository_test.go @@ -216,15 +216,9 @@ var _ = Describe("Storage diffs repository", func() { insertTestDiff(fakePersistedDiff, db) ======= _, insertErr := db.Exec(`INSERT INTO public.storage_diff (id, block_height, block_hash, -<<<<<<< HEAD - hashed_address, storage_key, storage_value, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6, $7)`, fakePersistedDiff.ID, - fakeRawDiff.BlockHeight, fakeRawDiff.BlockHash.Bytes(), fakeRawDiff.HashedAddress.Bytes(), - fakeRawDiff.StorageKey.Bytes(), fakeRawDiff.StorageValue.Bytes(), fakePersistedDiff.EthNodeID) -======= - address, storage_key, storage_value) VALUES ($1, $2, $3, $4, $5, $6)`, fakePersistedDiff.ID, + address, storage_key, storage_value, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6, $7)`, fakePersistedDiff.ID, fakeRawDiff.BlockHeight, fakeRawDiff.BlockHash.Bytes(), fakeRawDiff.Address.Bytes(), - fakeRawDiff.StorageKey.Bytes(), fakeRawDiff.StorageValue.Bytes()) ->>>>>>> a4eed9d1... Fixes diff repository and migrations + fakeRawDiff.StorageKey.Bytes(), fakeRawDiff.StorageValue.Bytes(), fakePersistedDiff.EthNodeID) Expect(insertErr).NotTo(HaveOccurred()) >>>>>>> fa657d32... Fixes diff repository and migrations @@ -285,17 +279,10 @@ var _ = Describe("Storage diffs repository", func() { insertTestDiff(noncanonicalPersistedDiff, db) ======= _, insertErr := db.Exec(`INSERT INTO public.storage_diff (id, block_height, block_hash, -<<<<<<< HEAD - hashed_address, storage_key, storage_value, checked, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, - fakePersistedDiff.ID, fakeRawDiff.BlockHeight, fakeRawDiff.BlockHash.Bytes(), - fakeRawDiff.HashedAddress.Bytes(), fakeRawDiff.StorageKey.Bytes(), fakeRawDiff.StorageValue.Bytes(), - fakePersistedDiff.Checked, fakePersistedDiff.EthNodeID) -======= - address, storage_key, storage_value, checked) VALUES ($1, $2, $3, $4, $5, $6, $7)`, + address, storage_key, storage_value, checked, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, fakePersistedDiff.ID, fakeRawDiff.BlockHeight, fakeRawDiff.BlockHash.Bytes(), fakeRawDiff.Address.Bytes(), fakeRawDiff.StorageKey.Bytes(), fakeRawDiff.StorageValue.Bytes(), - fakePersistedDiff.Checked) ->>>>>>> a4eed9d1... Fixes diff repository and migrations + fakePersistedDiff.Checked, fakePersistedDiff.EthNodeID) Expect(insertErr).NotTo(HaveOccurred()) >>>>>>> fa657d32... Fixes diff repository and migrations @@ -325,15 +312,9 @@ var _ = Describe("Storage diffs repository", func() { insertTestDiff(persistedDiff, db) ======= _, insertErr := db.Exec(`INSERT INTO public.storage_diff (block_height, block_hash, -<<<<<<< HEAD - hashed_address, storage_key, storage_value, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6)`, fakeRawDiff.BlockHeight, - fakeRawDiff.BlockHash.Bytes(), fakeRawDiff.HashedAddress.Bytes(), fakeRawDiff.StorageKey.Bytes(), - fakeRawDiff.StorageValue.Bytes(), nodeID) -======= - address, storage_key, storage_value) VALUES ($1, $2, $3, $4, $5)`, fakeRawDiff.BlockHeight, + address, storage_key, storage_value, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6)`, fakeRawDiff.BlockHeight, fakeRawDiff.BlockHash.Bytes(), fakeRawDiff.Address.Bytes(), fakeRawDiff.StorageKey.Bytes(), - fakeRawDiff.StorageValue.Bytes()) ->>>>>>> a4eed9d1... Fixes diff repository and migrations + fakeRawDiff.StorageValue.Bytes(), nodeID) Expect(insertErr).NotTo(HaveOccurred()) >>>>>>> fa657d32... Fixes diff repository and migrations } @@ -379,17 +360,10 @@ var _ = Describe("Storage diffs repository", func() { }) ======= _, insertErr := db.Exec(`INSERT INTO public.storage_diff (id, block_height, block_hash, -<<<<<<< HEAD - hashed_address, storage_key, storage_value, checked, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, - fakePersistedDiff.ID, fakeRawDiff.BlockHeight, fakeRawDiff.BlockHash.Bytes(), - fakeRawDiff.HashedAddress.Bytes(), fakeRawDiff.StorageKey.Bytes(), fakeRawDiff.StorageValue.Bytes(), - fakePersistedDiff.Checked, fakePersistedDiff.EthNodeID) -======= - address, storage_key, storage_value, checked) VALUES ($1, $2, $3, $4, $5, $6, $7)`, + address, storage_key, storage_value, checked, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, fakePersistedDiff.ID, fakeRawDiff.BlockHeight, fakeRawDiff.BlockHash.Bytes(), fakeRawDiff.Address.Bytes(), fakeRawDiff.StorageKey.Bytes(), fakeRawDiff.StorageValue.Bytes(), - fakePersistedDiff.Checked) ->>>>>>> a4eed9d1... Fixes diff repository and migrations + fakePersistedDiff.Checked, fakePersistedDiff.EthNodeID) Expect(insertErr).NotTo(HaveOccurred()) >>>>>>> fa657d32... Fixes diff repository and migrations @@ -493,8 +467,17 @@ var _ = Describe("Storage diffs repository", func() { Status: storage.Transformed, EthNodeID: db.NodeID, } +<<<<<<< HEAD insertTestDiff(fakePersistedDiff, db) +======= + _, insertErr := db.Exec(`INSERT INTO public.storage_diff (id, block_height, block_hash, + address, storage_key, storage_value, checked, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, + fakePersistedDiff.ID, fakeRawDiff.BlockHeight, fakeRawDiff.BlockHash.Bytes(), + fakeRawDiff.Address.Bytes(), fakeRawDiff.StorageKey.Bytes(), fakeRawDiff.StorageValue.Bytes(), + fakePersistedDiff.Checked, fakePersistedDiff.EthNodeID) + Expect(insertErr).NotTo(HaveOccurred()) +>>>>>>> b0be0750... Clean up after rebase var insertedDiffID int64 getInsertedDiffIDErr := db.Get(&insertedDiffID, `SELECT id FROM storage_diff LIMIT 1`) diff --git a/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher.go b/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher.go index c9cb62661..615a0ff10 100644 --- a/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher.go +++ b/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher.go @@ -88,7 +88,7 @@ func (fetcher GethRpcStorageFetcher) handleDiffPayload(payload filters.Payload, return } - logrus.Tracef(addingDiffsLogString, rawDiff.HashedAddress.Hex(), rawDiff.BlockHeight, rawDiff.StorageKey.Hex(), rawDiff.StorageValue.Hex()) + logrus.Tracef(addingDiffsLogString, rawDiff.Address.Hex(), rawDiff.BlockHeight, rawDiff.StorageKey.Hex(), rawDiff.StorageValue.Hex()) out <- rawDiff } } diff --git a/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher_test.go b/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher_test.go index dbe9b1b2c..60a097e59 100644 --- a/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher_test.go +++ b/libraries/shared/storage/fetcher/geth_rpc_storage_fetcher_test.go @@ -19,7 +19,6 @@ import ( "io" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/rlp" "github.com/makerdao/vulcanizedb/libraries/shared/mocks" @@ -123,21 +122,21 @@ var _ = Describe("Geth RPC Storage Fetcher", func() { Address: common.BytesToAddress(test_data.ContractLeafKey[:]), BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), BlockHeight: intHeight, - StorageKey: crypto.Keccak256Hash(test_data.StorageKey), + StorageKey: common.BytesToHash(test_data.StorageKey), StorageValue: common.BytesToHash(test_data.SmallStorageValue), } expectedDiff2 := types.RawDiff{ Address: common.BytesToAddress(test_data.AnotherContractLeafKey[:]), BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), BlockHeight: intHeight, - StorageKey: crypto.Keccak256Hash(test_data.StorageKey), + StorageKey: common.BytesToHash(test_data.StorageKey), StorageValue: common.BytesToHash(test_data.LargeStorageValue), } expectedDiff3 := types.RawDiff{ Address: common.BytesToAddress(test_data.AnotherContractLeafKey[:]), BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), BlockHeight: intHeight, - StorageKey: crypto.Keccak256Hash(test_data.StorageKey), + StorageKey: common.BytesToHash(test_data.StorageKey), StorageValue: common.BytesToHash(test_data.SmallStorageValue), } diff --git a/libraries/shared/storage/types/diff.go b/libraries/shared/storage/types/diff.go index 6c20f8748..183875b4d 100644 --- a/libraries/shared/storage/types/diff.go +++ b/libraries/shared/storage/types/diff.go @@ -72,7 +72,7 @@ func FromGethStateDiff(account filters.AccountDiff, stateDiff *filters.StateDiff Address: common.BytesToAddress(account.Key), BlockHash: stateDiff.BlockHash, BlockHeight: int(stateDiff.BlockNumber.Int64()), - StorageKey: crypto.Keccak256Hash(storage.Key), + StorageKey: common.BytesToHash(storage.Key), StorageValue: common.BytesToHash(decodedRLPStorageValue), }, nil } From b3e0b6f6e1844ca938f328fc49ea809b18a0fc89 Mon Sep 17 00:00:00 2001 From: Andrew J Yao <1547809+yaoandrew@users.noreply.github.com> Date: Tue, 18 Aug 2020 17:43:22 -0700 Subject: [PATCH 25/66] Address PR feedback --- go.mod | 2 -- go.sum | 13 +++++++++++++ .../shared/factories/storage/transformer_test.go | 3 ++- .../shared/storage/backfill/storage_value_loader.go | 2 +- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index a87ff2cc8..a5c0695ce 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( github.com/hpcloud/tail v1.0.0 github.com/jmoiron/sqlx v0.0.0-20181024163419-82935fac6c1a github.com/lib/pq v1.0.0 - github.com/makerdao/vdb-mcd-transformers v0.2.17 github.com/mitchellh/go-homedir v1.1.0 github.com/onsi/ginkgo v1.10.1 github.com/onsi/gomega v1.10.0 @@ -18,7 +17,6 @@ require ( github.com/spf13/cobra v0.0.3 github.com/spf13/viper v1.3.2 golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0 - golang.org/x/sync v0.0.0-20190423024810-112230192c58 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 ) diff --git a/go.sum b/go.sum index 02e64238b..64d5808fd 100644 --- a/go.sum +++ b/go.sum @@ -35,6 +35,7 @@ github.com/apilayer/freegeoip v3.5.0+incompatible h1:z1u2gv0/rsSi/HqMDB436AiUROX github.com/apilayer/freegeoip v3.5.0+incompatible/go.mod h1:CUfFqErhFhXneJendyQ/rRcuA8kH8JxHvYnbOozmlCU= github.com/aristanetworks/fsnotify v1.4.2/go.mod h1:D/rtu7LpjYM8tRJphJ0hUBYpjai8SfX+aSNsWDTq/Ks= github.com/aristanetworks/glog v0.0.0-20180419172825-c15b03b3054f/go.mod h1:KASm+qXFKs/xjSoWn30NrWBBvdTTQq+UjkhjEJHfSFA= +github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847 h1:rtI0fD4oG/8eVokGVPYJEW1F88p1ZNgXiEIs9thEE4A= github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015 h1:7ABPr1+uJdqESAdlVevnc/2FJGiC/K3uMg1JiELeF+0= github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= @@ -83,6 +84,7 @@ github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea h1:j4317fAZh7X6GqbFowYdYdI0L9bwxL07jyPZIdepyZ0= github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= github.com/deckarep/golang-set v1.7.1 h1:SCQV0S6gTtp6itiFrTqI+pfmJ4LN85S1YzhDf9rTHJQ= github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= @@ -97,6 +99,7 @@ github.com/dop251/goja v0.0.0-20200219165308-d1232e640a87/go.mod h1:Mw6PkjjMXWbT github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c h1:JHHhtb9XWJrGNMcrVP6vyzO4dusgi/HnceHTgxSejUM= github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= @@ -135,6 +138,7 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2-0.20190517061210-b285ee9cfc6c h1:zqAKixg3cTcIasAMJV+EcfVbWwLpOZ7LeoWJvcuD/5Q= github.com/golang/protobuf v1.3.2-0.20190517061210-b285ee9cfc6c/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -148,6 +152,7 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989 h1:giknQ4mEuDFmmHSrGcbargOuLHQGtywqo4mheITex54= github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -178,6 +183,7 @@ github.com/influxdata/influxdb v1.7.9/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOpr github.com/influxdata/influxdb1-client v0.0.0-20190809212627-fc22c7df067e/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/jackpal/go-nat-pmp v1.0.1 h1:i0LektDkO1QlrTm/cSuP+PyBCDnYvjPLGl4LdWEMiaA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= @@ -196,6 +202,7 @@ github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/karalabe/usb v0.0.0-20190819132248-550797b1cad8 h1:VhnqxaTIudc9IWKx8uXRLnpdSb9noCEj+vHacjmhp68= github.com/karalabe/usb v0.0.0-20190819132248-550797b1cad8/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356 h1:I/yrLt2WilKxlQKCM52clh5rGzTKpVctGT1lH4Dc8Jw= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= github.com/karalabe/usb v0.0.0-20191104083709-911d15fe12a9 h1:ZHuwnjpP8LsVsUYqTqeVAI+GfDfJ6UNPrExZF+vX/DQ= github.com/karalabe/usb v0.0.0-20191104083709-911d15fe12a9/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= @@ -263,6 +270,7 @@ github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c h1:1RHs3tNxjXGHeul8z2t6H2N2TlAqpKe5yryJztRx4Jk= github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.3 h1:i0LBnzgiChAWHJYTQAZJDOgf8MNxAVYZJ2m63SIDimI= github.com/olekukonko/tablewriter v0.0.3/go.mod h1:YZeBtGzYYEsCHp2LST/u/0NDwGkRoBtmn1cIWCJiS6M= @@ -284,6 +292,7 @@ github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsq github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/oschwald/maxminddb-golang v1.5.0 h1:rmyoIV6z2/s9TCJedUuDiKht2RN12LWJ1L7iRGtWY64= github.com/oschwald/maxminddb-golang v1.5.0/go.mod h1:3jhIUymTJ5VREKyIhWm66LJiQt04F0UCDdodShpjWsY= +github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222 h1:goeTyGkArOZIVOMA0dQbyuPWGNQJZGPwPu/QS9GlpnA= github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= @@ -312,10 +321,12 @@ github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150 h1:ZeU+auZj1iNzN8iVhff6M38Mfu73FQiJve/GEXYJBjE= github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/prometheus/tsdb v0.10.0 h1:If5rVCMTp6W2SiRAQFlbpJNgVlgMEd+U2GZckwK38ic= github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= github.com/rjeczalik/notify v0.9.2 h1:MiTWrPj55mNDHEiIX5YUSKefw/+lCQVoAFmD6oQm5w8= github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa4QEjJeqM= @@ -344,6 +355,7 @@ github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48 h1:ju5UTwk5Odtm4trrY+4Ca4RMj5OyXbmVeDAVad2T0Jw= github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= @@ -367,6 +379,7 @@ github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJ github.com/templexxx/cpufeat v0.0.0-20180724012125-cef66df7f161/go.mod h1:wM7WEvslTq+iOEAMDLSzhVuOt5BRZ05WirO+b09GHQU= github.com/templexxx/xor v0.0.0-20181023030647-4e92f724b73b/go.mod h1:5XA7W9S6mni3h5uvOC75dA3m9CCCaS83lltmc0ukdi4= github.com/tjfoc/gmsm v1.0.1/go.mod h1:XxO4hdhhrzAd+G4CjDqaOkd0hUzmtPR/d3EiBBMn/wc= +github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZW24CsNJDfeh9Ex6Pm0Rcpc7qrgKBiL44vF4= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/tyler-smith/go-bip39 v1.0.2 h1:+t3w+KwLXO6154GNJY+qUtIxLTmFjfUmpguQT1OlOT8= github.com/tyler-smith/go-bip39 v1.0.2/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= diff --git a/libraries/shared/factories/storage/transformer_test.go b/libraries/shared/factories/storage/transformer_test.go index b10939915..e6e53d970 100644 --- a/libraries/shared/factories/storage/transformer_test.go +++ b/libraries/shared/factories/storage/transformer_test.go @@ -23,6 +23,7 @@ import ( "github.com/makerdao/vulcanizedb/libraries/shared/factories/storage" "github.com/makerdao/vulcanizedb/libraries/shared/mocks" "github.com/makerdao/vulcanizedb/libraries/shared/storage/types" + "github.com/makerdao/vulcanizedb/libraries/shared/test_data" "github.com/makerdao/vulcanizedb/pkg/fakes" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -132,7 +133,7 @@ var _ = Describe("Storage transformer", func() { ID: rand.Int63(), HeaderID: fakeHeaderID, RawDiff: types.RawDiff{ - Address: fakes.FakeAddress, + Address: test_data.FakeAddress(), BlockHash: common.HexToHash(fakeBlockHash), BlockHeight: fakeBlockNumber, StorageKey: common.Hash{}, diff --git a/libraries/shared/storage/backfill/storage_value_loader.go b/libraries/shared/storage/backfill/storage_value_loader.go index c8f34d40c..94832fe39 100644 --- a/libraries/shared/storage/backfill/storage_value_loader.go +++ b/libraries/shared/storage/backfill/storage_value_loader.go @@ -125,7 +125,7 @@ func (r *StorageValueLoader) getAndPersistStorageValues(blockNumber int64, heade Address: address, BlockHash: blockHash, BlockHeight: int(blockNumber), - StorageKey: common.BytesToHash(key.Bytes()), + StorageKey: key, StorageValue: newValueHash, } createDiffErr := r.StorageDiffRepo.CreateBackFilledStorageValue(diff) From 26792e782fabdc6d6ff2b98c96986db4906fc1d0 Mon Sep 17 00:00:00 2001 From: Andrew J Yao <1547809+yaoandrew@users.noreply.github.com> Date: Wed, 26 Aug 2020 11:53:08 -0700 Subject: [PATCH 26/66] Clean up after rebase --- .../00005_create_storage_diffs_table.sql | 10 +- ...reate_create_back_filled_diff_function.sql | 2 +- db/schema.sql | 7 +- .../shared/storage/diff_repository_test.go | 92 +------------------ 4 files changed, 8 insertions(+), 103 deletions(-) diff --git a/db/migrations/00005_create_storage_diffs_table.sql b/db/migrations/00005_create_storage_diffs_table.sql index 0330f2a28..c4e010881 100644 --- a/db/migrations/00005_create_storage_diffs_table.sql +++ b/db/migrations/00005_create_storage_diffs_table.sql @@ -15,17 +15,11 @@ CREATE TABLE public.storage_diff block_hash BYTEA, storage_key BYTEA, storage_value BYTEA, -<<<<<<< HEAD eth_node_id INTEGER NOT NULL REFERENCES public.eth_nodes (id) ON DELETE CASCADE, status diff_status NOT NULL DEFAULT 'new', from_backfill BOOLEAN NOT NULL DEFAULT FALSE, - UNIQUE (block_height, block_hash, hashed_address, storage_key, storage_value) -======= - eth_node_id INTEGER NOT NULL REFERENCES public.eth_nodes (id) ON DELETE CASCADE, - checked BOOLEAN NOT NULL DEFAULT FALSE, - from_backfill BOOLEAN NOT NULL DEFAULT FALSE, - UNIQUE (address, block_height, block_hash, storage_key, storage_value) ->>>>>>> fa657d32... Fixes diff repository and migrations + UNIQUE (block_height, block_hash, address, storage_key, storage_value) + ); CREATE INDEX storage_diff_new_status_index diff --git a/db/migrations/00011_create_create_back_filled_diff_function.sql b/db/migrations/00011_create_create_back_filled_diff_function.sql index a2abb2761..1acfe2706 100644 --- a/db/migrations/00011_create_create_back_filled_diff_function.sql +++ b/db/migrations/00011_create_create_back_filled_diff_function.sql @@ -1,6 +1,6 @@ -- +goose Up -- +goose StatementBegin -CREATE OR REPLACE FUNCTION public.create_back_filled_diff(block_height BIGINT, block_hash BYTEA, hashed_address BYTEA, +CREATE OR REPLACE FUNCTION public.create_back_filled_diff(block_height BIGINT, block_hash BYTEA, address BYTEA, storage_key BYTEA, storage_value BYTEA, eth_node_id INTEGER) RETURNS VOID AS $$ diff --git a/db/schema.sql b/db/schema.sql index 8f55e696e..4820af6f0 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -156,8 +156,7 @@ SET default_with_oids = false; CREATE TABLE public.addresses ( id bigint NOT NULL, - address character varying(42), - hashed_address character varying(66) + address character varying(42) ); @@ -671,11 +670,11 @@ ALTER TABLE ONLY public.receipts -- --- Name: storage_diff storage_diff_address_block_height_block_hash_storage_key_st_key; Type: CONSTRAINT; Schema: public; Owner: - +-- Name: storage_diff storage_diff_block_height_block_hash_address_storage_key_st_key; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY public.storage_diff - ADD CONSTRAINT storage_diff_address_block_height_block_hash_storage_key_st_key UNIQUE (address, block_height, block_hash, storage_key, storage_value); + ADD CONSTRAINT storage_diff_block_height_block_hash_address_storage_key_st_key UNIQUE (block_height, block_hash, address, storage_key, storage_value); -- diff --git a/libraries/shared/storage/diff_repository_test.go b/libraries/shared/storage/diff_repository_test.go index d64350e00..a7ddfe3c4 100644 --- a/libraries/shared/storage/diff_repository_test.go +++ b/libraries/shared/storage/diff_repository_test.go @@ -194,33 +194,14 @@ var _ = Describe("Storage diffs repository", func() { }) Describe("GetNewDiffs", func() { -<<<<<<< HEAD It("sends diffs that are marked as 'new'", func() { -======= - It("sends diffs that are not marked as checked", func() { - fakeRawDiff := types.RawDiff{ - Address: test_data.FakeAddress(), - BlockHash: test_data.FakeHash(), - BlockHeight: rand.Int(), - StorageKey: test_data.FakeHash(), - StorageValue: test_data.FakeHash(), - } ->>>>>>> 1999ea8c... Remove hashed address fakePersistedDiff := types.PersistedDiff{ RawDiff: fakeStorageDiff, ID: rand.Int63(), EthNodeID: db.NodeID, Status: storage.New, } -<<<<<<< HEAD insertTestDiff(fakePersistedDiff, db) -======= - _, insertErr := db.Exec(`INSERT INTO public.storage_diff (id, block_height, block_hash, - address, storage_key, storage_value, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6, $7)`, fakePersistedDiff.ID, - fakeRawDiff.BlockHeight, fakeRawDiff.BlockHash.Bytes(), fakeRawDiff.Address.Bytes(), - fakeRawDiff.StorageKey.Bytes(), fakeRawDiff.StorageValue.Bytes(), fakePersistedDiff.EthNodeID) - Expect(insertErr).NotTo(HaveOccurred()) ->>>>>>> fa657d32... Fixes diff repository and migrations diffs, err := repo.GetNewDiffs(0, 1) @@ -228,22 +209,12 @@ var _ = Describe("Storage diffs repository", func() { Expect(diffs).To(ConsistOf(fakePersistedDiff)) }) -<<<<<<< HEAD It("sends diffs that are marked as 'unrecognized'", func() { unrecognizedPersistedDiff := types.PersistedDiff{ RawDiff: fakeStorageDiff, ID: rand.Int63(), Status: storage.Unrecognized, EthNodeID: db.NodeID, -======= - It("does not send diff that's marked as checked", func() { - fakeRawDiff := types.RawDiff{ - Address: test_data.FakeAddress(), - BlockHash: test_data.FakeHash(), - BlockHeight: rand.Int(), - StorageKey: test_data.FakeHash(), - StorageValue: test_data.FakeHash(), ->>>>>>> 1999ea8c... Remove hashed address } insertTestDiff(unrecognizedPersistedDiff, db) @@ -260,7 +231,6 @@ var _ = Describe("Storage diffs repository", func() { Status: storage.Transformed, EthNodeID: db.NodeID, } -<<<<<<< HEAD insertTestDiff(transformedPersistedDiff, db) diffs, err := repo.GetNewDiffs(0, 1) @@ -277,14 +247,6 @@ var _ = Describe("Storage diffs repository", func() { EthNodeID: db.NodeID, } insertTestDiff(noncanonicalPersistedDiff, db) -======= - _, insertErr := db.Exec(`INSERT INTO public.storage_diff (id, block_height, block_hash, - address, storage_key, storage_value, checked, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, - fakePersistedDiff.ID, fakeRawDiff.BlockHeight, fakeRawDiff.BlockHash.Bytes(), - fakeRawDiff.Address.Bytes(), fakeRawDiff.StorageKey.Bytes(), fakeRawDiff.StorageValue.Bytes(), - fakePersistedDiff.Checked, fakePersistedDiff.EthNodeID) - Expect(insertErr).NotTo(HaveOccurred()) ->>>>>>> fa657d32... Fixes diff repository and migrations diffs, err := repo.GetNewDiffs(0, 1) @@ -302,7 +264,6 @@ var _ = Describe("Storage diffs repository", func() { StorageKey: test_data.FakeHash(), StorageValue: test_data.FakeHash(), } -<<<<<<< HEAD persistedDiff := types.PersistedDiff{ RawDiff: fakeRawDiff, ID: rand.Int63(), @@ -310,13 +271,6 @@ var _ = Describe("Storage diffs repository", func() { EthNodeID: db.NodeID, } insertTestDiff(persistedDiff, db) -======= - _, insertErr := db.Exec(`INSERT INTO public.storage_diff (block_height, block_hash, - address, storage_key, storage_value, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6)`, fakeRawDiff.BlockHeight, - fakeRawDiff.BlockHash.Bytes(), fakeRawDiff.Address.Bytes(), fakeRawDiff.StorageKey.Bytes(), - fakeRawDiff.StorageValue.Bytes(), nodeID) - Expect(insertErr).NotTo(HaveOccurred()) ->>>>>>> fa657d32... Fixes diff repository and migrations } minID := 0 @@ -332,40 +286,17 @@ var _ = Describe("Storage diffs repository", func() { }) }) -<<<<<<< HEAD Describe("Changing the diff status", func() { var fakePersistedDiff types.PersistedDiff BeforeEach(func() { fakePersistedDiff = types.PersistedDiff{ RawDiff: fakeStorageDiff, -======= - Describe("MarkChecked", func() { - It("marks a diff as checked", func() { - fakeRawDiff := types.RawDiff{ - Address: test_data.FakeAddress(), - BlockHash: test_data.FakeHash(), - BlockHeight: rand.Int(), - StorageKey: test_data.FakeHash(), - StorageValue: test_data.FakeHash(), - } - fakePersistedDiff := types.PersistedDiff{ - RawDiff: fakeRawDiff, ->>>>>>> 1999ea8c... Remove hashed address ID: rand.Int63(), Status: storage.New, EthNodeID: db.NodeID, } -<<<<<<< HEAD insertTestDiff(fakePersistedDiff, db) }) -======= - _, insertErr := db.Exec(`INSERT INTO public.storage_diff (id, block_height, block_hash, - address, storage_key, storage_value, checked, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, - fakePersistedDiff.ID, fakeRawDiff.BlockHeight, fakeRawDiff.BlockHash.Bytes(), - fakeRawDiff.Address.Bytes(), fakeRawDiff.StorageKey.Bytes(), fakeRawDiff.StorageValue.Bytes(), - fakePersistedDiff.Checked, fakePersistedDiff.EthNodeID) - Expect(insertErr).NotTo(HaveOccurred()) ->>>>>>> fa657d32... Fixes diff repository and migrations It("marks a diff as transformed", func() { err := repo.MarkTransformed(fakePersistedDiff.ID) @@ -451,33 +382,14 @@ var _ = Describe("Storage diffs repository", func() { }) It("won't fail if all of the diffs within the id range are already checked", func() { -<<<<<<< HEAD -======= - fakeRawDiff := types.RawDiff{ - Address: test_data.FakeAddress(), - BlockHash: test_data.FakeHash(), - BlockHeight: rand.Int(), - StorageKey: test_data.FakeHash(), - StorageValue: test_data.FakeHash(), - } ->>>>>>> 1999ea8c... Remove hashed address fakePersistedDiff := types.PersistedDiff{ RawDiff: fakeStorageDiff, ID: rand.Int63(), Status: storage.Transformed, EthNodeID: db.NodeID, } -<<<<<<< HEAD insertTestDiff(fakePersistedDiff, db) -======= - _, insertErr := db.Exec(`INSERT INTO public.storage_diff (id, block_height, block_hash, - address, storage_key, storage_value, checked, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, - fakePersistedDiff.ID, fakeRawDiff.BlockHeight, fakeRawDiff.BlockHash.Bytes(), - fakeRawDiff.Address.Bytes(), fakeRawDiff.StorageKey.Bytes(), fakeRawDiff.StorageValue.Bytes(), - fakePersistedDiff.Checked, fakePersistedDiff.EthNodeID) - Expect(insertErr).NotTo(HaveOccurred()) ->>>>>>> b0be0750... Clean up after rebase var insertedDiffID int64 getInsertedDiffIDErr := db.Get(&insertedDiffID, `SELECT id FROM storage_diff LIMIT 1`) @@ -500,9 +412,9 @@ var _ = Describe("Storage diffs repository", func() { func insertTestDiff(persistedDiff types.PersistedDiff, db *postgres.DB) { rawDiff := persistedDiff.RawDiff _, insertErr := db.Exec(`INSERT INTO public.storage_diff (id, block_height, block_hash, - hashed_address, storage_key, storage_value, status, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, + address, storage_key, storage_value, status, eth_node_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, persistedDiff.ID, rawDiff.BlockHeight, rawDiff.BlockHash.Bytes(), - rawDiff.HashedAddress.Bytes(), rawDiff.StorageKey.Bytes(), rawDiff.StorageValue.Bytes(), + rawDiff.Address.Bytes(), rawDiff.StorageKey.Bytes(), rawDiff.StorageValue.Bytes(), persistedDiff.Status, persistedDiff.EthNodeID) Expect(insertErr).NotTo(HaveOccurred()) } From eb10a5a6cc0138028d9265dba2918adc681b54bc Mon Sep 17 00:00:00 2001 From: Andrew J Yao <1547809+yaoandrew@users.noreply.github.com> Date: Fri, 14 Aug 2020 19:02:20 -0700 Subject: [PATCH 27/66] Adds created/updated columns for event logs --- .../00008_create_event_logs_table.sql | 22 ++++++++- db/schema.sql | 25 ++++++++++- .../shared/repository/event_trigger_test.go | 45 +++++++++++++++++++ .../repositories/event_log_repository.go | 4 +- .../repositories/event_log_repository_test.go | 6 ++- 5 files changed, 97 insertions(+), 5 deletions(-) create mode 100644 libraries/shared/repository/event_trigger_test.go diff --git a/db/migrations/00008_create_event_logs_table.sql b/db/migrations/00008_create_event_logs_table.sql index ad5c77253..84aa3527d 100644 --- a/db/migrations/00008_create_event_logs_table.sql +++ b/db/migrations/00008_create_event_logs_table.sql @@ -12,9 +12,26 @@ CREATE TABLE public.event_logs tx_index INTEGER, log_index INTEGER, raw JSONB, - transformed BOOL NOT NULL DEFAULT FALSE, + transformed BOOL NOT NULL DEFAULT FALSE, + created TIMESTAMP NOT NULL DEFAULT NOW(), + updated TIMESTAMP NOT NULL DEFAULT NOW(), UNIQUE (header_id, tx_index, log_index) ); +-- +goose StatementBegin +CREATE FUNCTION set_event_log_updated() RETURNS TRIGGER AS +$$ +BEGIN + NEW.updated = NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; +-- +goose StatementEnd + +CREATE TRIGGER event_log_updated + BEFORE UPDATE + ON public.event_logs + FOR EACH ROW +EXECUTE PROCEDURE set_event_log_updated(); CREATE INDEX event_logs_address ON event_logs (address); @@ -25,4 +42,7 @@ CREATE INDEX event_logs_untransformed WHERE transformed = false; -- +goose Down +DROP TRIGGER event_log_updated ON public.event_logs; +DROP FUNCTION set_event_log_updated(); + DROP TABLE event_logs; \ No newline at end of file diff --git a/db/schema.sql b/db/schema.sql index 4820af6f0..775f33dad 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -132,6 +132,20 @@ $$; COMMENT ON FUNCTION public.get_or_create_header(block_number bigint, hash character varying, raw jsonb, block_timestamp numeric, eth_node_id integer) IS '@omit'; +-- +-- Name: set_event_log_updated(); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.set_event_log_updated() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN + NEW.updated = NOW(); + RETURN NEW; +END; +$$; + + -- -- Name: set_header_updated(); Type: FUNCTION; Schema: public; Owner: - -- @@ -258,7 +272,9 @@ CREATE TABLE public.event_logs ( tx_index integer, log_index integer, raw jsonb, - transformed boolean DEFAULT false NOT NULL + transformed boolean DEFAULT false NOT NULL, + created timestamp without time zone DEFAULT now() NOT NULL, + updated timestamp without time zone DEFAULT now() NOT NULL ); @@ -800,6 +816,13 @@ CREATE INDEX storage_diff_unrecognized_status_index ON public.storage_diff USING CREATE INDEX transactions_header ON public.transactions USING btree (header_id); +-- +-- Name: event_logs event_log_updated; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER event_log_updated BEFORE UPDATE ON public.event_logs FOR EACH ROW EXECUTE PROCEDURE public.set_event_log_updated(); + + -- -- Name: headers header_updated; Type: TRIGGER; Schema: public; Owner: - -- diff --git a/libraries/shared/repository/event_trigger_test.go b/libraries/shared/repository/event_trigger_test.go new file mode 100644 index 000000000..063da281d --- /dev/null +++ b/libraries/shared/repository/event_trigger_test.go @@ -0,0 +1,45 @@ +package repository_test + +import ( + "github.com/makerdao/vulcanizedb/libraries/shared/test_data" + "github.com/makerdao/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/makerdao/vulcanizedb/pkg/fakes" + "github.com/makerdao/vulcanizedb/test_config" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("event updated trigger", func() { + var ( + db = test_config.NewTestDB(test_config.NewTestNode()) + headerID int64 + ) + + BeforeEach(func() { + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + var insertHeaderErr error + headerID, insertHeaderErr = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) + Expect(insertHeaderErr).NotTo(HaveOccurred()) + + }) + + type dbEvent struct { + Created string + Updated string + } + + It("indicates when a record was created or updated", func() { + var eventLogRes dbEvent + test_data.CreateTestLog(headerID, db) + initialEventLogErr := db.Get(&eventLogRes, `SELECT created, updated FROM public.event_logs`) + Expect(initialEventLogErr).NotTo(HaveOccurred()) + Expect(eventLogRes.Created).To(Equal(eventLogRes.Updated)) + + _, updateErr := db.Exec(`UPDATE public.event_logs SET block_hash = '{"new_block_hash"}' where header_id = $1`, headerID) + Expect(updateErr).NotTo(HaveOccurred()) + updatedEventErr := db.Get(&eventLogRes, `SELECT created, updated FROM public.event_logs`) + Expect(updatedEventErr).NotTo(HaveOccurred()) + Expect(eventLogRes.Created).NotTo(Equal(eventLogRes.Updated)) + }) +}) diff --git a/pkg/datastore/postgres/repositories/event_log_repository.go b/pkg/datastore/postgres/repositories/event_log_repository.go index 13c896042..6bf5d6fec 100644 --- a/pkg/datastore/postgres/repositories/event_log_repository.go +++ b/pkg/datastore/postgres/repositories/event_log_repository.go @@ -60,7 +60,9 @@ type rawEventLog struct { func (repo EventLogRepository) GetUntransformedEventLogs(minID, limit int) ([]core.EventLog, error) { var rawLogs []rawEventLog - query := fmt.Sprintf("SELECT * FROM public.event_logs WHERE transformed = false AND id > %d ORDER BY id ASC LIMIT %d", minID, limit) + query := fmt.Sprintf("SELECT id, header_id, address, topics, data, block_number, block_hash,"+ + "tx_hash, tx_index, log_index, transformed, raw FROM public.event_logs "+ + "WHERE transformed = false AND id > %d ORDER BY id ASC LIMIT %d", minID, limit) err := repo.db.Select(&rawLogs, query) if err != nil { return nil, err diff --git a/pkg/datastore/postgres/repositories/event_log_repository_test.go b/pkg/datastore/postgres/repositories/event_log_repository_test.go index 57240e6ee..0cc1448b3 100644 --- a/pkg/datastore/postgres/repositories/event_log_repository_test.go +++ b/pkg/datastore/postgres/repositories/event_log_repository_test.go @@ -71,7 +71,8 @@ var _ = Describe("Event log repository", func() { Expect(err).NotTo(HaveOccurred()) var dbLog rawEventLog - lookupErr := db.Get(&dbLog, `SELECT * FROM public.event_logs`) + lookupErr := db.Get(&dbLog, `SELECT id, header_id, address, topics, data, block_number, block_hash, + tx_hash, tx_index, log_index, transformed, raw FROM public.event_logs`) Expect(lookupErr).NotTo(HaveOccurred()) Expect(dbLog.ID).NotTo(BeZero()) Expect(dbLog.HeaderID).To(Equal(headerID)) @@ -119,7 +120,8 @@ var _ = Describe("Event log repository", func() { Expect(err).NotTo(HaveOccurred()) var dbLog rawEventLog - lookupErr := db.Get(&dbLog, `SELECT * FROM public.event_logs`) + lookupErr := db.Get(&dbLog, `SELECT id, header_id, address, topics, data, block_number, block_hash, + tx_hash, tx_index, log_index, transformed, raw FROM public.event_logs`) Expect(lookupErr).NotTo(HaveOccurred()) var logTopics []common.Hash From ceb0bb717f22d84b08fe9cc465abf2f6ec553573 Mon Sep 17 00:00:00 2001 From: yaoandrew <1547809+yaoandrew@users.noreply.github.com> Date: Sun, 16 Aug 2020 20:30:02 -0700 Subject: [PATCH 28/66] Adds created/updated columns for transactions --- .../00006_create_transactions_table.sql | 23 ++++++- db/schema.sql | 25 ++++++- .../repository/transaction_trigger_test.go | 66 +++++++++++++++++++ 3 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 libraries/shared/repository/transaction_trigger_test.go diff --git a/db/migrations/00006_create_transactions_table.sql b/db/migrations/00006_create_transactions_table.sql index 676892070..45c31b31a 100644 --- a/db/migrations/00006_create_transactions_table.sql +++ b/db/migrations/00006_create_transactions_table.sql @@ -12,11 +12,32 @@ CREATE TABLE public.transactions tx_from VARCHAR(44), tx_index INTEGER, tx_to VARCHAR(44), - "value" NUMERIC + "value" NUMERIC, + created TIMESTAMP NOT NULL DEFAULT NOW(), + updated TIMESTAMP NOT NULL DEFAULT NOW() ); +-- +goose StatementBegin +CREATE FUNCTION set_transaction_updated() RETURNS TRIGGER AS +$$ +BEGIN + NEW.updated = NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; +-- +goose StatementEnd + +CREATE TRIGGER transaction_updated + BEFORE UPDATE + ON public.transactions + FOR EACH ROW +EXECUTE PROCEDURE set_transaction_updated(); + CREATE INDEX transactions_header ON transactions (header_id); -- +goose Down +DROP TRIGGER transaction_updated ON public.transactions; +DROP FUNCTION set_transaction_updated(); + DROP TABLE transactions; diff --git a/db/schema.sql b/db/schema.sql index 775f33dad..f65195daf 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -160,6 +160,20 @@ END; $$; +-- +-- Name: set_transaction_updated(); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.set_transaction_updated() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN + NEW.updated = NOW(); + RETURN NEW; +END; +$$; + + SET default_tablespace = ''; SET default_with_oids = false; @@ -456,7 +470,9 @@ CREATE TABLE public.transactions ( tx_from character varying(44), tx_index integer, tx_to character varying(44), - value numeric + value numeric, + created timestamp without time zone DEFAULT now() NOT NULL, + updated timestamp without time zone DEFAULT now() NOT NULL ); @@ -830,6 +846,13 @@ CREATE TRIGGER event_log_updated BEFORE UPDATE ON public.event_logs FOR EACH ROW CREATE TRIGGER header_updated BEFORE UPDATE ON public.headers FOR EACH ROW EXECUTE PROCEDURE public.set_header_updated(); +-- +-- Name: transactions transaction_updated; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER transaction_updated BEFORE UPDATE ON public.transactions FOR EACH ROW EXECUTE PROCEDURE public.set_transaction_updated(); + + -- -- Name: checked_headers checked_headers_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- diff --git a/libraries/shared/repository/transaction_trigger_test.go b/libraries/shared/repository/transaction_trigger_test.go new file mode 100644 index 000000000..68ca36b1e --- /dev/null +++ b/libraries/shared/repository/transaction_trigger_test.go @@ -0,0 +1,66 @@ +package repository_test + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/makerdao/vulcanizedb/pkg/core" + "github.com/makerdao/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/makerdao/vulcanizedb/pkg/fakes" + "github.com/makerdao/vulcanizedb/test_config" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "math/big" +) + +var _ = Describe("transaction updated trigger", func() { + var ( + db = test_config.NewTestDB(test_config.NewTestNode()) + headerID int64 + ) + + BeforeEach(func() { + test_config.CleanTestDB(db) + }) + + type dbTrans struct { + Created string + Updated string + } + + It("indicates when a record was created or updated", func() { + headerRepository := repositories.NewHeaderRepository(db) + var insertHeaderErr error + headerID, insertHeaderErr = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) + Expect(insertHeaderErr).NotTo(HaveOccurred()) + fromAddress := common.HexToAddress("0x1234") + toAddress := common.HexToAddress("0x5678") + txHash := common.HexToHash("0x9876") + txIndex := big.NewInt(123) + transaction := []core.TransactionModel{{ + Data: []byte{}, + From: fromAddress.Hex(), + GasLimit: 0, + GasPrice: 0, + Hash: txHash.Hex(), + Nonce: 0, + Raw: []byte{}, + To: toAddress.Hex(), + TxIndex: txIndex.Int64(), + Value: "0", + Receipt: core.Receipt{}, + }} + + txError := headerRepository.CreateTransactions(headerID, transaction) + Expect(txError).NotTo(HaveOccurred()) + + var transUpdateRes dbTrans + initialTxErr := db.Get(&transUpdateRes, `SELECT created, updated FROM public.transactions`) + Expect(initialTxErr).NotTo(HaveOccurred()) + Expect(transUpdateRes.Created).To(Equal(transUpdateRes.Updated)) + + _, updateErr := db.Exec(`UPDATE public.transactions SET hash = '{"new_hash"}' WHERE header_id = $1`, headerID) + Expect(updateErr).NotTo(HaveOccurred()) + updatedTransErr := db.Get(&transUpdateRes, `SELECT created, updated FROM public.transactions`) + Expect(updatedTransErr).NotTo(HaveOccurred()) + Expect(transUpdateRes.Created).NotTo(Equal(transUpdateRes.Updated)) + }) +}) From 8f577d401c4dd1da22c08a924ceabd45a9f93839 Mon Sep 17 00:00:00 2001 From: yaoandrew <1547809+yaoandrew@users.noreply.github.com> Date: Sun, 16 Aug 2020 21:52:07 -0700 Subject: [PATCH 29/66] Adds created/updated columns for storage diffs --- .../00005_create_storage_diffs_table.sql | 22 +++++++++++++++- db/schema.sql | 21 ++++++++++++++++ .../shared/storage/diff_repository_test.go | 25 +++++++++++++++++-- 3 files changed, 65 insertions(+), 3 deletions(-) diff --git a/db/migrations/00005_create_storage_diffs_table.sql b/db/migrations/00005_create_storage_diffs_table.sql index c4e010881..4e7109473 100644 --- a/db/migrations/00005_create_storage_diffs_table.sql +++ b/db/migrations/00005_create_storage_diffs_table.sql @@ -18,8 +18,9 @@ CREATE TABLE public.storage_diff eth_node_id INTEGER NOT NULL REFERENCES public.eth_nodes (id) ON DELETE CASCADE, status diff_status NOT NULL DEFAULT 'new', from_backfill BOOLEAN NOT NULL DEFAULT FALSE, + created TIMESTAMP NOT NULL DEFAULT NOW(), + updated TIMESTAMP NOT NULL DEFAULT NOW(), UNIQUE (block_height, block_hash, address, storage_key, storage_value) - ); CREATE INDEX storage_diff_new_status_index @@ -29,6 +30,25 @@ CREATE INDEX storage_diff_unrecognized_status_index CREATE INDEX storage_diff_eth_node ON public.storage_diff (eth_node_id); +-- +goose StatementBegin +CREATE FUNCTION set_storage_updated() RETURNS TRIGGER AS +$$ +BEGIN + NEW.updated = NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; +-- +goose StatementEnd + +CREATE TRIGGER storage_updated + BEFORE UPDATE + ON public.storage_diff + FOR EACH ROW +EXECUTE PROCEDURE set_storage_updated(); + -- +goose Down DROP TYPE public.diff_status CASCADE; +DROP TRIGGER storage_updated ON public.storage_diff; +DROP FUNCTION set_storage_updated(); + DROP TABLE public.storage_diff; diff --git a/db/schema.sql b/db/schema.sql index f65195daf..7c1a9acbf 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -160,6 +160,20 @@ END; $$; +-- +-- Name: set_storage_updated(); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.set_storage_updated() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN + NEW.updated = NOW(); + RETURN NEW; +END; +$$; + + -- -- Name: set_transaction_updated(); Type: FUNCTION; Schema: public; Owner: - -- @@ -846,6 +860,13 @@ CREATE TRIGGER event_log_updated BEFORE UPDATE ON public.event_logs FOR EACH ROW CREATE TRIGGER header_updated BEFORE UPDATE ON public.headers FOR EACH ROW EXECUTE PROCEDURE public.set_header_updated(); +-- +-- Name: storage_diff storage_updated; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER storage_updated BEFORE UPDATE ON public.storage_diff FOR EACH ROW EXECUTE PROCEDURE public.set_storage_updated(); + + -- -- Name: transactions transaction_updated; Type: TRIGGER; Schema: public; Owner: - -- diff --git a/libraries/shared/storage/diff_repository_test.go b/libraries/shared/storage/diff_repository_test.go index a7ddfe3c4..5f5943412 100644 --- a/libraries/shared/storage/diff_repository_test.go +++ b/libraries/shared/storage/diff_repository_test.go @@ -49,6 +49,11 @@ var _ = Describe("Storage diffs repository", func() { } }) + type dbStorageDiff struct { + Created string + Updated string + } + Describe("CreateStorageDiff", func() { It("adds a storage diff to the db, returning id", func() { id, createErr := repo.CreateStorageDiff(fakeStorageDiff) @@ -56,7 +61,7 @@ var _ = Describe("Storage diffs repository", func() { Expect(createErr).NotTo(HaveOccurred()) Expect(id).NotTo(BeZero()) var persisted types.PersistedDiff - getErr := db.Get(&persisted, `SELECT * FROM public.storage_diff`) + getErr := db.Get(&persisted, `SELECT id, hashed_address, block_hash, block_height, storage_key, storage_value FROM public.storage_diff`) Expect(getErr).NotTo(HaveOccurred()) Expect(persisted.ID).To(Equal(id)) Expect(persisted.Address).To(Equal(fakeStorageDiff.Address)) @@ -80,6 +85,22 @@ var _ = Describe("Storage diffs repository", func() { Expect(getErr).NotTo(HaveOccurred()) Expect(count).To(Equal(1)) }) + + It("indicates when a record was created or updated", func() { + id, createErr := repo.CreateStorageDiff(fakeStorageDiff) + Expect(createErr).NotTo(HaveOccurred()) + + var storageDiffUpdatedRes dbStorageDiff + initialStorageErr := db.Get(&storageDiffUpdatedRes, `SELECT created, updated FROM public.storage_diff`) + Expect(initialStorageErr).NotTo(HaveOccurred()) + Expect(storageDiffUpdatedRes.Created).To(Equal(storageDiffUpdatedRes.Updated)) + + _, updateErr := db.Exec(`UPDATE public.storage_diff SET block_hash = '{"new_block_hash"}' where id = $1`, id) + Expect(updateErr).NotTo(HaveOccurred()) + updatedDiffErr := db.Get(&storageDiffUpdatedRes, `SELECT created, updated FROM public.storage_diff`) + Expect(updatedDiffErr).NotTo(HaveOccurred()) + Expect(storageDiffUpdatedRes.Created).NotTo(Equal(storageDiffUpdatedRes.Updated)) + }) }) Describe("CreateBackFilledStorageValue", func() { @@ -88,7 +109,7 @@ var _ = Describe("Storage diffs repository", func() { Expect(createErr).NotTo(HaveOccurred()) var persisted types.PersistedDiff - getErr := db.Get(&persisted, `SELECT * FROM public.storage_diff`) + getErr := db.Get(&persisted, `SELECT hashed_address, block_hash, block_height, storage_key, storage_value FROM public.storage_diff`) Expect(getErr).NotTo(HaveOccurred()) Expect(persisted.Address).To(Equal(fakeStorageDiff.Address)) Expect(persisted.BlockHash).To(Equal(fakeStorageDiff.BlockHash)) From 8376415506dbbae12c6070d298e9d8f177774687 Mon Sep 17 00:00:00 2001 From: Andrew J Yao <1547809+yaoandrew@users.noreply.github.com> Date: Thu, 27 Aug 2020 15:16:51 -0700 Subject: [PATCH 30/66] Adds created/updated for receipts table --- db/migrations/00007_create_receipts_table.sql | 19 ++++ db/schema.sql | 25 ++++- .../shared/repository/event_trigger_test.go | 1 - .../shared/repository/receipt_trigger_test.go | 92 +++++++++++++++++++ 4 files changed, 135 insertions(+), 2 deletions(-) create mode 100644 libraries/shared/repository/receipt_trigger_test.go diff --git a/db/migrations/00007_create_receipts_table.sql b/db/migrations/00007_create_receipts_table.sql index 200ef2398..a6e47fa36 100644 --- a/db/migrations/00007_create_receipts_table.sql +++ b/db/migrations/00007_create_receipts_table.sql @@ -11,8 +11,25 @@ CREATE TABLE public.receipts status INTEGER, tx_hash VARCHAR(66), rlp BYTEA, + created TIMESTAMP NOT NULL DEFAULT NOW(), + updated TIMESTAMP NOT NULL DEFAULT NOW(), UNIQUE (header_id, transaction_id) ); +-- +goose StatementBegin +CREATE FUNCTION set_receipt_updated() RETURNS TRIGGER AS +$$ +BEGIN + NEW.updated = NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; +-- +goose StatementEnd + +CREATE TRIGGER receipt_updated + BEFORE UPDATE + ON public.receipts + FOR EACH ROW +EXECUTE PROCEDURE set_receipt_updated(); CREATE INDEX receipts_contract_address ON public.receipts (contract_address_id); @@ -20,4 +37,6 @@ CREATE INDEX receipts_transaction ON public.receipts (transaction_id); -- +goose Down +DROP TRIGGER receipt_updated ON public.receipts; +DROP FUNCTION set_receipt_updated(); DROP TABLE receipts; diff --git a/db/schema.sql b/db/schema.sql index 7c1a9acbf..324ad21a1 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -160,6 +160,20 @@ END; $$; +-- +-- Name: set_receipt_updated(); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.set_receipt_updated() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN + NEW.updated = NOW(); + RETURN NEW; +END; +$$; + + -- -- Name: set_storage_updated(); Type: FUNCTION; Schema: public; Owner: - -- @@ -408,7 +422,9 @@ CREATE TABLE public.receipts ( state_root character varying(66), status integer, tx_hash character varying(66), - rlp bytea + rlp bytea, + created timestamp without time zone DEFAULT now() NOT NULL, + updated timestamp without time zone DEFAULT now() NOT NULL ); @@ -860,6 +876,13 @@ CREATE TRIGGER event_log_updated BEFORE UPDATE ON public.event_logs FOR EACH ROW CREATE TRIGGER header_updated BEFORE UPDATE ON public.headers FOR EACH ROW EXECUTE PROCEDURE public.set_header_updated(); +-- +-- Name: receipts receipt_updated; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER receipt_updated BEFORE UPDATE ON public.receipts FOR EACH ROW EXECUTE PROCEDURE public.set_receipt_updated(); + + -- -- Name: storage_diff storage_updated; Type: TRIGGER; Schema: public; Owner: - -- diff --git a/libraries/shared/repository/event_trigger_test.go b/libraries/shared/repository/event_trigger_test.go index 063da281d..f5a78341c 100644 --- a/libraries/shared/repository/event_trigger_test.go +++ b/libraries/shared/repository/event_trigger_test.go @@ -21,7 +21,6 @@ var _ = Describe("event updated trigger", func() { var insertHeaderErr error headerID, insertHeaderErr = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) Expect(insertHeaderErr).NotTo(HaveOccurred()) - }) type dbEvent struct { diff --git a/libraries/shared/repository/receipt_trigger_test.go b/libraries/shared/repository/receipt_trigger_test.go new file mode 100644 index 000000000..efcad9688 --- /dev/null +++ b/libraries/shared/repository/receipt_trigger_test.go @@ -0,0 +1,92 @@ +package repository_test + +import ( + "math/big" + "math/rand" + + "github.com/makerdao/vulcanizedb/libraries/shared/test_data" + "github.com/makerdao/vulcanizedb/pkg/core" + "github.com/makerdao/vulcanizedb/pkg/datastore" + "github.com/makerdao/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/makerdao/vulcanizedb/pkg/fakes" + "github.com/makerdao/vulcanizedb/test_config" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("receipt updated trigger", func() { + var ( + db = test_config.NewTestDB(test_config.NewTestNode()) + receiptRepo repositories.ReceiptRepository + headerRepo datastore.HeaderRepository + headerID int64 + ) + + BeforeEach(func() { + test_config.CleanTestDB(db) + receiptRepo = repositories.ReceiptRepository{} + headerRepo = repositories.NewHeaderRepository(db) + var insertHeaderErr error + headerID, insertHeaderErr = headerRepo.CreateOrUpdateHeader(fakes.FakeHeader) + Expect(insertHeaderErr).NotTo(HaveOccurred()) + }) + + type dbEvent struct { + Created string + Updated string + } + + It("indicates when a receipt record was created or updated", func() { + var receiptRes dbEvent + fromAddress := test_data.FakeAddress() + toAddress := test_data.FakeAddress() + txHash := test_data.FakeHash() + txIndex := big.NewInt(123) + transaction := core.TransactionModel{ + Data: []byte{}, + From: fromAddress.Hex(), + GasLimit: 0, + GasPrice: 0, + Hash: txHash.Hex(), + Nonce: 0, + Raw: []byte{}, + To: toAddress.Hex(), + TxIndex: txIndex.Int64(), + Value: "0", + } + tx, err := db.Beginx() + Expect(err).ToNot(HaveOccurred()) + txId, txErr := headerRepo.CreateTransactionInTx(tx, headerID, transaction) + Expect(txErr).ToNot(HaveOccurred()) + receipt := core.Receipt{ + ContractAddress: fromAddress.Hex(), + TxHash: txHash.Hex(), + GasUsed: uint64(rand.Int31()), + CumulativeGasUsed: uint64(rand.Int31()), + StateRoot: test_data.FakeHash().Hex(), + Rlp: test_data.FakeHash().Bytes(), + } + + _, receiptErr := receiptRepo.CreateReceiptInTx(headerID, txId, receipt, tx) + Expect(receiptErr).ToNot(HaveOccurred()) + if receiptErr == nil { + // this hangs if called when receiptsErr != nil + commitErr := tx.Commit() + Expect(commitErr).ToNot(HaveOccurred()) + } else { + // lookup on public.receipts below hangs if tx is still open + rollbackErr := tx.Rollback() + Expect(rollbackErr).NotTo(HaveOccurred()) + } + + createdReceiptErr := db.Get(&receiptRes, `SELECT created, updated FROM receipts`) + Expect(createdReceiptErr).NotTo(HaveOccurred()) + Expect(receiptRes.Created).To(Equal(receiptRes.Updated)) + + _, updateErr := db.Exec(`UPDATE public.receipts SET tx_hash = '{"new_hash"}' WHERE header_id = $1`, headerID) + Expect(updateErr).NotTo(HaveOccurred()) + updatedReceiptErr := db.Get(&receiptRes, `SELECT created, updated FROM receipts`) + Expect(updatedReceiptErr).NotTo(HaveOccurred()) + Expect(receiptRes.Created).NotTo(Equal(receiptRes.Updated)) + }) +}) From 12cdd9a5273f46c5fa729009dce3843ece4dd350 Mon Sep 17 00:00:00 2001 From: Andrew J Yao <1547809+yaoandrew@users.noreply.github.com> Date: Fri, 28 Aug 2020 16:16:48 -0700 Subject: [PATCH 31/66] Clean up after rebase --- db/schema.sql | 4 +++- libraries/shared/storage/diff_repository.go | 3 ++- libraries/shared/storage/diff_repository_test.go | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/db/schema.sql b/db/schema.sql index 324ad21a1..e4654a8a0 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -461,7 +461,9 @@ CREATE TABLE public.storage_diff ( storage_value bytea, eth_node_id integer NOT NULL, status public.diff_status DEFAULT 'new'::public.diff_status NOT NULL, - from_backfill boolean DEFAULT false NOT NULL + from_backfill boolean DEFAULT false NOT NULL, + created timestamp without time zone DEFAULT now() NOT NULL, + updated timestamp without time zone DEFAULT now() NOT NULL ); diff --git a/libraries/shared/storage/diff_repository.go b/libraries/shared/storage/diff_repository.go index ddc92f790..b550a1271 100644 --- a/libraries/shared/storage/diff_repository.go +++ b/libraries/shared/storage/diff_repository.go @@ -78,7 +78,8 @@ func (repository diffRepository) GetNewDiffs(minID, limit int) ([]types.Persiste var result []types.PersistedDiff err := repository.db.Select( &result, - `SELECT * FROM public.storage_diff WHERE (status = $1 OR status = $2) AND id > $3 ORDER BY id ASC LIMIT $4`, + `SELECT id, address, block_height, block_hash, storage_key, storage_value, eth_node_id, status, from_backfill + FROM public.storage_diff WHERE (status = $1 OR status = $2) AND id > $3 ORDER BY id ASC LIMIT $4`, New, Unrecognized, minID, limit, ) if err != nil { diff --git a/libraries/shared/storage/diff_repository_test.go b/libraries/shared/storage/diff_repository_test.go index 5f5943412..536df7522 100644 --- a/libraries/shared/storage/diff_repository_test.go +++ b/libraries/shared/storage/diff_repository_test.go @@ -61,7 +61,7 @@ var _ = Describe("Storage diffs repository", func() { Expect(createErr).NotTo(HaveOccurred()) Expect(id).NotTo(BeZero()) var persisted types.PersistedDiff - getErr := db.Get(&persisted, `SELECT id, hashed_address, block_hash, block_height, storage_key, storage_value FROM public.storage_diff`) + getErr := db.Get(&persisted, `SELECT id, address, block_hash, block_height, storage_key, storage_value, status FROM public.storage_diff`) Expect(getErr).NotTo(HaveOccurred()) Expect(persisted.ID).To(Equal(id)) Expect(persisted.Address).To(Equal(fakeStorageDiff.Address)) @@ -109,7 +109,7 @@ var _ = Describe("Storage diffs repository", func() { Expect(createErr).NotTo(HaveOccurred()) var persisted types.PersistedDiff - getErr := db.Get(&persisted, `SELECT hashed_address, block_hash, block_height, storage_key, storage_value FROM public.storage_diff`) + getErr := db.Get(&persisted, `SELECT address, block_hash, block_height, storage_key, storage_value, status FROM public.storage_diff`) Expect(getErr).NotTo(HaveOccurred()) Expect(persisted.Address).To(Equal(fakeStorageDiff.Address)) Expect(persisted.BlockHash).To(Equal(fakeStorageDiff.BlockHash)) From 8214744506ac022a6e4c8fdceb51c7dcca02927e Mon Sep 17 00:00:00 2001 From: Rob Mulholand Date: Tue, 11 Aug 2020 14:33:28 -0500 Subject: [PATCH 32/66] Add header repository to log extractor initializer - so we can lookup headers to backfill --- libraries/shared/logs/extractor.go | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/shared/logs/extractor.go b/libraries/shared/logs/extractor.go index ec9d9227a..5dadde0ab 100644 --- a/libraries/shared/logs/extractor.go +++ b/libraries/shared/logs/extractor.go @@ -67,6 +67,7 @@ func NewLogExtractor(db *postgres.DB, bc core.BlockChain) *LogExtractor { CheckedHeadersRepository: repositories.NewCheckedHeadersRepository(db), CheckedLogsRepository: repositories.NewCheckedLogsRepository(db), Fetcher: fetcher.NewLogFetcher(bc), + HeaderRepository: repositories.NewHeaderRepository(db), LogRepository: repositories.NewEventLogRepository(db), Syncer: transactions.NewTransactionsSyncer(db, bc), RecheckHeaderCap: constants.RecheckHeaderCap, From 27b6b2afff66b496aaaf9a5db16e4d2206c9f77b Mon Sep 17 00:00:00 2001 From: Rob Mulholand Date: Tue, 25 Aug 2020 10:59:46 -0500 Subject: [PATCH 33/66] Remove error when no storage back-fill required - Prevent auto-restart when no storage transformers configured for back-fill --- cmd/backfillStorage.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/backfillStorage.go b/cmd/backfillStorage.go index aa7d2c2e5..dbdc2445a 100644 --- a/cmd/backfillStorage.go +++ b/cmd/backfillStorage.go @@ -64,7 +64,8 @@ func backfillStorage() error { } if len(storageInitializers) == 0 { - return fmt.Errorf("SubCommand %v: no storage transformers found in the given config", SubCommand) + logrus.Warn("not back-filling storage because no contracts configured for back-fill") + return nil } var loader backfill.StorageValueLoader From 26e6f5146b55f4facd626a60fd03d862c391948e Mon Sep 17 00:00:00 2001 From: Rob Mulholand Date: Thu, 27 Aug 2020 11:24:43 -0500 Subject: [PATCH 34/66] Remove error when no event back-fill required - Prevent auto-restart when no event transformers configured for back-fill --- cmd/backfillEvents.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/backfillEvents.go b/cmd/backfillEvents.go index 2bd418787..959fae833 100644 --- a/cmd/backfillEvents.go +++ b/cmd/backfillEvents.go @@ -42,6 +42,11 @@ func backFillEvents() error { LogWithCommand.Fatalf("SubCommand %v: exporting transformers failed: %v", SubCommand, exportTransformersErr) } + if len(ethEventInitializers) < 1 { + logrus.Warn("not back-filling events because no transformers configured for back-fill") + return nil + } + blockChain := getBlockChain() db := utils.LoadPostgres(databaseConfig, blockChain.Node()) From bd309968fc5d75773b9caca8a46ae08bd0ae4707 Mon Sep 17 00:00:00 2001 From: Rob Mulholand Date: Tue, 11 Aug 2020 14:33:28 -0500 Subject: [PATCH 35/66] Handle errors in tests --- go.sum | 5 +++++ libraries/shared/logs/extractor_test.go | 9 ++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/go.sum b/go.sum index 64d5808fd..0f756a36b 100644 --- a/go.sum +++ b/go.sum @@ -109,6 +109,7 @@ github.com/elastic/gosigar v0.10.4/go.mod h1:cdorVVzy1fhmEqmtgqkoE3bYtCfSCkVyjTy github.com/elastic/gosigar v0.10.5 h1:GzPQ+78RaAb4J63unidA/JavQRKrB6s8IOzN6Ib59jo= github.com/elastic/gosigar v0.10.5/go.mod h1:cdorVVzy1fhmEqmtgqkoE3bYtCfSCkVyjTyCIo22xvs= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc h1:jtW8jbpkO4YirRSyepBOH8E+2HEw6/hKkBvFPwhUN8c= github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= @@ -232,6 +233,7 @@ github.com/makerdao/go-ethereum v1.9.15-statechange-filter h1:jLX9CleT+i/gUn5LLT github.com/makerdao/go-ethereum v1.9.15-statechange-filter/go.mod h1:slT8bPPRhXsyNTwHQxrOnjuTZ1sDXRajW11EkJ84QJ0= github.com/makerdao/vdb-mcd-transformers v0.2.17/go.mod h1:lon896AMm/ZzNGu4n1Cfa1XW2V3C1jtHs13VRvTHKrg= github.com/makerdao/vulcanizedb v0.0.11-rc.1/go.mod h1:FZZ/SKy+7Ejdc+4Wa5oTGrr2cFuaCgGBxc2xoifaEoY= +github.com/mattn/go-colorable v0.1.0 h1:v2XXALHHh6zHfYTJ+cSkwtyffnaOyR1MXaA91mTrb8o= github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -239,6 +241,7 @@ github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaa github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035 h1:USWjF42jDCSEeikX/G1g40ZWnsPXN5WkZ4jMHZWyBK4= github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -330,9 +333,11 @@ github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeC github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= github.com/rjeczalik/notify v0.9.2 h1:MiTWrPj55mNDHEiIX5YUSKefw/+lCQVoAFmD6oQm5w8= github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa4QEjJeqM= +github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00 h1:8DPul/X0IT/1TNMIxoKLwdemEOBBHDC/K4EB16Cw5WE= github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521 h1:3hxavr+IHMsQBrYUPQM5v0CgENFktkkbg1sfpgM3h20= github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubrpZHtQLnOf6EwhN2hEMusxZOhcW9H3UQ= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= diff --git a/libraries/shared/logs/extractor_test.go b/libraries/shared/logs/extractor_test.go index b63d9e07a..75895ffb8 100644 --- a/libraries/shared/logs/extractor_test.go +++ b/libraries/shared/logs/extractor_test.go @@ -170,7 +170,8 @@ var _ = Describe("Log extractor", func() { mockCheckedHeadersRepository.UncheckedHeadersReturnHeaders = []core.Header{{}} extractor.CheckedHeadersRepository = mockCheckedHeadersRepository startingBlockNumber := rand.Int63() - extractor.AddTransformerConfig(getTransformerConfig(startingBlockNumber, defaultEndingBlockNumber)) + addErr := extractor.AddTransformerConfig(getTransformerConfig(startingBlockNumber, defaultEndingBlockNumber)) + Expect(addErr).NotTo(HaveOccurred()) err := extractor.ExtractLogs(constants.HeaderUnchecked) @@ -187,7 +188,8 @@ var _ = Describe("Log extractor", func() { mockCheckedHeadersRepository.UncheckedHeadersReturnHeaders = []core.Header{{}} extractor.CheckedHeadersRepository = mockCheckedHeadersRepository startingBlockNumber := rand.Int63() - extractor.AddTransformerConfig(getTransformerConfig(startingBlockNumber, defaultEndingBlockNumber)) + addErr := extractor.AddTransformerConfig(getTransformerConfig(startingBlockNumber, defaultEndingBlockNumber)) + Expect(addErr).NotTo(HaveOccurred()) err := extractor.ExtractLogs(constants.HeaderRecheck) @@ -594,7 +596,8 @@ func addTransformerConfig(extractor *logs.LogExtractor) int64 { Topic: fakes.FakeHash.Hex(), StartingBlockNumber: rand.Int63(), } - extractor.AddTransformerConfig(fakeConfig) + addErr := extractor.AddTransformerConfig(fakeConfig) + Expect(addErr).NotTo(HaveOccurred()) return fakeConfig.StartingBlockNumber } From f8ff707f160573bb56210febd366612639f6a555 Mon Sep 17 00:00:00 2001 From: Elizabeth Date: Wed, 9 Sep 2020 15:27:11 -0500 Subject: [PATCH 36/66] Vdb 1528 decouple diff transforming (#131) * Add a GetUnrecognizedDiffs method to storage diff repo * Return transform err if it isn't a common/expected error * Start enabling branching on diff status in watcher * Pull out shared storage watcher execute test behaviors * Simiplify storage_watcher_test * Create test helper methods to return diffs and errors from mock diff repo * Switch on request diff status when getting diffs from repo * Add watcher for unrecognized diffs to execute * Allow config to update starting block for unrecognized diff transformation * Apply gofmt fixes * Fix storage_watcher_test formatting * Clean up execute cmd --- cmd/execute.go | 20 +- cmd/root.go | 27 +- .../shared/mocks/storage_diff_repository.go | 22 +- libraries/shared/storage/diff_repository.go | 25 +- .../shared/storage/diff_repository_test.go | 101 ++++- libraries/shared/watcher/storage_watcher.go | 63 ++- .../shared/watcher/storage_watcher_test.go | 375 ++++++++++-------- 7 files changed, 422 insertions(+), 211 deletions(-) diff --git a/cmd/execute.go b/cmd/execute.go index b77ead1b7..a8641e017 100644 --- a/cmd/execute.go +++ b/cmd/execute.go @@ -77,7 +77,8 @@ func init() { executeCmd.Flags().BoolVarP(&recheckHeadersArg, "recheck-headers", "r", false, "whether to re-check headers for watched events") executeCmd.Flags().DurationVarP(&retryInterval, "retry-interval", "i", 7*time.Second, "interval duration between retries on execution error") executeCmd.Flags().IntVarP(&maxUnexpectedErrors, "max-unexpected-errs", "m", 5, "maximum number of unexpected errors to allow (with retries) before exiting") - executeCmd.Flags().Int64VarP(&diffBlockFromHeadOfChain, "diff-blocks-from-head", "d", -1, "number of blocks from head of chain to start reprocessing diffs, defaults to -1 so all diffs are processsed") + executeCmd.Flags().Int64VarP(&newDiffBlockFromHeadOfChain, "new-diff-blocks-from-head", "d", -1, "number of blocks from head of chain to start reprocessing new diffs, defaults to -1 so all diffs are processsed") + executeCmd.Flags().Int64VarP(&unrecognizedDiffBlockFromHeadOfChain, "unrecognized-diff-blocks-from-head", "u", -1, "number of blocks from head of chain to start reprocessing unrecognized diffs, defaults to -1 so all diffs are processsed") } func executeTransformers() { @@ -109,12 +110,19 @@ func executeTransformers() { } if len(ethStorageInitializers) > 0 { - storageHealthCheckMessage := []byte("storage watcher starting\n") - statusWriter := fs.NewStatusWriter(healthCheckFile, storageHealthCheckMessage) - sw := watcher.NewStorageWatcher(&db, diffBlockFromHeadOfChain, statusWriter) - sw.AddTransformers(ethStorageInitializers) + newDiffStorageHealthCheckMessage := []byte("storage watcher for new diffs starting\n") + newDiffStatusWriter := fs.NewStatusWriter(healthCheckFile, newDiffStorageHealthCheckMessage) + newDiffStorageWatcher := watcher.NewStorageWatcher(&db, newDiffBlockFromHeadOfChain, newDiffStatusWriter, watcher.New) + newDiffStorageWatcher.AddTransformers(ethStorageInitializers) wg.Add(1) - go watchEthStorage(&sw, &wg) + go watchEthStorage(&newDiffStorageWatcher, &wg) + + unrecognizedDiffStorageHealthCheckMessage := []byte("storage watcher for unrecognized diffs starting\n") + unrecognizedDiffStatusWriter := fs.NewStatusWriter(healthCheckFile, unrecognizedDiffStorageHealthCheckMessage) + unrecognizedDiffStorageWatcher := watcher.NewStorageWatcher(&db, unrecognizedDiffBlockFromHeadOfChain, unrecognizedDiffStatusWriter, watcher.Unrecognized) + unrecognizedDiffStorageWatcher.AddTransformers(ethStorageInitializers) + wg.Add(1) + go watchEthStorage(&unrecognizedDiffStorageWatcher, &wg) } if len(ethContractInitializers) > 0 { diff --git a/cmd/root.go b/cmd/root.go index 085e7236a..123a7b2c5 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -40,19 +40,20 @@ import ( ) var ( - LogWithCommand logrus.Entry - SubCommand string - cfgFile string - databaseConfig config.Database - diffBlockFromHeadOfChain int64 - genConfig config.Plugin - ipc string - maxUnexpectedErrors int - recheckHeadersArg bool - retryInterval time.Duration - startingBlockNumber int64 - storageDiffsPath string - storageDiffsSource string + LogWithCommand logrus.Entry + SubCommand string + cfgFile string + databaseConfig config.Database + newDiffBlockFromHeadOfChain int64 + unrecognizedDiffBlockFromHeadOfChain int64 + genConfig config.Plugin + ipc string + maxUnexpectedErrors int + recheckHeadersArg bool + retryInterval time.Duration + startingBlockNumber int64 + storageDiffsPath string + storageDiffsSource string ) const ( diff --git a/libraries/shared/mocks/storage_diff_repository.go b/libraries/shared/mocks/storage_diff_repository.go index 8afd2bac8..df96db43c 100644 --- a/libraries/shared/mocks/storage_diff_repository.go +++ b/libraries/shared/mocks/storage_diff_repository.go @@ -24,11 +24,15 @@ type MockStorageDiffRepository struct { CreateBackFilledStorageValuePassedRawDiffs []types.RawDiff CreateBackFilledStorageValueReturnError error CreatePassedRawDiffs []types.RawDiff - GetNewDiffsDiffs []types.PersistedDiff + GetNewDiffsToReturn []types.PersistedDiff GetNewDiffsErrors []error GetNewDiffsPassedMinIDs []int GetNewDiffsPassedLimits []int - MarkCheckedPassedID int64 + GetUnrecognizedDiffsToReturn []types.PersistedDiff + GetUnrecognizedDiffsErrors []error + GetUnrecognizedDiffsPassedMinIDs []int + GetUnrecognizedDiffsPassedLimits []int + MarkTransformedPassedID int64 MarkUnrecognizedPassedID int64 MarkNoncanonicalPassedID int64 MarkUnwatchedPassedID int64 @@ -54,11 +58,21 @@ func (repository *MockStorageDiffRepository) GetNewDiffs(minID, limit int) ([]ty if len(repository.GetNewDiffsErrors) > 1 { repository.GetNewDiffsErrors = repository.GetNewDiffsErrors[1:] } - return repository.GetNewDiffsDiffs, err + return repository.GetNewDiffsToReturn, err +} + +func (repository *MockStorageDiffRepository) GetUnrecognizedDiffs(minID, limit int) ([]types.PersistedDiff, error) { + repository.GetUnrecognizedDiffsPassedMinIDs = append(repository.GetUnrecognizedDiffsPassedMinIDs, minID) + repository.GetUnrecognizedDiffsPassedLimits = append(repository.GetUnrecognizedDiffsPassedLimits, limit) + err := repository.GetUnrecognizedDiffsErrors[0] + if len(repository.GetUnrecognizedDiffsErrors) > 1 { + repository.GetUnrecognizedDiffsErrors = repository.GetUnrecognizedDiffsErrors[1:] + } + return repository.GetUnrecognizedDiffsToReturn, err } func (repository *MockStorageDiffRepository) MarkTransformed(id int64) error { - repository.MarkCheckedPassedID = id + repository.MarkTransformedPassedID = id return nil } diff --git a/libraries/shared/storage/diff_repository.go b/libraries/shared/storage/diff_repository.go index b550a1271..79c47e64e 100644 --- a/libraries/shared/storage/diff_repository.go +++ b/libraries/shared/storage/diff_repository.go @@ -27,6 +27,7 @@ type DiffRepository interface { CreateStorageDiff(rawDiff types.RawDiff) (int64, error) CreateBackFilledStorageValue(rawDiff types.RawDiff) error GetNewDiffs(minID, limit int) ([]types.PersistedDiff, error) + GetUnrecognizedDiffs(minID, limit int) ([]types.PersistedDiff, error) MarkTransformed(id int64) error MarkNoncanonical(id int64) error MarkUnrecognized(id int64) error @@ -78,12 +79,28 @@ func (repository diffRepository) GetNewDiffs(minID, limit int) ([]types.Persiste var result []types.PersistedDiff err := repository.db.Select( &result, - `SELECT id, address, block_height, block_hash, storage_key, storage_value, eth_node_id, status, from_backfill - FROM public.storage_diff WHERE (status = $1 OR status = $2) AND id > $3 ORDER BY id ASC LIMIT $4`, - New, Unrecognized, minID, limit, + `SELECT id, address, block_height, block_hash, storage_key, storage_value, eth_node_id, status, from_backfill + FROM public.storage_diff + WHERE status = $1 AND id > $2 ORDER BY id ASC LIMIT $3`, + New, minID, limit, ) if err != nil { - return nil, fmt.Errorf("error getting unchecked storage diffs with id greater than %d: %w", minID, err) + return nil, fmt.Errorf("error getting new storage diffs with id greater than %d: %w", minID, err) + } + return result, nil +} + +func (repository diffRepository) GetUnrecognizedDiffs(minID, limit int) ([]types.PersistedDiff, error) { + var result []types.PersistedDiff + err := repository.db.Select( + &result, + `SELECT id, address, block_height, block_hash, storage_key, storage_value, eth_node_id, status, from_backfill + FROM public.storage_diff + WHERE status = $1 AND id > $2 ORDER BY id ASC LIMIT $3`, + Unrecognized, minID, limit, + ) + if err != nil { + return nil, fmt.Errorf("error getting unrecognized storage diffs with id greater than %d: %w", minID, err) } return result, nil } diff --git a/libraries/shared/storage/diff_repository_test.go b/libraries/shared/storage/diff_repository_test.go index 536df7522..497259a54 100644 --- a/libraries/shared/storage/diff_repository_test.go +++ b/libraries/shared/storage/diff_repository_test.go @@ -230,7 +230,7 @@ var _ = Describe("Storage diffs repository", func() { Expect(diffs).To(ConsistOf(fakePersistedDiff)) }) - It("sends diffs that are marked as 'unrecognized'", func() { + It("does not sends diffs that are marked as 'unrecognized'", func() { unrecognizedPersistedDiff := types.PersistedDiff{ RawDiff: fakeStorageDiff, ID: rand.Int63(), @@ -242,10 +242,10 @@ var _ = Describe("Storage diffs repository", func() { diffs, err := repo.GetNewDiffs(0, 1) Expect(err).NotTo(HaveOccurred()) - Expect(diffs).To(ConsistOf(unrecognizedPersistedDiff)) + Expect(diffs).To(BeEmpty()) }) - It("does not send diffs that are marked as transformed", func() { + It("does not send diffs that are marked as 'transformed'", func() { transformedPersistedDiff := types.PersistedDiff{ RawDiff: fakeStorageDiff, ID: rand.Int63(), @@ -260,7 +260,7 @@ var _ = Describe("Storage diffs repository", func() { Expect(diffs).To(BeEmpty()) }) - It("does not send diffs that are marked as noncanonical", func() { + It("does not send diffs that are marked as 'noncanonical'", func() { noncanonicalPersistedDiff := types.PersistedDiff{ RawDiff: fakeStorageDiff, ID: rand.Int63(), @@ -307,6 +307,99 @@ var _ = Describe("Storage diffs repository", func() { }) }) + Describe("GetUnrecognizedDiffs", func() { + It("sends diffs that are marked as 'unrecognized'", func() { + fakePersistedDiff := types.PersistedDiff{ + RawDiff: fakeStorageDiff, + ID: rand.Int63(), + EthNodeID: db.NodeID, + Status: "unrecognized", + } + insertTestDiff(fakePersistedDiff, db) + + diffs, err := repo.GetUnrecognizedDiffs(0, 1) + + Expect(err).NotTo(HaveOccurred()) + Expect(diffs).To(ConsistOf(fakePersistedDiff)) + }) + + It("does not sends diffs that are marked as 'new'", func() { + unrecognizedPersistedDiff := types.PersistedDiff{ + RawDiff: fakeStorageDiff, + ID: rand.Int63(), + Status: "new", + EthNodeID: db.NodeID, + } + insertTestDiff(unrecognizedPersistedDiff, db) + + diffs, err := repo.GetUnrecognizedDiffs(0, 1) + + Expect(err).NotTo(HaveOccurred()) + Expect(diffs).To(BeEmpty()) + }) + + It("does not send diffs that are marked as 'transformed'", func() { + transformedPersistedDiff := types.PersistedDiff{ + RawDiff: fakeStorageDiff, + ID: rand.Int63(), + Status: "transformed", + EthNodeID: db.NodeID, + } + insertTestDiff(transformedPersistedDiff, db) + + diffs, err := repo.GetUnrecognizedDiffs(0, 1) + + Expect(err).NotTo(HaveOccurred()) + Expect(diffs).To(BeEmpty()) + }) + + It("does not send diffs that are marked as 'noncanonical'", func() { + noncanonicalPersistedDiff := types.PersistedDiff{ + RawDiff: fakeStorageDiff, + ID: rand.Int63(), + Status: "noncanonical", + EthNodeID: db.NodeID, + } + insertTestDiff(noncanonicalPersistedDiff, db) + + diffs, err := repo.GetUnrecognizedDiffs(0, 1) + + Expect(err).NotTo(HaveOccurred()) + Expect(diffs).To(BeEmpty()) + }) + + It("enables seeking diffs with greater ID", func() { + blockZero := rand.Int() + for i := 0; i < 2; i++ { + fakeRawDiff := types.RawDiff{ + Address: test_data.FakeAddress(), + BlockHash: test_data.FakeHash(), + BlockHeight: blockZero + i, + StorageKey: test_data.FakeHash(), + StorageValue: test_data.FakeHash(), + } + persistedDiff := types.PersistedDiff{ + RawDiff: fakeRawDiff, + ID: rand.Int63(), + Status: "new", + EthNodeID: db.NodeID, + } + insertTestDiff(persistedDiff, db) + } + + minID := 0 + limit := 1 + diffsOne, errOne := repo.GetNewDiffs(minID, limit) + Expect(errOne).NotTo(HaveOccurred()) + Expect(len(diffsOne)).To(Equal(1)) + nextID := int(diffsOne[0].ID) + diffsTwo, errTwo := repo.GetNewDiffs(nextID, limit) + Expect(errTwo).NotTo(HaveOccurred()) + Expect(len(diffsTwo)).To(Equal(1)) + Expect(int(diffsTwo[0].ID) > nextID).To(BeTrue()) + }) + }) + Describe("Changing the diff status", func() { var fakePersistedDiff types.PersistedDiff BeforeEach(func() { diff --git a/libraries/shared/watcher/storage_watcher.go b/libraries/shared/watcher/storage_watcher.go index 5ccd37562..f2f42e839 100644 --- a/libraries/shared/watcher/storage_watcher.go +++ b/libraries/shared/watcher/storage_watcher.go @@ -50,9 +50,17 @@ type StorageWatcher struct { StorageDiffRepository storage.DiffRepository DiffBlocksFromHeadOfChain int64 // the number of blocks from the head of the chain where diffs should be processed StatusWriter fs.StatusWriter + DiffStatus DiffStatusToWatch } -func NewStorageWatcher(db *postgres.DB, backFromHeadOfChain int64, statusWriter fs.StatusWriter) StorageWatcher { +type DiffStatusToWatch int + +const ( + New DiffStatusToWatch = iota + Unrecognized +) + +func NewStorageWatcher(db *postgres.DB, backFromHeadOfChain int64, statusWriter fs.StatusWriter, diffStatusToWatch DiffStatusToWatch) StorageWatcher { headerRepository := repositories.NewHeaderRepository(db) storageDiffRepository := storage.NewDiffRepository(db) transformers := make(map[common.Address]storage2.ITransformer) @@ -63,6 +71,7 @@ func NewStorageWatcher(db *postgres.DB, backFromHeadOfChain int64, statusWriter StorageDiffRepository: storageDiffRepository, DiffBlocksFromHeadOfChain: backFromHeadOfChain, StatusWriter: statusWriter, + DiffStatus: diffStatusToWatch, } } @@ -88,26 +97,14 @@ func (watcher StorageWatcher) Execute() error { } } -func (watcher StorageWatcher) getMinDiffID() (int, error) { - var minID = 0 - if watcher.DiffBlocksFromHeadOfChain != -1 { - mostRecentHeaderBlockNumber, getHeaderErr := watcher.HeaderRepository.GetMostRecentHeaderBlockNumber() - if getHeaderErr != nil { - return 0, fmt.Errorf("error getting most recent header block number: %w", getHeaderErr) - } - blockNumber := mostRecentHeaderBlockNumber - watcher.DiffBlocksFromHeadOfChain - diffID, getDiffErr := watcher.StorageDiffRepository.GetFirstDiffIDForBlockHeight(blockNumber) - if getDiffErr != nil { - return 0, fmt.Errorf("error getting first diff id for block height %d: %w", blockNumber, getDiffErr) - } - - // We are subtracting an offset from the diffID because it will be passed to GetNewDiffs which returns diffs with ids - // greater than id passed in (minID), and we want to make sure that this diffID here is included in that collection - diffOffset := int64(1) - minID = int(diffID - diffOffset) +func (watcher StorageWatcher) getDiffs(minID, ResultsLimit int) ([]types.PersistedDiff, error) { + switch watcher.DiffStatus { + case New: + return watcher.StorageDiffRepository.GetNewDiffs(minID, ResultsLimit) + case Unrecognized: + return watcher.StorageDiffRepository.GetUnrecognizedDiffs(minID, ResultsLimit) } - - return minID, nil + return nil, errors.New("Unrecognized diff status") } func (watcher StorageWatcher) transformDiffs() error { @@ -117,7 +114,8 @@ func (watcher StorageWatcher) transformDiffs() error { } for { - diffs, extractErr := watcher.StorageDiffRepository.GetNewDiffs(minID, ResultsLimit) + diffs, extractErr := watcher.getDiffs(minID, ResultsLimit) + if extractErr != nil { return fmt.Errorf("error getting new diffs: %w", extractErr) } @@ -137,6 +135,27 @@ func (watcher StorageWatcher) transformDiffs() error { } } +func (watcher StorageWatcher) getMinDiffID() (int, error) { + var minID = 0 + if watcher.DiffBlocksFromHeadOfChain != -1 { + mostRecentHeaderBlockNumber, getHeaderErr := watcher.HeaderRepository.GetMostRecentHeaderBlockNumber() + if getHeaderErr != nil { + return 0, fmt.Errorf("error getting most recent header block number: %w", getHeaderErr) + } + blockNumber := mostRecentHeaderBlockNumber - watcher.DiffBlocksFromHeadOfChain + diffID, getDiffErr := watcher.StorageDiffRepository.GetFirstDiffIDForBlockHeight(blockNumber) + if getDiffErr != nil { + return 0, fmt.Errorf("error getting first diff id for block height %d: %w", blockNumber, getDiffErr) + } + + // We are subtracting an offset from the diffID because it will be passed to GetNewDiffs which returns diffs with ids + // greater than id passed in (minID), and we want to make sure that this diffID here is included in that collection + diffOffset := int64(1) + minID = int(diffID - diffOffset) + } + + return minID, nil +} func (watcher StorageWatcher) transformDiff(diff types.PersistedDiff) error { t, watching := watcher.getTransformer(diff) if !watching { @@ -213,7 +232,7 @@ func (watcher StorageWatcher) handleTransformError(transformErr error, diff type logrus.Infof("error transforming diff: %s", transformErr.Error()) } } - return nil + return transformErr } func isCommonTransformError(err error) bool { diff --git a/libraries/shared/watcher/storage_watcher_test.go b/libraries/shared/watcher/storage_watcher_test.go index 3a50f7856..a174d3bf0 100644 --- a/libraries/shared/watcher/storage_watcher_test.go +++ b/libraries/shared/watcher/storage_watcher_test.go @@ -39,7 +39,7 @@ var _ = Describe("Storage Watcher", func() { It("adds transformers", func() { fakeAddress := fakes.FakeAddress fakeTransformer := &mocks.MockStorageTransformer{Address: fakeAddress} - w := watcher.NewStorageWatcher(test_config.NewTestDB(test_config.NewTestNode()), -1, &statusWriter) + w := watcher.NewStorageWatcher(test_config.NewTestDB(test_config.NewTestNode()), -1, &statusWriter, watcher.New) w.AddTransformers([]storage.TransformerInitializer{fakeTransformer.FakeTransformerInitializer}) @@ -48,36 +48,72 @@ var _ = Describe("Storage Watcher", func() { }) Describe("Execute", func() { - var ( - storageWatcher watcher.StorageWatcher - mockDiffsRepository *mocks.MockStorageDiffRepository - mockHeaderRepository *fakes.MockHeaderRepository - ) - - BeforeEach(func() { - mockDiffsRepository = &mocks.MockStorageDiffRepository{} - mockHeaderRepository = &fakes.MockHeaderRepository{} - storageWatcher = watcher.NewStorageWatcher(test_config.NewTestDB(test_config.NewTestNode()), -1, &statusWriter) - storageWatcher.HeaderRepository = mockHeaderRepository - storageWatcher.StorageDiffRepository = mockDiffsRepository + When("a watcher is configured to watches 'new' storage diffs", func() { + statusWriter := fakes.MockStatusWriter{} + storageWatcher := watcher.NewStorageWatcher(test_config.NewTestDB(test_config.NewTestNode()), -1, &statusWriter, watcher.New) + input := ExecuteInput{ + watcher: &storageWatcher, + statusWriter: &statusWriter, + } + SharedExecuteBehavior(&input) + }) + + When("a watcher is configured to watches 'unrecognized' storage diffs", func() { + statusWriter := fakes.MockStatusWriter{} + storageWatcher := watcher.NewStorageWatcher(test_config.NewTestDB(test_config.NewTestNode()), -1, &statusWriter, watcher.Unrecognized) + input := ExecuteInput{ + watcher: &storageWatcher, + statusWriter: &statusWriter, + } + SharedExecuteBehavior(&input) }) + }) +}) + +type ExecuteInput struct { + watcher *watcher.StorageWatcher + statusWriter *fakes.MockStatusWriter +} + +func SharedExecuteBehavior(input *ExecuteInput) { + var ( + mockDiffsRepository *mocks.MockStorageDiffRepository + mockHeaderRepository *fakes.MockHeaderRepository + statusWriter = input.statusWriter + storageWatcher = input.watcher + contractAddress common.Address + mockTransformer *mocks.MockStorageTransformer + ) + + BeforeEach(func() { + mockDiffsRepository = &mocks.MockStorageDiffRepository{} + mockHeaderRepository = &fakes.MockHeaderRepository{} + contractAddress = test_data.FakeAddress() + mockTransformer = &mocks.MockStorageTransformer{Address: contractAddress} + storageWatcher.HeaderRepository = mockHeaderRepository + storageWatcher.StorageDiffRepository = mockDiffsRepository + storageWatcher.AddTransformers([]storage.TransformerInitializer{mockTransformer.FakeTransformerInitializer}) + }) + Describe("Execute", func() { It("creates file for health check", func() { - mockDiffsRepository.GetNewDiffsErrors = []error{fakes.FakeError} + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{fakes.FakeError}) err := storageWatcher.Execute() Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) Expect(statusWriter.WriteCalled).To(BeTrue()) }) It("fetches diffs with results limit", func() { - mockDiffsRepository.GetNewDiffsErrors = []error{fakes.FakeError} + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{fakes.FakeError}) err := storageWatcher.Execute() Expect(err).To(HaveOccurred()) - Expect(mockDiffsRepository.GetNewDiffsPassedLimits).To(ConsistOf(watcher.ResultsLimit)) + Expect(err).To(MatchError(fakes.FakeError)) + assertGetDiffsLimits(storageWatcher.DiffStatus, mockDiffsRepository, watcher.ResultsLimit) }) It("fetches diffs with min ID from subsequent queries when previous query returns max results", func() { @@ -93,13 +129,14 @@ var _ = Describe("Storage Watcher", func() { } diffs = append(diffs, diff) } - mockDiffsRepository.GetNewDiffsDiffs = diffs - mockDiffsRepository.GetNewDiffsErrors = []error{nil, fakes.FakeError} + setDiffsToReturn(storageWatcher.DiffStatus, mockDiffsRepository, diffs) + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{nil, fakes.FakeError}) err := storageWatcher.Execute() Expect(err).To(HaveOccurred()) - Expect(mockDiffsRepository.GetNewDiffsPassedMinIDs).To(ConsistOf(0, diffID)) + Expect(err).To(MatchError(fakes.FakeError)) + assertGetDiffsMinIDs(storageWatcher.DiffStatus, mockDiffsRepository, []int{0, diffID}) }) It("resets min ID to zero when previous query returns fewer than max results", func() { @@ -115,24 +152,26 @@ var _ = Describe("Storage Watcher", func() { } diffs = append(diffs, diff) } - mockDiffsRepository.GetNewDiffsDiffs = diffs - mockDiffsRepository.GetNewDiffsErrors = []error{nil, fakes.FakeError} + setDiffsToReturn(storageWatcher.DiffStatus, mockDiffsRepository, diffs) + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{nil, fakes.FakeError}) err := storageWatcher.Execute() Expect(err).To(HaveOccurred()) - Expect(mockDiffsRepository.GetNewDiffsPassedMinIDs).To(ConsistOf(0, 0)) + Expect(err).To(MatchError(fakes.FakeError)) + assertGetDiffsMinIDs(storageWatcher.DiffStatus, mockDiffsRepository, []int{0, 0}) }) - It("marks diff as unwatched if no transformer is watching its address", func() { + It("marks diff as 'unwatched' if no transformer is watching its address", func() { + unwatchedAddress := test_data.FakeAddress() unwatchedDiff := types.PersistedDiff{ RawDiff: types.RawDiff{ - Address: test_data.FakeAddress(), + Address: unwatchedAddress, }, ID: rand.Int63(), } - mockDiffsRepository.GetNewDiffsDiffs = []types.PersistedDiff{unwatchedDiff} - mockDiffsRepository.GetNewDiffsErrors = []error{nil, fakes.FakeError} + setDiffsToReturn(storageWatcher.DiffStatus, mockDiffsRepository, []types.PersistedDiff{unwatchedDiff}) + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{nil, fakes.FakeError}) err := storageWatcher.Execute() @@ -141,18 +180,34 @@ var _ = Describe("Storage Watcher", func() { Expect(mockDiffsRepository.MarkUnwatchedPassedID).To(Equal(unwatchedDiff.ID)) }) + It("does not change a diff's status if there's no header for the given block number", func() { + diffWithoutHeader := types.PersistedDiff{ + RawDiff: types.RawDiff{ + Address: contractAddress, + BlockHash: test_data.FakeHash(), + BlockHeight: rand.Int(), + }, + ID: rand.Int63(), + } + setDiffsToReturn(storageWatcher.DiffStatus, mockDiffsRepository, []types.PersistedDiff{diffWithoutHeader}) + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{nil, fakes.FakeError}) + mockHeaderRepository.GetHeaderByBlockNumberError = sql.ErrNoRows + + err := storageWatcher.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(sql.ErrNoRows)) + Expect(mockDiffsRepository.MarkTransformedPassedID).NotTo(Equal(diffWithoutHeader.ID)) + }) + Describe("When the watcher is configured to skip old diffs", func() { var diffs []types.PersistedDiff var numberOfBlocksFromHeadOfChain = int64(500) BeforeEach(func() { - storageWatcher = watcher.StorageWatcher{ - HeaderRepository: mockHeaderRepository, - StorageDiffRepository: mockDiffsRepository, - AddressTransformers: map[common.Address]storage.ITransformer{}, - DiffBlocksFromHeadOfChain: numberOfBlocksFromHeadOfChain, - StatusWriter: &statusWriter, - } + storageWatcher.AddressTransformers = map[common.Address]storage.ITransformer{} + storageWatcher.DiffBlocksFromHeadOfChain = numberOfBlocksFromHeadOfChain + diffID := rand.Int() for i := 0; i < watcher.ResultsLimit; i++ { diffID = diffID + i @@ -171,8 +226,8 @@ var _ = Describe("Storage Watcher", func() { mockHeaderRepository.MostRecentHeaderBlockNumber = headerBlockNumber mockDiffsRepository.GetFirstDiffIDToReturn = diffs[0].ID - mockDiffsRepository.GetNewDiffsDiffs = diffs - mockDiffsRepository.GetNewDiffsErrors = []error{nil, fakes.FakeError} + setDiffsToReturn(storageWatcher.DiffStatus, mockDiffsRepository, diffs) + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{nil, fakes.FakeError}) expectedFirstMinDiffID := int(diffs[0].ID - 1) expectedSecondMinDiffID := int(diffs[len(diffs)-1].ID) @@ -182,7 +237,7 @@ var _ = Describe("Storage Watcher", func() { Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) Expect(mockDiffsRepository.GetFirstDiffBlockHeightPassed).To(Equal(headerBlockNumber - numberOfBlocksFromHeadOfChain)) - Expect(mockDiffsRepository.GetNewDiffsPassedMinIDs).To(ConsistOf(expectedFirstMinDiffID, expectedSecondMinDiffID)) + assertGetDiffsMinIDs(storageWatcher.DiffStatus, mockDiffsRepository, []int{expectedFirstMinDiffID, expectedSecondMinDiffID}) }) It("resets min ID back to new min diff when previous query returns fewer than max results", func() { @@ -203,8 +258,8 @@ var _ = Describe("Storage Watcher", func() { mockHeaderRepository.MostRecentHeaderBlockNumber = headerBlockNumber mockDiffsRepository.GetFirstDiffIDToReturn = diffs[0].ID - mockDiffsRepository.GetNewDiffsDiffs = diffs - mockDiffsRepository.GetNewDiffsErrors = []error{nil, fakes.FakeError} + setDiffsToReturn(storageWatcher.DiffStatus, mockDiffsRepository, diffs) + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{nil, fakes.FakeError}) expectedFirstMinDiffID := int(diffs[0].ID - 1) @@ -213,13 +268,13 @@ var _ = Describe("Storage Watcher", func() { Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) Expect(mockDiffsRepository.GetFirstDiffBlockHeightPassed).To(Equal(headerBlockNumber - numberOfBlocksFromHeadOfChain)) - Expect(mockDiffsRepository.GetNewDiffsPassedMinIDs).To(ConsistOf(expectedFirstMinDiffID, expectedFirstMinDiffID)) + assertGetDiffsMinIDs(storageWatcher.DiffStatus, mockDiffsRepository, []int{expectedFirstMinDiffID, expectedFirstMinDiffID}) }) It("sets minID to 0 if there are no headers with the given block height", func() { mockHeaderRepository.MostRecentHeaderBlockNumberErr = sql.ErrNoRows - mockDiffsRepository.GetNewDiffsDiffs = diffs - mockDiffsRepository.GetNewDiffsErrors = []error{nil, fakes.FakeError} + setDiffsToReturn(storageWatcher.DiffStatus, mockDiffsRepository, diffs) + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{nil, fakes.FakeError}) err := storageWatcher.Execute() Expect(err).To(HaveOccurred()) @@ -227,13 +282,13 @@ var _ = Describe("Storage Watcher", func() { expectedFirstMinDiffID := 0 expectedSecondMinDiffID := int(diffs[len(diffs)-1].ID) - Expect(mockDiffsRepository.GetNewDiffsPassedMinIDs).To(ConsistOf(expectedFirstMinDiffID, expectedSecondMinDiffID)) + assertGetDiffsMinIDs(storageWatcher.DiffStatus, mockDiffsRepository, []int{expectedFirstMinDiffID, expectedSecondMinDiffID}) }) It("sets minID to 0 if there are no diffs with given block range", func() { mockDiffsRepository.GetFirstDiffIDErr = sql.ErrNoRows - mockDiffsRepository.GetNewDiffsDiffs = diffs - mockDiffsRepository.GetNewDiffsErrors = []error{nil, fakes.FakeError} + setDiffsToReturn(storageWatcher.DiffStatus, mockDiffsRepository, diffs) + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{nil, fakes.FakeError}) err := storageWatcher.Execute() Expect(err).To(HaveOccurred()) @@ -241,154 +296,158 @@ var _ = Describe("Storage Watcher", func() { expectedFirstMinDiffID := 0 expectedSecondMinDiffID := int(diffs[len(diffs)-1].ID) - Expect(mockDiffsRepository.GetNewDiffsPassedMinIDs).To(ConsistOf(expectedFirstMinDiffID, expectedSecondMinDiffID)) + assertGetDiffsMinIDs(storageWatcher.DiffStatus, mockDiffsRepository, []int{expectedFirstMinDiffID, expectedSecondMinDiffID}) }) }) - Describe("when diff's address is watched", func() { + Describe("When a header with a non-matching hash is found", func() { var ( - contractAddress common.Address - mockTransformer *mocks.MockStorageTransformer + blockNumber int + fakePersistedDiff types.PersistedDiff ) BeforeEach(func() { - contractAddress = test_data.FakeAddress() - mockTransformer = &mocks.MockStorageTransformer{Address: contractAddress} - storageWatcher.AddTransformers([]storage.TransformerInitializer{mockTransformer.FakeTransformerInitializer}) - }) + blockNumber = rand.Int() + fakeRawDiff := types.RawDiff{ + Address: contractAddress, + BlockHash: test_data.FakeHash(), + BlockHeight: blockNumber, + StorageKey: test_data.FakeHash(), + StorageValue: test_data.FakeHash(), + } + mockHeaderRepository.GetHeaderByBlockNumberReturnID = int64(blockNumber) + mockHeaderRepository.GetHeaderByBlockNumberReturnHash = test_data.FakeHash().Hex() - It("does not mark diff checked if no matching header", func() { - diffWithoutHeader := types.PersistedDiff{ - RawDiff: types.RawDiff{ - Address: contractAddress, - BlockHash: test_data.FakeHash(), - BlockHeight: rand.Int(), - }, - ID: rand.Int63(), + fakePersistedDiff = types.PersistedDiff{ + RawDiff: fakeRawDiff, + ID: rand.Int63(), } - mockDiffsRepository.GetNewDiffsDiffs = []types.PersistedDiff{diffWithoutHeader} - mockDiffsRepository.GetNewDiffsErrors = []error{nil, fakes.FakeError} - mockHeaderRepository.GetHeaderByBlockNumberError = errors.New("no matching header") + setDiffsToReturn(storageWatcher.DiffStatus, mockDiffsRepository, []types.PersistedDiff{fakePersistedDiff}) + }) + + It("does not change a diff's status if getting max known block height fails", func() { + maxHeaderErr := errors.New("getting max header failed") + mockHeaderRepository.MostRecentHeaderBlockNumberErr = maxHeaderErr + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{nil}) err := storageWatcher.Execute() Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) - Expect(mockDiffsRepository.MarkCheckedPassedID).NotTo(Equal(diffWithoutHeader.ID)) + Expect(err).To(MatchError(maxHeaderErr)) + Expect(mockDiffsRepository.MarkTransformedPassedID).NotTo(Equal(fakePersistedDiff.ID)) }) - Describe("when non-matching header found", func() { - var ( - blockNumber int - fakePersistedDiff types.PersistedDiff - ) - - BeforeEach(func() { - blockNumber = rand.Int() - fakeRawDiff := types.RawDiff{ - Address: contractAddress, - BlockHash: test_data.FakeHash(), - BlockHeight: blockNumber, - StorageKey: test_data.FakeHash(), - StorageValue: test_data.FakeHash(), - } - mockHeaderRepository.GetHeaderByBlockNumberReturnID = int64(blockNumber) - mockHeaderRepository.GetHeaderByBlockNumberReturnHash = test_data.FakeHash().Hex() - - fakePersistedDiff = types.PersistedDiff{ - RawDiff: fakeRawDiff, - ID: rand.Int63(), - } - mockDiffsRepository.GetNewDiffsDiffs = []types.PersistedDiff{fakePersistedDiff} - }) - - It("does not mark diff checked if getting max known block height fails", func() { - mockHeaderRepository.MostRecentHeaderBlockNumberErr = errors.New("getting max header failed") - mockDiffsRepository.GetNewDiffsErrors = []error{nil, fakes.FakeError} + It("marks diff 'noncanonical' if block height less than max known block height minus reorg window", func() { + mockHeaderRepository.MostRecentHeaderBlockNumber = int64(blockNumber + watcher.ReorgWindow + 1) + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{nil, fakes.FakeError}) - err := storageWatcher.Execute() - - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) - Expect(mockDiffsRepository.MarkCheckedPassedID).NotTo(Equal(fakePersistedDiff.ID)) - }) - - It("marks diff noncanonical if block height less than max known block height minus reorg window", func() { - mockHeaderRepository.MostRecentHeaderBlockNumber = int64(blockNumber + watcher.ReorgWindow + 1) - mockDiffsRepository.GetNewDiffsErrors = []error{nil, fakes.FakeError} - - err := storageWatcher.Execute() + err := storageWatcher.Execute() - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) - Expect(mockDiffsRepository.MarkNoncanonicalPassedID).To(Equal(fakePersistedDiff.ID)) - }) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + Expect(mockDiffsRepository.MarkNoncanonicalPassedID).To(Equal(fakePersistedDiff.ID)) + }) - It("does not mark diff checked if block height is within reorg window", func() { - mockHeaderRepository.MostRecentHeaderBlockNumber = int64(blockNumber + watcher.ReorgWindow) - mockDiffsRepository.GetNewDiffsErrors = []error{nil, fakes.FakeError} + It("does not mark diff as 'noncanonical' if block height is within reorg window", func() { + mockHeaderRepository.MostRecentHeaderBlockNumber = int64(blockNumber + watcher.ReorgWindow) + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{nil, fakes.FakeError}) - err := storageWatcher.Execute() + err := storageWatcher.Execute() - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) - Expect(mockDiffsRepository.MarkCheckedPassedID).NotTo(Equal(fakePersistedDiff.ID)) - }) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + Expect(mockDiffsRepository.MarkTransformedPassedID).NotTo(Equal(fakePersistedDiff.ID)) }) + }) - Describe("when matching header exists", func() { - var fakePersistedDiff types.PersistedDiff + Describe("When a header with a matching hash exists", func() { + var fakePersistedDiff types.PersistedDiff - BeforeEach(func() { - fakeBlockHash := test_data.FakeHash() - fakeRawDiff := types.RawDiff{ - Address: contractAddress, - BlockHash: fakeBlockHash, - } + BeforeEach(func() { + fakeBlockHash := test_data.FakeHash() + fakeRawDiff := types.RawDiff{ + Address: contractAddress, + BlockHash: fakeBlockHash, + } - mockHeaderRepository.GetHeaderByBlockNumberReturnID = rand.Int63() - mockHeaderRepository.GetHeaderByBlockNumberReturnHash = fakeBlockHash.Hex() + mockHeaderRepository.GetHeaderByBlockNumberReturnID = rand.Int63() + mockHeaderRepository.GetHeaderByBlockNumberReturnHash = fakeBlockHash.Hex() - fakePersistedDiff = types.PersistedDiff{ - RawDiff: fakeRawDiff, - ID: rand.Int63(), - } - mockDiffsRepository.GetNewDiffsDiffs = []types.PersistedDiff{fakePersistedDiff} - }) + fakePersistedDiff = types.PersistedDiff{ + RawDiff: fakeRawDiff, + ID: rand.Int63(), + } + setDiffsToReturn(storageWatcher.DiffStatus, mockDiffsRepository, []types.PersistedDiff{fakePersistedDiff}) + }) - It("does not mark diff checked if transformer execution fails", func() { - mockTransformer.ExecuteErr = errors.New("execute failed") - mockDiffsRepository.GetNewDiffsErrors = []error{nil, fakes.FakeError} + It("does not change diff's status if transformer execution fails", func() { + executeErr := errors.New("execute failed") + mockTransformer.ExecuteErr = executeErr + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{nil}) - err := storageWatcher.Execute() + err := storageWatcher.Execute() - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) - Expect(mockDiffsRepository.MarkCheckedPassedID).NotTo(Equal(fakePersistedDiff.ID)) - }) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(executeErr)) + Expect(mockDiffsRepository.MarkTransformedPassedID).NotTo(Equal(fakePersistedDiff.ID)) + }) - It("marks diff as 'unrecognized' when transforming the diff returns a ErrKeyNotFound error", func() { - mockTransformer.ExecuteErr = types.ErrKeyNotFound - mockDiffsRepository.GetNewDiffsErrors = []error{nil, types.ErrKeyNotFound} + It("marks diff as 'unrecognized' when transforming the diff returns a ErrKeyNotFound error", func() { + mockTransformer.ExecuteErr = types.ErrKeyNotFound + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{nil, types.ErrKeyNotFound}) - err := storageWatcher.Execute() + err := storageWatcher.Execute() - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(types.ErrKeyNotFound)) - Expect(mockDiffsRepository.MarkUnrecognizedPassedID).To(Equal(fakePersistedDiff.ID)) - }) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(types.ErrKeyNotFound)) + Expect(mockDiffsRepository.MarkUnrecognizedPassedID).To(Equal(fakePersistedDiff.ID)) + }) - It("marks diff checked if transformer execution doesn't fail", func() { - mockDiffsRepository.GetNewDiffsDiffs = []types.PersistedDiff{fakePersistedDiff} - mockDiffsRepository.GetNewDiffsErrors = []error{nil, fakes.FakeError} + It("marks diff transformed if transformer execution doesn't fail", func() { + setDiffsToReturn(storageWatcher.DiffStatus, mockDiffsRepository, []types.PersistedDiff{fakePersistedDiff}) + setGetDiffsErrors(storageWatcher.DiffStatus, mockDiffsRepository, []error{nil, fakes.FakeError}) - err := storageWatcher.Execute() + err := storageWatcher.Execute() - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) - Expect(mockDiffsRepository.MarkCheckedPassedID).To(Equal(fakePersistedDiff.ID)) - }) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + Expect(mockDiffsRepository.MarkTransformedPassedID).To(Equal(fakePersistedDiff.ID)) }) }) }) -}) +} + +func setGetDiffsErrors(diffStatus watcher.DiffStatusToWatch, mockDiffsRepo *mocks.MockStorageDiffRepository, diffErrors []error) { + switch diffStatus { + case watcher.New: + mockDiffsRepo.GetNewDiffsErrors = diffErrors + case watcher.Unrecognized: + mockDiffsRepo.GetUnrecognizedDiffsErrors = diffErrors + } +} +func setDiffsToReturn(diffStatus watcher.DiffStatusToWatch, mockDiffsRepo *mocks.MockStorageDiffRepository, diffs []types.PersistedDiff) { + switch diffStatus { + case watcher.New: + mockDiffsRepo.GetNewDiffsToReturn = diffs + case watcher.Unrecognized: + mockDiffsRepo.GetUnrecognizedDiffsToReturn = diffs + } +} + +func assertGetDiffsLimits(diffStatus watcher.DiffStatusToWatch, mockDiffRepo *mocks.MockStorageDiffRepository, watcherLimit int) { + switch diffStatus { + case watcher.New: + Expect(mockDiffRepo.GetNewDiffsPassedLimits).To(ConsistOf(watcherLimit)) + case watcher.Unrecognized: + Expect(mockDiffRepo.GetUnrecognizedDiffsPassedLimits).To(ConsistOf(watcherLimit)) + } +} + +func assertGetDiffsMinIDs(diffStatus watcher.DiffStatusToWatch, mockDiffRepo *mocks.MockStorageDiffRepository, passedMinIDs []int) { + switch diffStatus { + case watcher.New: + Expect(mockDiffRepo.GetNewDiffsPassedMinIDs).To(ConsistOf(passedMinIDs)) + case watcher.Unrecognized: + Expect(mockDiffRepo.GetUnrecognizedDiffsPassedMinIDs).To(ConsistOf(passedMinIDs)) + } +} From b5dfa1c59bdc80dda9e83daa2973bb0c3d49b7fa Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Mon, 24 Aug 2020 16:58:15 -0500 Subject: [PATCH 37/66] Update to goose module 2.7.0-rc5 This is release candidate 5. To ensure that you're using the right version of goose use the Makefile, which uses go run to ensure it will pull the binary from the same version as the module. --- Makefile | 17 +++++++---------- go.mod | 32 +++++++++++++++++++++++++++++--- go.sum | 18 ++++++++++++++++++ 3 files changed, 54 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index cabc1e4df..f5c5adbbc 100644 --- a/Makefile +++ b/Makefile @@ -9,10 +9,7 @@ $(BIN)/ginkgo: go get -u github.com/onsi/ginkgo/ginkgo ## Migration tool -GOOSE = $(BIN)/goose -$(BIN)/goose: - go get -u -d github.com/pressly/goose/cmd/goose - go build -tags='no_mysql no_sqlite' -o $(BIN)/goose github.com/pressly/goose/cmd/goose +GOOSE = go run -tags='no_mysql no_sqlite3 no_mssql no_redshift' github.com/pressly/goose/cmd/goose ## Source linter LINT = $(BIN)/golint @@ -27,7 +24,7 @@ $(BIN)/gometalinter.v2: .PHONY: installtools -installtools: | $(LINT) $(GOOSE) $(GINKGO) +installtools: | $(LINT) $(GINKGO) echo "Installing tools" .PHONY: metalint @@ -100,30 +97,30 @@ checkmigname: # Migration operations ## Rollback the last migration .PHONY: rollback -rollback: $(GOOSE) checkdbvars +rollback: checkdbvars $(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" down pg_dump -O -s $(CONNECT_STRING) > db/schema.sql ## Rollbackt to a select migration (id/timestamp) .PHONY: rollback_to -rollback_to: $(GOOSE) checkmigration checkdbvars +rollback_to: checkmigration checkdbvars $(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" down-to "$(MIGRATION)" ## Apply all migrations not already run .PHONY: migrate -migrate: $(GOOSE) checkdbvars +migrate: checkdbvars $(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" up pg_dump -O -s $(CONNECT_STRING) > db/schema.sql ## Create a new migration file .PHONY: new_migration -new_migration: $(GOOSE) checkmigname +new_migration: checkmigname $(GOOSE) -dir db/migrations create $(NAME) sql ## Check which migrations are applied at the moment .PHONY: migration_status -migration_status: $(GOOSE) checkdbvars +migration_status: checkdbvars $(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" status # Convert timestamped migrations to versioned (to be run in CI); diff --git a/go.mod b/go.mod index a5c0695ce..ddf424493 100644 --- a/go.mod +++ b/go.mod @@ -3,19 +3,45 @@ module github.com/makerdao/vulcanizedb go 1.12 require ( - github.com/dave/jennifer v1.3.0 github.com/ethereum/go-ethereum v1.9.8 + github.com/ClickHouse/clickhouse-go v1.4.3 // indirect + github.com/allegro/bigcache v1.2.1 // indirect + github.com/apilayer/freegeoip v3.5.0+incompatible // indirect + github.com/aristanetworks/goarista v0.0.0-20191106175434-873d404c7f40 // indirect + github.com/btcsuite/btcd v0.20.1-beta // indirect + github.com/cespare/cp v1.1.1 // indirect + github.com/dave/jennifer v1.3.0 + github.com/deckarep/golang-set v1.7.1 // indirect + github.com/denisenkom/go-mssqldb v0.0.0-20200620013148-b91950f658ec // indirect + github.com/docker/docker v1.13.1 // indirect + github.com/edsrzf/mmap-go v1.0.0 // indirect + github.com/elastic/gosigar v0.10.5 // indirect + github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect + github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 // indirect + github.com/gorilla/websocket v1.4.1 // indirect github.com/hashicorp/golang-lru v0.5.4 github.com/hpcloud/tail v1.0.0 - github.com/jmoiron/sqlx v0.0.0-20181024163419-82935fac6c1a + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/influxdata/influxdb v1.7.9 // indirect + github.com/jackpal/go-nat-pmp v1.0.2 // indirect + github.com/jmoiron/sqlx v1.2.0 + github.com/karalabe/usb v0.0.0-20191104083709-911d15fe12a9 // indirect github.com/lib/pq v1.0.0 github.com/mitchellh/go-homedir v1.1.0 github.com/onsi/ginkgo v1.10.1 github.com/onsi/gomega v1.10.0 - github.com/pressly/goose v2.6.0+incompatible + github.com/oschwald/maxminddb-golang v1.5.0 // indirect + github.com/pborman/uuid v1.2.0 // indirect + github.com/pressly/goose v2.7.0-rc5+incompatible + github.com/prometheus/tsdb v0.10.0 // indirect + github.com/rjeczalik/notify v0.9.2 // indirect + github.com/rs/cors v1.7.0 // indirect github.com/sirupsen/logrus v1.2.0 github.com/spf13/cobra v0.0.3 github.com/spf13/viper v1.3.2 + github.com/status-im/keycard-go v0.0.0-20191119114148-6dd40a46baa0 // indirect + github.com/tyler-smith/go-bip39 v1.0.2 // indirect + github.com/ziutek/mymysql v1.5.4 // indirect golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 ) diff --git a/go.sum b/go.sum index 0f756a36b..57e0fbfa7 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,8 @@ github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6L github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/ClickHouse/clickhouse-go v1.4.3 h1:iAFMa2UrQdR5bHJ2/yaSLffZkxpcOYQMCUuKeNXGdqc= +github.com/ClickHouse/clickhouse-go v1.4.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= @@ -47,6 +49,7 @@ github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpi github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= @@ -74,6 +77,7 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= +github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -88,6 +92,8 @@ github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea h1:j4317fAZh7X github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= github.com/deckarep/golang-set v1.7.1 h1:SCQV0S6gTtp6itiFrTqI+pfmJ4LN85S1YzhDf9rTHJQ= github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= +github.com/denisenkom/go-mssqldb v0.0.0-20200620013148-b91950f658ec h1:NfhRXXFDPxcF5Cwo06DzeIaE7uuJtAUhsDwH3LNsjos= +github.com/denisenkom/go-mssqldb v0.0.0-20200620013148-b91950f658ec/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= @@ -134,6 +140,8 @@ github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -194,6 +202,8 @@ github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmoiron/sqlx v0.0.0-20181024163419-82935fac6c1a h1:Jyg5PpIc1nLGrNDM5blVkiSySmRhaD/IiXkvaHzBYnw= github.com/jmoiron/sqlx v0.0.0-20181024163419-82935fac6c1a/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jrick/logrotate v1.0.0 h1:lQ1bL/n9mBNeIXoTUoYRlK4dHuNJVofX9oWqBtPnSzI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -304,6 +314,7 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9 github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= @@ -313,6 +324,10 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pressly/goose v2.6.0+incompatible h1:3f8zIQ8rfgP9tyI0Hmcs2YNAqUCL1c+diLe3iU8Qd/k= github.com/pressly/goose v2.6.0+incompatible/go.mod h1:m+QHWCqxR3k8D9l7qfzuC/djtlfzxr34mozWDYEu1z8= +github.com/pressly/goose v2.7.0-rc4.0.20200813131502-ad369d945ee9+incompatible h1:fdUQTuuYNFD+/OjaK2jKR/gDtE1QkDDMn7+Xbi2f+Kw= +github.com/pressly/goose v2.7.0-rc4.0.20200813131502-ad369d945ee9+incompatible/go.mod h1:m+QHWCqxR3k8D9l7qfzuC/djtlfzxr34mozWDYEu1z8= +github.com/pressly/goose v2.7.0-rc5+incompatible h1:txvo810iG1P/rafOx31LYDlOyikBK8A/8prKP4j066w= +github.com/pressly/goose v2.7.0-rc5+incompatible/go.mod h1:m+QHWCqxR3k8D9l7qfzuC/djtlfzxr34mozWDYEu1z8= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= @@ -403,11 +418,14 @@ github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/xtaci/kcp-go v5.4.5+incompatible/go.mod h1:bN6vIwHQbfHaHtFpEssmWsN45a+AZwO7eyRCmEIbtvE= github.com/xtaci/lossyconn v0.0.0-20190602105132-8df528c0c9ae/go.mod h1:gXtu8J62kEgmN++bm9BVICuT/e8yiLI2KFobd/TRFsE= +github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= +github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5 h1:bselrhR0Or1vomJZC8ZIjWtbDmn9OYFLX5Ik9alpJpE= golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190926114937-fa1a29108794/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= From 307531f09376f93d6aa955d923cdeb3403021ed0 Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Thu, 27 Aug 2020 16:28:23 -0500 Subject: [PATCH 38/66] Make headersync Dockerfile use 2.7.0-rc explicitly --- dockerfiles/header_sync/Dockerfile | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/dockerfiles/header_sync/Dockerfile b/dockerfiles/header_sync/Dockerfile index 2095028b6..77b9b5fe2 100644 --- a/dockerfiles/header_sync/Dockerfile +++ b/dockerfiles/header_sync/Dockerfile @@ -2,18 +2,12 @@ FROM golang:alpine as builder RUN apk --update --no-cache add g++ git linux-headers -# Build migration tool -WORKDIR /go -RUN GO111MODULE=auto go get -u -d github.com/pressly/goose/cmd/goose -WORKDIR /go/src/github.com/pressly/goose/cmd/goose -RUN GO111MODULE=auto go build -a -ldflags '-s' -tags='no_mysql no_sqlite' -o goose - ENV GO111MODULE on WORKDIR /vulcanizedb COPY . . RUN go build - +RUN go get -tags='no_mysql no_sqlite3 no_mssql no_redshift' github.com/pressly/goose/cmd/goose@v2.7.0-rc5 FROM golang:alpine @@ -25,20 +19,18 @@ RUN apk update \ && apk upgrade \ && apk add --no-cache \ ca-certificates \ - bash \ + bash \ && update-ca-certificates 2>/dev/null || true -# setup environment -ENV GO111MODULE on - # Direct logs to stdout for docker log driver RUN ln -sf /dev/stdout /app/vulcanizedb.log # add required files COPY --from=builder /vulcanizedb/vulcanizedb . +COPY --from=builder /vulcanizedb/Makefile . COPY --from=builder /vulcanizedb/dockerfiles/header_sync/startup_script.sh . COPY --from=builder /vulcanizedb/db/migrations/* db/migrations/ -COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose +COPY --from=builder /go/bin/goose goose # needed for waiting until postgres is ready before starting from docker-compose COPY --from=builder /vulcanizedb/dockerfiles/wait-for-it.sh . From af7a8f2c32c581ad28d1b4edabd0fc27579dd260 Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Mon, 24 Aug 2020 16:49:32 -0500 Subject: [PATCH 39/66] Only export the public schema Don't export other schemes which are not managed by this repo. --- Makefile | 4 ++-- db/schema.sql | 21 ++++++++++++++------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index f5c5adbbc..c2af0c674 100644 --- a/Makefile +++ b/Makefile @@ -99,7 +99,7 @@ checkmigname: .PHONY: rollback rollback: checkdbvars $(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" down - pg_dump -O -s $(CONNECT_STRING) > db/schema.sql + pg_dump -n 'public' -O -s $(CONNECT_STRING) > db/schema.sql ## Rollbackt to a select migration (id/timestamp) @@ -111,7 +111,7 @@ rollback_to: checkmigration checkdbvars .PHONY: migrate migrate: checkdbvars $(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" up - pg_dump -O -s $(CONNECT_STRING) > db/schema.sql + pg_dump -n 'public' -O -s $(CONNECT_STRING) > db/schema.sql ## Create a new migration file .PHONY: new_migration diff --git a/db/schema.sql b/db/schema.sql index e4654a8a0..4096d5e43 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -16,6 +16,20 @@ SET xmloption = content; SET client_min_messages = warning; SET row_security = off; +-- +-- Name: public; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA public; + + +-- +-- Name: SCHEMA public; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON SCHEMA public IS 'standard public schema'; + + -- -- Name: diff_status; Type: TYPE; Schema: public; Owner: - -- @@ -53,18 +67,15 @@ BEGIN IF last_storage_value = create_back_filled_diff.storage_value THEN RETURN; END IF; - IF last_storage_value is null and create_back_filled_diff.storage_value = empty_storage_value THEN RETURN; END IF; - INSERT INTO public.storage_diff (block_height, block_hash, address, storage_key, storage_value, eth_node_id, from_backfill) VALUES (create_back_filled_diff.block_height, create_back_filled_diff.block_hash, create_back_filled_diff.address, create_back_filled_diff.storage_key, create_back_filled_diff.storage_value, create_back_filled_diff.eth_node_id, true) ON CONFLICT DO NOTHING; - RETURN; END $$; @@ -106,20 +117,16 @@ BEGIN IF matching_header_id != 0 THEN RETURN matching_header_id; END IF; - IF nonmatching_header_id != 0 AND block_number <= max_block_number - 15 THEN RETURN nonmatching_header_id; END IF; - IF nonmatching_header_id != 0 AND block_number > max_block_number - 15 THEN DELETE FROM public.headers WHERE id = nonmatching_header_id; END IF; - INSERT INTO public.headers (hash, block_number, raw, block_timestamp, eth_node_id) VALUES (get_or_create_header.hash, get_or_create_header.block_number, get_or_create_header.raw, get_or_create_header.block_timestamp, get_or_create_header.eth_node_id) RETURNING id INTO inserted_header_id; - RETURN inserted_header_id; END $$; From 4a652bafad8dedb7ff3dfedb53a440c0aecc9de1 Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Thu, 27 Aug 2020 16:28:52 -0500 Subject: [PATCH 40/66] Add helpful make tasks around Docker - Add one to run headersync - Add one to build any docker image in the dockerfiles directory. - Remove Rinkeby, which doesn't appear to exist or work. - A local copy of header_sync:latest is the default (built with the dockerbuild task) but it's handy to be able to also specify a name. --- Makefile | 47 +++++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index c2af0c674..4eb9a1598 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,6 @@ $(BIN)/gometalinter.v2: go get -u gopkg.in/alecthomas/gometalinter.v2 $(METALINT) --install - .PHONY: installtools installtools: | $(LINT) $(GINKGO) echo "Installing tools" @@ -135,25 +134,29 @@ import: test -n "$(NAME)" # $$NAME psql $(NAME) < db/schema.sql - # Docker actions -## Rinkeby docker environment -RINKEBY_COMPOSE_FILE=dockerfiles/rinkeby/docker-compose.yml - -.PHONY: rinkeby_env_up -rinkeby_env_up: - docker-compose -f $(RINKEBY_COMPOSE_FILE) up -d geth - docker-compose -f $(RINKEBY_COMPOSE_FILE) up --build migrations - docker-compose -f $(RINKEBY_COMPOSE_FILE) up -d --build vulcanizedb - -.PHONY: rinkeby_env_deploy -rinkeby_env_deploy: - docker-compose -f $(RINKEBY_COMPOSE_FILE) up -d --build vulcanizedb - -.PHONY: dev_env_migrate -rinkeby_env_migrate: - docker-compose -f $(RINKEBY_COMPOSE_FILE) up --build migrations - -.PHONY: rinkeby_env_down -rinkeby_env_down: - docker-compose -f $(RINKEBY_COMPOSE_FILE) down +# Build any docker image in dockerfiles +.PHONY: dockerbuild +dockerbuild: + test -n "$(IMAGE)" # $$IMAGE + docker build -t $(IMAGE) -f dockerfiles/$(IMAGE)/Dockerfile . + +.PHONY: header_sync +header_sync: STARTING_BLOCK_NUMBER ?= 10000000 +header_sync: HOST ?= host.docker.internal +header_sync: DATABASE_PASSWORD ?= postgres +header_sync: IMAGE_WITH_TAG ?= header_sync:latest +header_sync: + test -n "$(NAME)" # $$(NAME) - Database Name + test -n "$(CLIENT_IPCPATH)" # $$(CLIENT_IPCPATH) + docker run \ + -it \ + -p "5432:5432" \ + -e "STARTING_BLOCK_NUMBER=$(STARTING_BLOCK_NUMBER)" \ + -e "DATABASE_NAME=$(NAME)" \ + -e "DATABASE_HOSTNAME=$(HOST)" \ + -e "DATABASE_PORT=$(PORT)" \ + -e "DATABASE_USER=$(USER)" \ + -e "DATABASE_PASSWORD=$(DATABASE_PASSWORD)" \ + -e "CLIENT_IPCPATH=$(CLIENT_IPCPATH)" \ + $(IMAGE_WITH_TAG) From 26a04e3e5abb0a81dead235ef1a1aeb2c1a5e889 Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Tue, 25 Aug 2020 14:23:35 -0500 Subject: [PATCH 41/66] Get Compose to work with multiple schemas Add a schema to the plugin config. It's required for composing plugins, since plugins should have their own schema. Update compose doc to include the need for a schema Give default schema as 'public' Run the public schema migrations before the plugins --- cmd/compose.go | 1 + cmd/root.go | 1 + .../composeAndExecuteAccountTransformer.toml | 1 + pkg/config/plugin.go | 1 + pkg/plugin/manager/manager.go | 53 +++++++++++++++++++ 5 files changed, 57 insertions(+) diff --git a/cmd/compose.go b/cmd/compose.go index 0a3daf483..4a29104e4 100644 --- a/cmd/compose.go +++ b/cmd/compose.go @@ -41,6 +41,7 @@ var composeCmd = &cobra.Command{ [exporter] home = "github.com/makerdao/vulcanizedb" name = "exampleTransformerExporter" + schema = "" save = false transformerNames = [ "transformer1", diff --git a/cmd/root.go b/cmd/root.go index 123a7b2c5..2f5892aa4 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -220,6 +220,7 @@ func prepConfig() error { genConfig = config.Plugin{ Transformers: transformers, FilePath: "$GOPATH/src/github.com/makerdao/vulcanizedb/plugins", + Schema: viper.GetString("exporter.schema"), FileName: viper.GetString("exporter.name"), Save: viper.GetBool("exporter.save"), Home: viper.GetString("exporter.home"), diff --git a/environments/composeAndExecuteAccountTransformer.toml b/environments/composeAndExecuteAccountTransformer.toml index 5b51262a7..9bb644a8c 100644 --- a/environments/composeAndExecuteAccountTransformer.toml +++ b/environments/composeAndExecuteAccountTransformer.toml @@ -8,6 +8,7 @@ [exporter] home = "github.com/makerdao/vulcanizedb" + schema = "public" name = "accountTransformerExporter" save = false transformerNames = [ diff --git a/pkg/config/plugin.go b/pkg/config/plugin.go index 25f0d6014..fe9423803 100644 --- a/pkg/config/plugin.go +++ b/pkg/config/plugin.go @@ -31,6 +31,7 @@ type Plugin struct { FileName string Save bool Home string + Schema string } type Transformer struct { diff --git a/pkg/plugin/manager/manager.go b/pkg/plugin/manager/manager.go index 6e8b7f148..5b6ebf976 100644 --- a/pkg/plugin/manager/manager.go +++ b/pkg/plugin/manager/manager.go @@ -67,6 +67,10 @@ func (m *manager) setDB() error { } func (m *manager) RunMigrations() error { + if len(m.GenConfig.Schema) <= 0 { + return fmt.Errorf("config is missing a schema, required for plugin migrations") + } + // Get paths to db migrations from the plugin config paths, err := m.GenConfig.GetMigrationsPaths() if err != nil { @@ -75,12 +79,20 @@ func (m *manager) RunMigrations() error { if len(paths) < 1 { return nil } + // Init directory for temporary copies of migrations err = m.setupMigrationEnv() if err != nil { return err } defer m.cleanUp() + + // First run the public schema migrations + err = m.runPublicMigrations() + if err != nil { + return err + } + // Creates copies of migrations for all the plugin's transformers in a tmp dir err = m.createMigrationCopies(paths) if err != nil { @@ -90,6 +102,38 @@ func (m *manager) RunMigrations() error { return nil } +func (m *manager) runPublicMigrations() error { + // Setup DB if not set + if m.db == nil { + setErr := m.setDB() + if setErr != nil { + return errors.New(fmt.Sprintf("could not open db: %s", setErr.Error())) + } + } + + goose.SetTableName("public.goose_db_version") + + // Fix the migrations + path, err := helpers.CleanPath(filepath.Join("$GOPATH/src", m.GenConfig.Home, "db", "migrations")) + + if err != nil { + return err + } + + fixErr := goose.Fix(path) + if fixErr != nil { + return errors.New(fmt.Sprintf("version fixing for plugin migrations at %s failed: %s", path, fixErr.Error())) + } + + // Run the copied migrations with goose + upErr := goose.Up(m.db, path) + if upErr != nil { + return errors.New(fmt.Sprintf("db migrations for plugin transformers at %s failed: %s", path, upErr.Error())) + } + return nil + +} + // Setup a temporary directory to hold transformer db migrations func (m *manager) setupMigrationEnv() error { var err error @@ -149,11 +193,20 @@ func (m *manager) fixAndRun(path string) error { return errors.New(fmt.Sprintf("could not open db: %s", setErr.Error())) } } + + _, execErr := m.db.Exec(fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS %s", m.GenConfig.Schema)) + if execErr != nil { + return execErr + } + + goose.SetTableName(fmt.Sprintf("%s.goose_db_version", m.GenConfig.Schema)) + // Fix the migrations fixErr := goose.Fix(m.tmpMigDir) if fixErr != nil { return errors.New(fmt.Sprintf("version fixing for plugin migrations at %s failed: %s", path, fixErr.Error())) } + // Run the copied migrations with goose upErr := goose.Up(m.db, m.tmpMigDir) if upErr != nil { From ce96350b5ff5216831076c084326c78fee866a33 Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Tue, 1 Sep 2020 19:17:44 -0500 Subject: [PATCH 42/66] Allow injecting the MigrationManager for testing --- pkg/plugin/generator.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/plugin/generator.go b/pkg/plugin/generator.go index 842fd37e2..695c6e140 100644 --- a/pkg/plugin/generator.go +++ b/pkg/plugin/generator.go @@ -18,6 +18,7 @@ package plugin import ( "errors" + "github.com/makerdao/vulcanizedb/pkg/config" "github.com/makerdao/vulcanizedb/pkg/plugin/builder" "github.com/makerdao/vulcanizedb/pkg/plugin/manager" @@ -27,6 +28,7 @@ import ( // Generator is the top-level interface for creating transformer plugins type Generator interface { GenerateExporterPlugin() error + SetMigrationManager(manager.MigrationManager) } type generator struct { @@ -66,3 +68,8 @@ func (g *generator) GenerateExporterPlugin() error { // Perform db migrations for the transformers return g.MigrationManager.RunMigrations() } + +// Sets the Migration manager - typically used for testing +func (g *generator) SetMigrationManager(mgr manager.MigrationManager) { + g.MigrationManager = mgr +} From fd1f9b6ce0a755eff042cc7c51a5099d6f52a5d6 Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Fri, 4 Sep 2020 13:48:35 -0500 Subject: [PATCH 43/66] Cleanup a bunch of the error handling It's not all, but name error variables and annotate the errors --- pkg/plugin/manager/manager.go | 55 +++++++++++++++++------------------ 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/pkg/plugin/manager/manager.go b/pkg/plugin/manager/manager.go index 5b6ebf976..34ae6296f 100644 --- a/pkg/plugin/manager/manager.go +++ b/pkg/plugin/manager/manager.go @@ -60,7 +60,7 @@ func (m *manager) setDB() error { } dbConnector, err := pq.NewConnector(pgStr) if err != nil { - return errors.New(fmt.Sprintf("can't connect to db: %s", err.Error())) + return fmt.Errorf("can't connect to db: %w", err) } m.db = sql.OpenDB(dbConnector) return nil @@ -68,35 +68,35 @@ func (m *manager) setDB() error { func (m *manager) RunMigrations() error { if len(m.GenConfig.Schema) <= 0 { - return fmt.Errorf("config is missing a schema, required for plugin migrations") + return errors.New("config is missing a schema, required for plugin migrations") } // Get paths to db migrations from the plugin config paths, err := m.GenConfig.GetMigrationsPaths() if err != nil { - return err + return fmt.Errorf("could not get migration paths %w", err) } if len(paths) < 1 { return nil } // Init directory for temporary copies of migrations - err = m.setupMigrationEnv() - if err != nil { - return err + setupErr := m.setupMigrationEnv() + if setupErr != nil { + return fmt.Errorf("could not setup migration env %w", setupErr) } defer m.cleanUp() // First run the public schema migrations - err = m.runPublicMigrations() - if err != nil { - return err + migrationErr := m.runPublicMigrations() + if migrationErr != nil { + return fmt.Errorf("could not run public migrations %w", migrationErr) } // Creates copies of migrations for all the plugin's transformers in a tmp dir - err = m.createMigrationCopies(paths) - if err != nil { - return err + schemaMigrationErr := m.createMigrationCopies(paths) + if schemaMigrationErr != nil { + return fmt.Errorf("could not run schema migrations %w", schemaMigrationErr) } return nil @@ -107,28 +107,27 @@ func (m *manager) runPublicMigrations() error { if m.db == nil { setErr := m.setDB() if setErr != nil { - return errors.New(fmt.Sprintf("could not open db: %s", setErr.Error())) + return fmt.Errorf("could not open db: %w", setErr) } } goose.SetTableName("public.goose_db_version") - // Fix the migrations - path, err := helpers.CleanPath(filepath.Join("$GOPATH/src", m.GenConfig.Home, "db", "migrations")) + path, pathErr := helpers.CleanPath(filepath.Join("$GOPATH/src", m.GenConfig.Home, "db", "migrations")) - if err != nil { - return err + if pathErr != nil { + return fmt.Errorf("could not construct filepath %w", pathErr) } fixErr := goose.Fix(path) if fixErr != nil { - return errors.New(fmt.Sprintf("version fixing for plugin migrations at %s failed: %s", path, fixErr.Error())) + return fmt.Errorf("version fixing for plugin migrations at %s failed: %w", path, fixErr) } // Run the copied migrations with goose upErr := goose.Up(m.db, path) if upErr != nil { - return errors.New(fmt.Sprintf("db migrations for plugin transformers at %s failed: %s", path, upErr.Error())) + return fmt.Errorf("db migrations for plugin transformers at %s failed: %w", path, upErr) } return nil @@ -139,17 +138,17 @@ func (m *manager) setupMigrationEnv() error { var err error m.tmpMigDir, err = helpers.CleanPath(filepath.Join("$GOPATH/src", m.GenConfig.Home, ".plugin_migrations")) if err != nil { - return err + return fmt.Errorf("unable to construct clean path %w", err) } removeErr := os.RemoveAll(m.tmpMigDir) if removeErr != nil { - removeErrString := "unable to remove file found at %s where tmp directory needs to be written: %s" - return errors.New(fmt.Sprintf(removeErrString, m.tmpMigDir, removeErr.Error())) + removeErrString := "unable to remove file found at %s where tmp directory needs to be written: %w" + return fmt.Errorf(removeErrString, m.tmpMigDir, removeErr) } mkdirErr := os.Mkdir(m.tmpMigDir, os.ModePerm) if mkdirErr != nil { - mkdirErrString := "unable to create temporary migration directory %s: %s" - return errors.New(fmt.Sprintf(mkdirErrString, m.tmpMigDir, mkdirErr.Error())) + mkdirErrString := "unable to create temporary migration directory %s: %w" + return fmt.Errorf(mkdirErrString, m.tmpMigDir, mkdirErr) } return nil @@ -190,13 +189,13 @@ func (m *manager) fixAndRun(path string) error { if m.db == nil { setErr := m.setDB() if setErr != nil { - return errors.New(fmt.Sprintf("could not open db: %s", setErr.Error())) + return fmt.Errorf("could not open db: %w", setErr) } } _, execErr := m.db.Exec(fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS %s", m.GenConfig.Schema)) if execErr != nil { - return execErr + return fmt.Errorf("could not create schema %s, %w", m.GenConfig.Schema, execErr) } goose.SetTableName(fmt.Sprintf("%s.goose_db_version", m.GenConfig.Schema)) @@ -204,13 +203,13 @@ func (m *manager) fixAndRun(path string) error { // Fix the migrations fixErr := goose.Fix(m.tmpMigDir) if fixErr != nil { - return errors.New(fmt.Sprintf("version fixing for plugin migrations at %s failed: %s", path, fixErr.Error())) + return fmt.Errorf("version fixing for plugin migrations at %s failed: %w", path, fixErr) } // Run the copied migrations with goose upErr := goose.Up(m.db, m.tmpMigDir) if upErr != nil { - return errors.New(fmt.Sprintf("db migrations for plugin transformers at %s failed: %s", path, upErr.Error())) + return fmt.Errorf("db migrations for plugin transformers at %s failed: %w", path, upErr) } return nil } From 36e5ab8a2b241facc2196c40b6a68da8b953e044 Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Fri, 4 Sep 2020 10:53:55 -0500 Subject: [PATCH 44/66] Run go mod tidy This cleaned up all the nonsense that got in here by mistake. --- go.mod | 27 +------------ go.sum | 118 ++++----------------------------------------------------- 2 files changed, 9 insertions(+), 136 deletions(-) diff --git a/go.mod b/go.mod index ddf424493..4ef6d90c7 100644 --- a/go.mod +++ b/go.mod @@ -3,45 +3,20 @@ module github.com/makerdao/vulcanizedb go 1.12 require ( - github.com/ethereum/go-ethereum v1.9.8 github.com/ClickHouse/clickhouse-go v1.4.3 // indirect - github.com/allegro/bigcache v1.2.1 // indirect - github.com/apilayer/freegeoip v3.5.0+incompatible // indirect - github.com/aristanetworks/goarista v0.0.0-20191106175434-873d404c7f40 // indirect - github.com/btcsuite/btcd v0.20.1-beta // indirect - github.com/cespare/cp v1.1.1 // indirect github.com/dave/jennifer v1.3.0 - github.com/deckarep/golang-set v1.7.1 // indirect - github.com/denisenkom/go-mssqldb v0.0.0-20200620013148-b91950f658ec // indirect - github.com/docker/docker v1.13.1 // indirect - github.com/edsrzf/mmap-go v1.0.0 // indirect - github.com/elastic/gosigar v0.10.5 // indirect - github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect - github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 // indirect - github.com/gorilla/websocket v1.4.1 // indirect + github.com/ethereum/go-ethereum v1.9.8 github.com/hashicorp/golang-lru v0.5.4 github.com/hpcloud/tail v1.0.0 - github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/influxdata/influxdb v1.7.9 // indirect - github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jmoiron/sqlx v1.2.0 - github.com/karalabe/usb v0.0.0-20191104083709-911d15fe12a9 // indirect github.com/lib/pq v1.0.0 github.com/mitchellh/go-homedir v1.1.0 github.com/onsi/ginkgo v1.10.1 github.com/onsi/gomega v1.10.0 - github.com/oschwald/maxminddb-golang v1.5.0 // indirect - github.com/pborman/uuid v1.2.0 // indirect github.com/pressly/goose v2.7.0-rc5+incompatible - github.com/prometheus/tsdb v0.10.0 // indirect - github.com/rjeczalik/notify v0.9.2 // indirect - github.com/rs/cors v1.7.0 // indirect github.com/sirupsen/logrus v1.2.0 github.com/spf13/cobra v0.0.3 github.com/spf13/viper v1.3.2 - github.com/status-im/keycard-go v0.0.0-20191119114148-6dd40a46baa0 // indirect - github.com/tyler-smith/go-bip39 v1.0.2 // indirect - github.com/ziutek/mymysql v1.5.4 // indirect golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 ) diff --git a/go.sum b/go.sum index 57e0fbfa7..f0f1fccdd 100644 --- a/go.sum +++ b/go.sum @@ -18,12 +18,10 @@ github.com/ClickHouse/clickhouse-go v1.4.3 h1:iAFMa2UrQdR5bHJ2/yaSLffZkxpcOYQMCU github.com/ClickHouse/clickhouse-go v1.4.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/Shopify/sarama v1.23.1/go.mod h1:XLH1GYJnLVE0XCr6KdJGVJRTwY30moWNJ4sERjXX6fs= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/VictoriaMetrics/fastcache v1.5.3 h1:2odJnXLbFZcoV9KYtQ+7TH1UOq3dn3AssMgieaezkR4= -github.com/VictoriaMetrics/fastcache v1.5.3/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v49daAVERr0H3CuscE= github.com/VictoriaMetrics/fastcache v1.5.7 h1:4y6y0G8PRzszQUYIQHHssv/jgPHAb5qQuuDNdCbyAgw= github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8= github.com/aead/siphash v1.0.1 h1:FwHfE/T45KPKYuuSAKyyvE+oPWcaQ+CUmFW0bPlM+kg= @@ -33,14 +31,10 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/allegro/bigcache v1.2.1 h1:hg1sY1raCwic3Vnsvje6TT7/pnZba83LeFck5NrFKSc= github.com/allegro/bigcache v1.2.1/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= -github.com/apilayer/freegeoip v3.5.0+incompatible h1:z1u2gv0/rsSi/HqMDB436AiUROXXim7st5DOg4Ikl4A= -github.com/apilayer/freegeoip v3.5.0+incompatible/go.mod h1:CUfFqErhFhXneJendyQ/rRcuA8kH8JxHvYnbOozmlCU= github.com/aristanetworks/fsnotify v1.4.2/go.mod h1:D/rtu7LpjYM8tRJphJ0hUBYpjai8SfX+aSNsWDTq/Ks= github.com/aristanetworks/glog v0.0.0-20180419172825-c15b03b3054f/go.mod h1:KASm+qXFKs/xjSoWn30NrWBBvdTTQq+UjkhjEJHfSFA= github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847 h1:rtI0fD4oG/8eVokGVPYJEW1F88p1ZNgXiEIs9thEE4A= github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= -github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015 h1:7ABPr1+uJdqESAdlVevnc/2FJGiC/K3uMg1JiELeF+0= -github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= github.com/aristanetworks/goarista v0.0.0-20191106175434-873d404c7f40 h1:ZdRuixFqR3mfx4FHzclG3COrRgWrYq0VhNgIoYoObcM= github.com/aristanetworks/goarista v0.0.0-20191106175434-873d404c7f40/go.mod h1:Z4RTxGAuYhPzcq8+EdRM+R8M48Ssle2TsWtwRKa+vns= github.com/aristanetworks/splunk-hec-go v0.3.3/go.mod h1:1VHO9r17b0K7WmOlLb9nTk/2YanvOEnLMUgsFrxBROc= @@ -72,7 +66,6 @@ github.com/cespare/cp v1.1.1 h1:nCb6ZLdB7NRaqsm91JtQTAme2SKJzXVsdPIPkyJr1MU= github.com/cespare/cp v1.1.1/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18/go.mod h1:HD5P3vAIAh+Y2GAxg0PrPN1P8WkepXGpjbUPDHJqqKM= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -92,15 +85,10 @@ github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea h1:j4317fAZh7X github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= github.com/deckarep/golang-set v1.7.1 h1:SCQV0S6gTtp6itiFrTqI+pfmJ4LN85S1YzhDf9rTHJQ= github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= -github.com/denisenkom/go-mssqldb v0.0.0-20200620013148-b91950f658ec h1:NfhRXXFDPxcF5Cwo06DzeIaE7uuJtAUhsDwH3LNsjos= -github.com/denisenkom/go-mssqldb v0.0.0-20200620013148-b91950f658ec/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo= -github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/dop251/goja v0.0.0-20200106141417-aaec0e7bde29/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= github.com/dop251/goja v0.0.0-20200219165308-d1232e640a87/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= @@ -109,11 +97,6 @@ github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c h1:JHHhtb9XWJrGNMcr github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/elastic/gosigar v0.8.1-0.20180330100440-37f05ff46ffa/go.mod h1:cdorVVzy1fhmEqmtgqkoE3bYtCfSCkVyjTyCIo22xvs= -github.com/elastic/gosigar v0.10.4 h1:6jfw75dsoflhBMRdO6QPzQUgLqUYTsQQQRkkcsHsuPo= -github.com/elastic/gosigar v0.10.4/go.mod h1:cdorVVzy1fhmEqmtgqkoE3bYtCfSCkVyjTyCIo22xvs= -github.com/elastic/gosigar v0.10.5 h1:GzPQ+78RaAb4J63unidA/JavQRKrB6s8IOzN6Ib59jo= -github.com/elastic/gosigar v0.10.5/go.mod h1:cdorVVzy1fhmEqmtgqkoE3bYtCfSCkVyjTyCIo22xvs= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc h1:jtW8jbpkO4YirRSyepBOH8E+2HEw6/hKkBvFPwhUN8c= github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= @@ -130,18 +113,16 @@ github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0 h1:8HUsc87TaSWLKwrnumgC8/YconD2fJQsRJAsWaPg2ic= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= -github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= -github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -151,7 +132,6 @@ github.com/golang/protobuf v1.3.2-0.20190517061210-b285ee9cfc6c h1:zqAKixg3cTcIa github.com/golang/protobuf v1.3.2-0.20190517061210-b285ee9cfc6c/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -165,33 +145,21 @@ github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989 h1:giknQ4mEuDF github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/graph-gophers/graphql-go v0.0.0-20191024035216-0a9cfbec35a1 h1:jV0CRazQbnsAGKT1z8BjMvouE2pypynEjx/o7eHbkFM= -github.com/graph-gophers/graphql-go v0.0.0-20191024035216-0a9cfbec35a1/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/golang-lru v0.0.0-20160813221303-0a025b7e63ad/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk= -github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/howeyc/fsnotify v0.9.0 h1:0gtV5JmOKH4A8SsFxG2BczSeXWWPvcMT0euZt5gDAxY= -github.com/howeyc/fsnotify v0.9.0/go.mod h1:41HzSPxBGeFRQKEEwgh49TRw/nKBsYZ2cF1OzPjSJsA= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huin/goupnp v0.0.0-20161224104101-679507af18f3/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag= github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= -github.com/influxdata/influxdb v1.7.9 h1:uSeBTNO4rBkbp1Be5FKRsAmglM9nlx25TzVQRQt1An4= -github.com/influxdata/influxdb v1.7.9/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= github.com/influxdata/influxdb1-client v0.0.0-20190809212627-fc22c7df067e/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= -github.com/jackpal/go-nat-pmp v1.0.1 h1:i0LektDkO1QlrTm/cSuP+PyBCDnYvjPLGl4LdWEMiaA= -github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= @@ -200,8 +168,6 @@ github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0 github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89 h1:12K8AlpT0/6QUXSfV0yi4Q0jkbq8NDtIKFtF61AoqV0= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmoiron/sqlx v0.0.0-20181024163419-82935fac6c1a h1:Jyg5PpIc1nLGrNDM5blVkiSySmRhaD/IiXkvaHzBYnw= -github.com/jmoiron/sqlx v0.0.0-20181024163419-82935fac6c1a/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jrick/logrotate v1.0.0 h1:lQ1bL/n9mBNeIXoTUoYRlK4dHuNJVofX9oWqBtPnSzI= @@ -211,8 +177,6 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/karalabe/usb v0.0.0-20190819132248-550797b1cad8 h1:VhnqxaTIudc9IWKx8uXRLnpdSb9noCEj+vHacjmhp68= -github.com/karalabe/usb v0.0.0-20190819132248-550797b1cad8/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356 h1:I/yrLt2WilKxlQKCM52clh5rGzTKpVctGT1lH4Dc8Jw= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= github.com/karalabe/usb v0.0.0-20191104083709-911d15fe12a9 h1:ZHuwnjpP8LsVsUYqTqeVAI+GfDfJ6UNPrExZF+vX/DQ= @@ -225,45 +189,27 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGi github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/makerdao/go-ethereum v1.5.10-0.20200508151415-a332a1649dff h1:oH6AkDyruMAXlE1WELwaAc5iEnRBsQfQALS+gGzDHIM= -github.com/makerdao/go-ethereum v1.5.10-0.20200508151415-a332a1649dff/go.mod h1:qeDFnA9rW6VM6KC91PIlcpXMr8V+KY4udYi/RSZtPT4= -github.com/makerdao/go-ethereum v1.5.10-0.20200521202712-d5657e3ff589 h1:a6bFsUnwxMBq7MLW4vZDBj1DnEpWyFQRMGp71t1aJj4= -github.com/makerdao/go-ethereum v1.5.10-0.20200521202712-d5657e3ff589/go.mod h1:oP8FC5+TbICUyftkTWs+8JryntjIJLJvWvApK3z2AYw= -github.com/makerdao/go-ethereum v1.9.11-rc2 h1:ZjxmFdkar8P43EkK17ni8y3vXxHHFZjOd12HqNalf8M= -github.com/makerdao/go-ethereum v1.9.11-rc2/go.mod h1:7oC0Ni6dosMv5pxMigm6s0hN8g4haJMBnqmmo0D9YfQ= github.com/makerdao/go-ethereum v1.9.15-statechange-filter h1:jLX9CleT+i/gUn5LLTUoor1NXYLHSTnZM1pOFKrg2IA= github.com/makerdao/go-ethereum v1.9.15-statechange-filter/go.mod h1:slT8bPPRhXsyNTwHQxrOnjuTZ1sDXRajW11EkJ84QJ0= -github.com/makerdao/vdb-mcd-transformers v0.2.17/go.mod h1:lon896AMm/ZzNGu4n1Cfa1XW2V3C1jtHs13VRvTHKrg= -github.com/makerdao/vulcanizedb v0.0.11-rc.1/go.mod h1:FZZ/SKy+7Ejdc+4Wa5oTGrr2cFuaCgGBxc2xoifaEoY= github.com/mattn/go-colorable v0.1.0 h1:v2XXALHHh6zHfYTJ+cSkwtyffnaOyR1MXaA91mTrb8o= github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035 h1:USWjF42jDCSEeikX/G1g40ZWnsPXN5WkZ4jMHZWyBK4= github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= -github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= -github.com/mattn/go-isatty v0.0.10 h1:qxFzApOv4WsAL965uUPIsXzAKCZxN2p9UqdhFS4ZW10= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.6 h1:V2iyH+aX9C5fsYCpK60U8BYIvmhqxuOL3JZcqc1NB7k= -github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -275,8 +221,6 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= -github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= @@ -285,8 +229,6 @@ github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8u github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c h1:1RHs3tNxjXGHeul8z2t6H2N2TlAqpKe5yryJztRx4Jk= github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/olekukonko/tablewriter v0.0.3 h1:i0LBnzgiChAWHJYTQAZJDOgf8MNxAVYZJ2m63SIDimI= -github.com/olekukonko/tablewriter v0.0.3/go.mod h1:YZeBtGzYYEsCHp2LST/u/0NDwGkRoBtmn1cIWCJiS6M= github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= @@ -303,8 +245,6 @@ github.com/openconfig/gnmi v0.0.0-20190823184014-89b2bf29312c/go.mod h1:t+O9It+L github.com/openconfig/reference v0.0.0-20190727015836-8dfd928c9696/go.mod h1:ym2A+zigScwkSEb/cVQB0/ZMpU3rqiH6X7WRRsxgOGw= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/oschwald/maxminddb-golang v1.5.0 h1:rmyoIV6z2/s9TCJedUuDiKht2RN12LWJ1L7iRGtWY64= -github.com/oschwald/maxminddb-golang v1.5.0/go.mod h1:3jhIUymTJ5VREKyIhWm66LJiQt04F0UCDdodShpjWsY= github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222 h1:goeTyGkArOZIVOMA0dQbyuPWGNQJZGPwPu/QS9GlpnA= github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= @@ -322,10 +262,6 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pressly/goose v2.6.0+incompatible h1:3f8zIQ8rfgP9tyI0Hmcs2YNAqUCL1c+diLe3iU8Qd/k= -github.com/pressly/goose v2.6.0+incompatible/go.mod h1:m+QHWCqxR3k8D9l7qfzuC/djtlfzxr34mozWDYEu1z8= -github.com/pressly/goose v2.7.0-rc4.0.20200813131502-ad369d945ee9+incompatible h1:fdUQTuuYNFD+/OjaK2jKR/gDtE1QkDDMn7+Xbi2f+Kw= -github.com/pressly/goose v2.7.0-rc4.0.20200813131502-ad369d945ee9+incompatible/go.mod h1:m+QHWCqxR3k8D9l7qfzuC/djtlfzxr34mozWDYEu1z8= github.com/pressly/goose v2.7.0-rc5+incompatible h1:txvo810iG1P/rafOx31LYDlOyikBK8A/8prKP4j066w= github.com/pressly/goose v2.7.0-rc5+incompatible/go.mod h1:m+QHWCqxR3k8D9l7qfzuC/djtlfzxr34mozWDYEu1z8= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -362,7 +298,6 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spaolacci/murmur3 v1.0.1-0.20190317074736-539464a789e9/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= @@ -377,8 +312,6 @@ github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= -github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48 h1:ju5UTwk5Odtm4trrY+4Ca4RMj5OyXbmVeDAVad2T0Jw= -github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/status-im/keycard-go v0.0.0-20191119114148-6dd40a46baa0 h1:5UdlDkkBoPrJfh7zkfoR3X5utJhNs/MCQysK3x0ycgg= github.com/status-im/keycard-go v0.0.0-20191119114148-6dd40a46baa0/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 h1:gIlAHnH1vJb5vwEjIp5kBj/eu99p/bl0Ay2goiPe5xE= @@ -392,8 +325,6 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= -github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d h1:gZZadD8H+fF+n9CmNhYL1Y0dJB+kLOmKd7FbPJLeGHs= github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= github.com/templexxx/cpufeat v0.0.0-20180724012125-cef66df7f161/go.mod h1:wM7WEvslTq+iOEAMDLSzhVuOt5BRZ05WirO+b09GHQU= @@ -405,12 +336,6 @@ github.com/tyler-smith/go-bip39 v1.0.2 h1:+t3w+KwLXO6154GNJY+qUtIxLTmFjfUmpguQT1 github.com/tyler-smith/go-bip39 v1.0.2/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/vulcanize/go-ethereum v0.0.0-20190731183759-8e20673bd101 h1:fsHhBzscAwi4u7/F033SFJwTIz+46D8uDWMu2/ZdvzA= -github.com/vulcanize/go-ethereum v0.0.0-20190731183759-8e20673bd101 h1:fsHhBzscAwi4u7/F033SFJwTIz+46D8uDWMu2/ZdvzA= -github.com/vulcanize/go-ethereum v0.0.0-20190731183759-8e20673bd101/go.mod h1:9i0pGnKDUFFr8yC/n8xyrNBVfhYlpwE8J3Ge6ThKvug= -github.com/vulcanize/go-ethereum v0.0.0-20190731183759-8e20673bd101/go.mod h1:9i0pGnKDUFFr8yC/n8xyrNBVfhYlpwE8J3Ge6ThKvug= -github.com/vulcanize/vulcanizedb v0.0.9 h1:ozV2t/MG4/WLgP89blPOw4r1KwM5ji9+DW9CXPGFX08= -github.com/vulcanize/vulcanizedb v0.0.9/go.mod h1:9zfi6VIiCrDVHXe9Z0xmW4CDmNPFwdLuHLcRjjSrdLQ= github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 h1:1cngl9mPEoITZG8s8cVcUy5CeIBYhEESkOB7m6Gmkrk= github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= @@ -418,19 +343,13 @@ github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/xtaci/kcp-go v5.4.5+incompatible/go.mod h1:bN6vIwHQbfHaHtFpEssmWsN45a+AZwO7eyRCmEIbtvE= github.com/xtaci/lossyconn v0.0.0-20190602105132-8df528c0c9ae/go.mod h1:gXtu8J62kEgmN++bm9BVICuT/e8yiLI2KFobd/TRFsE= -github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= -github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5 h1:bselrhR0Or1vomJZC8ZIjWtbDmn9OYFLX5Ik9alpJpE= golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= -golang.org/x/crypto v0.0.0-20190926114937-fa1a29108794/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= -golang.org/x/crypto v0.0.0-20191119213627-4f8c1d86b1ba h1:9bFeDpN3gTqNanMVqNcoR/pJQuP5uroC3t1D7eXozTE= -golang.org/x/crypto v0.0.0-20191119213627-4f8c1d86b1ba/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4 h1:QmwruyY+bKbDDL0BaglrbZABEali68eoMFhTZpCjYVA= golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -440,15 +359,10 @@ golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4r golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65 h1:+rhAzEzT3f4JtomfC371qB+0Ola2caSKcY69NUBZrRQ= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190912160710-24e19bdeb0f2 h1:4dVFTC832rPn4pomLSz1vA+are2+dU19w1H8OngV7nc= golang.org/x/net v0.0.0-20190912160710-24e19bdeb0f2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0 h1:Jcxah/M+oLZ/R4/z5RzfPzGbPXnVDPkEDtf2JnuxN+U= golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -466,22 +380,11 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5 h1:mzjBh+S5frKOsOBobWIMAbXav golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190912141932-bc967efca4b8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 h1:rOhMmluY6kLMhdnrivzec6lLgaVbMHMn2ISQXJeJ5EM= -golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= @@ -496,15 +399,15 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190912185636-87d9f09c5d89/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/bsm/ratelimit.v1 v1.0.0-20160220154919-db14e161995a/go.mod h1:KF9sEfUPAXdG8Oev9e99iLGnl2uJMjc5B+4y3O7x610= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -515,14 +418,9 @@ gopkg.in/jcmturner/gokrb5.v7 v7.2.3/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuv gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= -gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190213234257-ec84240a7772 h1:hhsSf/5z74Ck/DJYc+R8zpq8KGm7uJvpdLRQED/IedA= -gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190213234257-ec84240a7772/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= -gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190709231704-1e4459ed25ff h1:uuol9OUzSvZntY1v963NAbVd7A+PHLMz1FlCe3Lorcs= -gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190709231704-1e4459ed25ff/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= -gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200316214253-d7b0ff38cac9/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= +gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200603215123-a4a8cb9d2cbc h1:17cdygvFw3DEyNMh81Bk687W74d5pcC5qEKQICv9N6g= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200603215123-a4a8cb9d2cbc/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/redis.v4 v4.2.4/go.mod h1:8KREHdypkCEojGKQcjMqAODMICIVwZAONWq8RowTITA= -gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= From 8f54946674435da9888694a9453a9e1a1353b185 Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Wed, 9 Sep 2020 10:00:12 -0500 Subject: [PATCH 45/66] Update readme for header_sync --- dockerfiles/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dockerfiles/README.md b/dockerfiles/README.md index 4e9ea585d..8f97eecfb 100644 --- a/dockerfiles/README.md +++ b/dockerfiles/README.md @@ -36,11 +36,16 @@ From project root directory: docker build -f dockerfiles/header_sync/Dockerfile . -t header_sync:latest ``` +You can also use the `dockerbuild` make task for any image in dockerfiles with `make dockerbuild IMAGE=header_sync` + + ### Run ``` docker run -e DATABASE_USER=user -e DATABASE_PASSWORD=password -e DATABASE_HOSTNAME=host -e DATABASE_PORT=port -e DATABASE_NAME=name -e STARTING_BLOCK_NUMBER=0 -e CLIENT_IPCPATH=path -it header_sync:latest ``` +`header_sync` had a make task to run it `make header_sync CLIENT_IPCPATH=path NAME=database_name`. It's meant for development, not production. + ## resetHeaderCheckCount Dockerfile for resetting the `headers.check_count` to zero in the database for the given header, so that the execute command will transform the associated events. This is useful in case an event log is missing. From f5f34db0ada87afd4f1016e266b90f240fe56aeb Mon Sep 17 00:00:00 2001 From: Rob Mulholand Date: Mon, 21 Sep 2020 15:03:01 -0500 Subject: [PATCH 46/66] (VDB-1614) Update Geth dep - use latest version of state-diffing patch --- go.mod | 12 +-- go.sum | 246 +++++++++++++++------------------------------------------ 2 files changed, 72 insertions(+), 186 deletions(-) diff --git a/go.mod b/go.mod index 4ef6d90c7..aec8278d4 100644 --- a/go.mod +++ b/go.mod @@ -5,22 +5,24 @@ go 1.12 require ( github.com/ClickHouse/clickhouse-go v1.4.3 // indirect github.com/dave/jennifer v1.3.0 - github.com/ethereum/go-ethereum v1.9.8 + github.com/ethereum/go-ethereum v1.9.21 github.com/hashicorp/golang-lru v0.5.4 github.com/hpcloud/tail v1.0.0 + github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jmoiron/sqlx v1.2.0 github.com/lib/pq v1.0.0 github.com/mitchellh/go-homedir v1.1.0 - github.com/onsi/ginkgo v1.10.1 - github.com/onsi/gomega v1.10.0 + github.com/onsi/ginkgo v1.14.0 + github.com/onsi/gomega v1.10.1 github.com/pressly/goose v2.7.0-rc5+incompatible github.com/sirupsen/logrus v1.2.0 github.com/spf13/cobra v0.0.3 github.com/spf13/viper v1.3.2 - golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0 + golang.org/x/net v0.0.0-20200822124328-c89045814202 + google.golang.org/appengine v1.6.6 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 ) -replace github.com/ethereum/go-ethereum => github.com/makerdao/go-ethereum v1.9.15-statechange-filter +replace github.com/ethereum/go-ethereum => github.com/makerdao/go-ethereum v1.9.21-rc1 replace gopkg.in/urfave/cli.v1 => gopkg.in/urfave/cli.v1 v1.20.0 diff --git a/go.sum b/go.sum index f0f1fccdd..c78295b51 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,3 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= @@ -16,59 +15,29 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ClickHouse/clickhouse-go v1.4.3 h1:iAFMa2UrQdR5bHJ2/yaSLffZkxpcOYQMCUuKeNXGdqc= github.com/ClickHouse/clickhouse-go v1.4.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= -github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/Shopify/sarama v1.23.1/go.mod h1:XLH1GYJnLVE0XCr6KdJGVJRTwY30moWNJ4sERjXX6fs= -github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VictoriaMetrics/fastcache v1.5.7 h1:4y6y0G8PRzszQUYIQHHssv/jgPHAb5qQuuDNdCbyAgw= github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8= -github.com/aead/siphash v1.0.1 h1:FwHfE/T45KPKYuuSAKyyvE+oPWcaQ+CUmFW0bPlM+kg= -github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= -github.com/allegro/bigcache v1.2.1 h1:hg1sY1raCwic3Vnsvje6TT7/pnZba83LeFck5NrFKSc= -github.com/allegro/bigcache v1.2.1/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= -github.com/aristanetworks/fsnotify v1.4.2/go.mod h1:D/rtu7LpjYM8tRJphJ0hUBYpjai8SfX+aSNsWDTq/Ks= -github.com/aristanetworks/glog v0.0.0-20180419172825-c15b03b3054f/go.mod h1:KASm+qXFKs/xjSoWn30NrWBBvdTTQq+UjkhjEJHfSFA= github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847 h1:rtI0fD4oG/8eVokGVPYJEW1F88p1ZNgXiEIs9thEE4A= github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= -github.com/aristanetworks/goarista v0.0.0-20191106175434-873d404c7f40 h1:ZdRuixFqR3mfx4FHzclG3COrRgWrYq0VhNgIoYoObcM= -github.com/aristanetworks/goarista v0.0.0-20191106175434-873d404c7f40/go.mod h1:Z4RTxGAuYhPzcq8+EdRM+R8M48Ssle2TsWtwRKa+vns= -github.com/aristanetworks/splunk-hec-go v0.3.3/go.mod h1:1VHO9r17b0K7WmOlLb9nTk/2YanvOEnLMUgsFrxBROc= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= +github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6 h1:Eey/GGQ/E5Xp1P2Lyx1qj007hLZfbi0+CoVeJruGCtI= github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= -github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= -github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= -github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo= -github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= -github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d h1:yJzD/yFppdVCf6ApMkVy8cUxV0XrxdP9rVf6D87/Mng= -github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= -github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd h1:R/opQEbFEy9JGkIguV40SvRY1uliPX8ifOvi6ICsFCw= -github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= -github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd h1:qdGvebPBDuYDPGi1WCPjy1tGyMpmDK8IEapSsszn7HE= -github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= -github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723 h1:ZA/jbKoGcVAnER6pCHPEkGdZOV7U1oLUedErBHCUMs0= -github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= -github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc= -github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= -github.com/btcsuite/winsvc v1.0.0 h1:J9B4L7e3oqhXOcm+2IuNApwzQec85lE+QaikUcCs+dk= -github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= -github.com/cespare/cp v1.1.1 h1:nCb6ZLdB7NRaqsm91JtQTAme2SKJzXVsdPIPkyJr1MU= -github.com/cespare/cp v1.1.1/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -77,44 +46,31 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/dave/jennifer v1.3.0 h1:p3tl41zjjCZTNBytMwrUuiAnherNUZktlhPTKoF/sEk= github.com/dave/jennifer v1.3.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= -github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea h1:j4317fAZh7X6GqbFowYdYdI0L9bwxL07jyPZIdepyZ0= github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= -github.com/deckarep/golang-set v1.7.1 h1:SCQV0S6gTtp6itiFrTqI+pfmJ4LN85S1YzhDf9rTHJQ= -github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/dop251/goja v0.0.0-20200219165308-d1232e640a87/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= -github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= -github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c h1:JHHhtb9XWJrGNMcrVP6vyzO4dusgi/HnceHTgxSejUM= github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= -github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc h1:jtW8jbpkO4YirRSyepBOH8E+2HEw6/hKkBvFPwhUN8c= github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= -github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= -github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/garyburd/redigo v1.6.0/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= -github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 h1:f6D9Hr8xV8uYKlyuj8XIruxlh9WjVjdh1gIicAS7ays= -github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0 h1:8HUsc87TaSWLKwrnumgC8/YconD2fJQsRJAsWaPg2ic= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= @@ -123,34 +79,34 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2-0.20190517061210-b285ee9cfc6c h1:zqAKixg3cTcIasAMJV+EcfVbWwLpOZ7LeoWJvcuD/5Q= -github.com/golang/protobuf v1.3.2-0.20190517061210-b285ee9cfc6c/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/golang/snappy v0.0.2-0.20200707131729-196ae77b8a26 h1:lMm2hD9Fy0ynom5+85/pbdkiYcBqM1JWmhpAXLmy0fw= +github.com/golang/snappy v0.0.2-0.20200707131729-196ae77b8a26/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989 h1:giknQ4mEuDFmmHSrGcbargOuLHQGtywqo4mheITex54= github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= -github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/holiman/uint256 v1.1.1 h1:4JywC80b+/hSfljFlEBLHrrh+CIONLDz9NuFl0af4Mw= +github.com/holiman/uint256 v1.1.1/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= @@ -159,32 +115,14 @@ github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= -github.com/influxdata/influxdb1-client v0.0.0-20190809212627-fc22c7df067e/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= -github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= -github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= -github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= -github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89 h1:12K8AlpT0/6QUXSfV0yi4Q0jkbq8NDtIKFtF61AoqV0= -github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= -github.com/jrick/logrotate v1.0.0 h1:lQ1bL/n9mBNeIXoTUoYRlK4dHuNJVofX9oWqBtPnSzI= -github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356 h1:I/yrLt2WilKxlQKCM52clh5rGzTKpVctGT1lH4Dc8Jw= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= -github.com/karalabe/usb v0.0.0-20191104083709-911d15fe12a9 h1:ZHuwnjpP8LsVsUYqTqeVAI+GfDfJ6UNPrExZF+vX/DQ= -github.com/karalabe/usb v0.0.0-20191104083709-911d15fe12a9/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= -github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23 h1:FOOIBWrEkLgmlgGfMuZT83xIwfPDxEI2OHu6xUmJMFE= -github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/reedsolomon v1.9.2/go.mod h1:CwCi+NUr9pqSVktrkN+Ondf06rkhYZ/pcNv7fu+8Un4= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= @@ -199,8 +137,8 @@ github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/makerdao/go-ethereum v1.9.15-statechange-filter h1:jLX9CleT+i/gUn5LLTUoor1NXYLHSTnZM1pOFKrg2IA= -github.com/makerdao/go-ethereum v1.9.15-statechange-filter/go.mod h1:slT8bPPRhXsyNTwHQxrOnjuTZ1sDXRajW11EkJ84QJ0= +github.com/makerdao/go-ethereum v1.9.21-rc1 h1:izuLZxMHei6UX89Q2Ld1v5PYcuYEhw34/Ncf/e/WPW8= +github.com/makerdao/go-ethereum v1.9.21-rc1/go.mod h1:RXAVzbGrSGmDkDnHymruTAIEjUR3E4TX0EOpaj702sI= github.com/mattn/go-colorable v0.1.0 h1:v2XXALHHh6zHfYTJ+cSkwtyffnaOyR1MXaA91mTrb8o= github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= @@ -217,13 +155,10 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= @@ -231,69 +166,44 @@ github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c h1:1RHs3t github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= -github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.10.0 h1:Gwkk+PTu/nfOwNMtUB/mRUv0X7ewW5dO4AERT1ThVKo= -github.com/onsi/gomega v1.10.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= -github.com/openconfig/gnmi v0.0.0-20190823184014-89b2bf29312c/go.mod h1:t+O9It+LKzfOAhKTT5O0ehDix+MTqbtT0T9t+7zzOvc= -github.com/openconfig/reference v0.0.0-20190727015836-8dfd928c9696/go.mod h1:ym2A+zigScwkSEb/cVQB0/ZMpU3rqiH6X7WRRsxgOGw= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222 h1:goeTyGkArOZIVOMA0dQbyuPWGNQJZGPwPu/QS9GlpnA= github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= -github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= -github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pressly/goose v2.7.0-rc5+incompatible h1:txvo810iG1P/rafOx31LYDlOyikBK8A/8prKP4j066w= github.com/pressly/goose v2.7.0-rc5+incompatible/go.mod h1:m+QHWCqxR3k8D9l7qfzuC/djtlfzxr34mozWDYEu1z8= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150 h1:ZeU+auZj1iNzN8iVhff6M38Mfu73FQiJve/GEXYJBjE= github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/prometheus/tsdb v0.10.0 h1:If5rVCMTp6W2SiRAQFlbpJNgVlgMEd+U2GZckwK38ic= -github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4= -github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= -github.com/rjeczalik/notify v0.9.2 h1:MiTWrPj55mNDHEiIX5YUSKefw/+lCQVoAFmD6oQm5w8= -github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa4QEjJeqM= github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00 h1:8DPul/X0IT/1TNMIxoKLwdemEOBBHDC/K4EB16Cw5WE= github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= -github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= -github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521 h1:3hxavr+IHMsQBrYUPQM5v0CgENFktkkbg1sfpgM3h20= github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubrpZHtQLnOf6EwhN2hEMusxZOhcW9H3UQ= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/shirou/gopsutil v2.20.5-0.20200531151128-663af789c085+incompatible h1:+gAR1bMhuoQnZMTWFIvp7ukynULPsteLzG+siZKLtD8= -github.com/shirou/gopsutil v2.20.5-0.20200531151128-663af789c085+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v2.20.5+incompatible h1:tYH07UPoQt0OCQdgWWMgYHy3/a9bcxNpBIysykNIP7I= +github.com/shirou/gopsutil v2.20.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -312,8 +222,6 @@ github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= -github.com/status-im/keycard-go v0.0.0-20191119114148-6dd40a46baa0 h1:5UdlDkkBoPrJfh7zkfoR3X5utJhNs/MCQysK3x0ycgg= -github.com/status-im/keycard-go v0.0.0-20191119114148-6dd40a46baa0/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 h1:gIlAHnH1vJb5vwEjIp5kBj/eu99p/bl0Ay2goiPe5xE= github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw= github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 h1:njlZPzLwU639dk2kqnCPPv+wNjq7Xb6EfUxe/oX0/NM= @@ -325,111 +233,87 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d h1:gZZadD8H+fF+n9CmNhYL1Y0dJB+kLOmKd7FbPJLeGHs= -github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= -github.com/templexxx/cpufeat v0.0.0-20180724012125-cef66df7f161/go.mod h1:wM7WEvslTq+iOEAMDLSzhVuOt5BRZ05WirO+b09GHQU= -github.com/templexxx/xor v0.0.0-20181023030647-4e92f724b73b/go.mod h1:5XA7W9S6mni3h5uvOC75dA3m9CCCaS83lltmc0ukdi4= -github.com/tjfoc/gmsm v1.0.1/go.mod h1:XxO4hdhhrzAd+G4CjDqaOkd0hUzmtPR/d3EiBBMn/wc= +github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca h1:Ld/zXl5t4+D69SiV4JoN7kkfvJdOWlPpfxrzxpLMoUk= +github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZW24CsNJDfeh9Ex6Pm0Rcpc7qrgKBiL44vF4= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= -github.com/tyler-smith/go-bip39 v1.0.2 h1:+t3w+KwLXO6154GNJY+qUtIxLTmFjfUmpguQT1OlOT8= -github.com/tyler-smith/go-bip39 v1.0.2/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 h1:1cngl9mPEoITZG8s8cVcUy5CeIBYhEESkOB7m6Gmkrk= github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees= -github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= -github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/xtaci/kcp-go v5.4.5+incompatible/go.mod h1:bN6vIwHQbfHaHtFpEssmWsN45a+AZwO7eyRCmEIbtvE= -github.com/xtaci/lossyconn v0.0.0-20190602105132-8df528c0c9ae/go.mod h1:gXtu8J62kEgmN++bm9BVICuT/e8yiLI2KFobd/TRFsE= -golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5 h1:bselrhR0Or1vomJZC8ZIjWtbDmn9OYFLX5Ik9alpJpE= -golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= -golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4 h1:QmwruyY+bKbDDL0BaglrbZABEali68eoMFhTZpCjYVA= -golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190912160710-24e19bdeb0f2 h1:4dVFTC832rPn4pomLSz1vA+are2+dU19w1H8OngV7nc= -golang.org/x/net v0.0.0-20190912160710-24e19bdeb0f2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0 h1:Jcxah/M+oLZ/R4/z5RzfPzGbPXnVDPkEDtf2JnuxN+U= -golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5 h1:mzjBh+S5frKOsOBobWIMAbXavqjmgO17k/2puhcFR94= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190912141932-bc967efca4b8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8 h1:AvbQYmiaaaza3cW3QXRyPo5kYgpFIzOAfeAAN7m3qQ4= +golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190912185636-87d9f09c5d89/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/bsm/ratelimit.v1 v1.0.0-20160220154919-db14e161995a/go.mod h1:KF9sEfUPAXdG8Oev9e99iLGnl2uJMjc5B+4y3O7x610= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= -gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= -gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= -gopkg.in/jcmturner/gokrb5.v7 v7.2.3/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= -gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= -gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200603215123-a4a8cb9d2cbc h1:17cdygvFw3DEyNMh81Bk687W74d5pcC5qEKQICv9N6g= -gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200603215123-a4a8cb9d2cbc/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= -gopkg.in/redis.v4 v4.2.4/go.mod h1:8KREHdypkCEojGKQcjMqAODMICIVwZAONWq8RowTITA= +gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6 h1:a6cXbcDDUkSBlpnkWV1bJ+vv3mOgQEltEJ2rPxroVu0= +gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= -gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 22447dca4407dea728e4b3626459fcbd725804d4 Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Mon, 14 Sep 2020 15:26:09 -0500 Subject: [PATCH 47/66] Inject the NewCheckedHeaders repo into LogExtractor This makes no changes, but will let each plugin provide a way to check their headers. --- cmd/backfillEvents.go | 3 ++- cmd/execute.go | 3 ++- libraries/shared/logs/extractor.go | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cmd/backfillEvents.go b/cmd/backfillEvents.go index 959fae833..a7838f330 100644 --- a/cmd/backfillEvents.go +++ b/cmd/backfillEvents.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/makerdao/vulcanizedb/libraries/shared/logs" + "github.com/makerdao/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/makerdao/vulcanizedb/utils" "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -50,7 +51,7 @@ func backFillEvents() error { blockChain := getBlockChain() db := utils.LoadPostgres(databaseConfig, blockChain.Node()) - extractor := logs.NewLogExtractor(&db, blockChain) + extractor := logs.NewLogExtractor(&db, blockChain, repositories.NewCheckedHeadersRepository(&db)) for _, initializer := range ethEventInitializers { transformer := initializer(&db) diff --git a/cmd/execute.go b/cmd/execute.go index a8641e017..ef23a54cc 100644 --- a/cmd/execute.go +++ b/cmd/execute.go @@ -26,6 +26,7 @@ import ( "github.com/makerdao/vulcanizedb/libraries/shared/logs" "github.com/makerdao/vulcanizedb/libraries/shared/transformer" "github.com/makerdao/vulcanizedb/libraries/shared/watcher" + "github.com/makerdao/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/makerdao/vulcanizedb/pkg/fs" "github.com/makerdao/vulcanizedb/utils" "github.com/sirupsen/logrus" @@ -96,7 +97,7 @@ func executeTransformers() { // Use WaitGroup to wait on both goroutines var wg sync.WaitGroup if len(ethEventInitializers) > 0 { - extractor := logs.NewLogExtractor(&db, blockChain) + extractor := logs.NewLogExtractor(&db, blockChain, repositories.NewCheckedHeadersRepository(&db)) delegator := logs.NewLogDelegator(&db) eventHealthCheckMessage := []byte("event watcher starting\n") statusWriter := fs.NewStatusWriter(healthCheckFile, eventHealthCheckMessage) diff --git a/libraries/shared/logs/extractor.go b/libraries/shared/logs/extractor.go index 5dadde0ab..3e1c6bbf3 100644 --- a/libraries/shared/logs/extractor.go +++ b/libraries/shared/logs/extractor.go @@ -62,9 +62,9 @@ type LogExtractor struct { RecheckHeaderCap int64 } -func NewLogExtractor(db *postgres.DB, bc core.BlockChain) *LogExtractor { +func NewLogExtractor(db *postgres.DB, bc core.BlockChain, chr datastore.CheckedHeadersRepository) *LogExtractor { return &LogExtractor{ - CheckedHeadersRepository: repositories.NewCheckedHeadersRepository(db), + CheckedHeadersRepository: chr, CheckedLogsRepository: repositories.NewCheckedLogsRepository(db), Fetcher: fetcher.NewLogFetcher(bc), HeaderRepository: repositories.NewHeaderRepository(db), From 6aa1af250f62af9b900bb143a0b787dd1ff874e7 Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Mon, 14 Sep 2020 16:29:32 -0500 Subject: [PATCH 48/66] Create a schema for header checking in test The code doesn't use this yet, this just gets us a clean setup/teardown. --- .../checked_headers_repository.go | 2 +- .../checked_headers_repository_test.go | 34 ++++++++++++++++--- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/pkg/datastore/postgres/repositories/checked_headers_repository.go b/pkg/datastore/postgres/repositories/checked_headers_repository.go index 25446e5a3..b599ccb39 100644 --- a/pkg/datastore/postgres/repositories/checked_headers_repository.go +++ b/pkg/datastore/postgres/repositories/checked_headers_repository.go @@ -29,7 +29,7 @@ type CheckedHeadersRepository struct { db *postgres.DB } -func NewCheckedHeadersRepository(db *postgres.DB) CheckedHeadersRepository { +func NewCheckedHeadersRepository(db *postgres.DB, schemaName string) CheckedHeadersRepository { return CheckedHeadersRepository{db: db} } diff --git a/pkg/datastore/postgres/repositories/checked_headers_repository_test.go b/pkg/datastore/postgres/repositories/checked_headers_repository_test.go index 7fae6b331..c5049c5ec 100644 --- a/pkg/datastore/postgres/repositories/checked_headers_repository_test.go +++ b/pkg/datastore/postgres/repositories/checked_headers_repository_test.go @@ -17,6 +17,7 @@ package repositories_test import ( + "fmt" "math/rand" "github.com/makerdao/vulcanizedb/pkg/core" @@ -30,13 +31,36 @@ import ( var _ = Describe("Checked Headers repository", func() { var ( - db = test_config.NewTestDB(test_config.NewTestNode()) - repo datastore.CheckedHeadersRepository + db = test_config.NewTestDB(test_config.NewTestNode()) + repo datastore.CheckedHeadersRepository + pluginSchemaName = "plugin" ) BeforeEach(func() { test_config.CleanTestDB(db) - repo = repositories.NewCheckedHeadersRepository(db) + createSchema := `CREATE SCHEMA IF NOT EXISTS %s` + _, schemaError := db.Exec(fmt.Sprintf(createSchema, pluginSchemaName)) + if schemaError != nil { + Fail(fmt.Errorf("Could not create schema %s, err %w", pluginSchemaName, schemaError).Error()) + } + + createTable := `CREATE TABLE %s.checked_headers ( +check_count INTEGER NOT NULL DEFAULT 0, +header_id INTEGER NOT NULL REFERENCES public.headers(id) ON DELETE CASCADE +);` + _, tableError := db.Exec(fmt.Sprintf(createTable, pluginSchemaName)) + if tableError != nil { + Fail(fmt.Errorf("Could not create table %s, err %w", pluginSchemaName, tableError).Error()) + } + + repo = repositories.NewCheckedHeadersRepository(db, pluginSchemaName) + }) + + AfterEach(func() { + _, err := db.Exec(fmt.Sprintf("DROP SCHEMA IF EXISTS %s CASCADE;", pluginSchemaName)) + if err != nil { + Fail(fmt.Errorf("Could not drop schema %s, err %w", pluginSchemaName, err).Error()) + } }) Describe("MarkHeaderChecked", func() { @@ -226,7 +250,7 @@ var _ = Describe("Checked Headers repository", func() { It("only returns headers associated with any node", func() { dbTwo := test_config.NewTestDB(core.Node{ID: "second"}) headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo) - repoTwo := repositories.NewCheckedHeadersRepository(dbTwo) + repoTwo := repositories.NewCheckedHeadersRepository(dbTwo, pluginSchemaName) for _, n := range blockNumbers { _, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n + 10)) Expect(err).NotTo(HaveOccurred()) @@ -321,7 +345,7 @@ var _ = Describe("Checked Headers repository", func() { It("returns headers associated with any node", func() { dbTwo := test_config.NewTestDB(core.Node{ID: "second"}) headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo) - repoTwo := repositories.NewCheckedHeadersRepository(dbTwo) + repoTwo := repositories.NewCheckedHeadersRepository(dbTwo, pluginSchemaName) for _, n := range blockNumbers { _, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n + 10)) Expect(err).NotTo(HaveOccurred()) From 5e39dcae90f51ff1043b137d52b7c9b70843995a Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Wed, 16 Sep 2020 10:13:05 -0500 Subject: [PATCH 49/66] Make the CHeckedHeadersRepository use a schema The checked_headers table should not be in the public schema, it is provided by the plugin with the expected name checked_headers. --- cmd/backfillEvents.go | 2 +- cmd/execute.go | 2 +- cmd/resetHeaderCheckCount.go | 2 +- .../checked_headers_repository.go | 60 +++++++++------ .../checked_headers_repository_test.go | 76 +++++++++++-------- 5 files changed, 86 insertions(+), 56 deletions(-) diff --git a/cmd/backfillEvents.go b/cmd/backfillEvents.go index a7838f330..ac91a1a09 100644 --- a/cmd/backfillEvents.go +++ b/cmd/backfillEvents.go @@ -51,7 +51,7 @@ func backFillEvents() error { blockChain := getBlockChain() db := utils.LoadPostgres(databaseConfig, blockChain.Node()) - extractor := logs.NewLogExtractor(&db, blockChain, repositories.NewCheckedHeadersRepository(&db)) + extractor := logs.NewLogExtractor(&db, blockChain, repositories.NewCheckedHeadersRepository(&db, genConfig.Schema)) for _, initializer := range ethEventInitializers { transformer := initializer(&db) diff --git a/cmd/execute.go b/cmd/execute.go index ef23a54cc..92f9dfc82 100644 --- a/cmd/execute.go +++ b/cmd/execute.go @@ -97,7 +97,7 @@ func executeTransformers() { // Use WaitGroup to wait on both goroutines var wg sync.WaitGroup if len(ethEventInitializers) > 0 { - extractor := logs.NewLogExtractor(&db, blockChain, repositories.NewCheckedHeadersRepository(&db)) + extractor := logs.NewLogExtractor(&db, blockChain, repositories.NewCheckedHeadersRepository(&db, genConfig.Schema)) delegator := logs.NewLogDelegator(&db) eventHealthCheckMessage := []byte("event watcher starting\n") statusWriter := fs.NewStatusWriter(healthCheckFile, eventHealthCheckMessage) diff --git a/cmd/resetHeaderCheckCount.go b/cmd/resetHeaderCheckCount.go index 86df7e03a..43411e2a6 100644 --- a/cmd/resetHeaderCheckCount.go +++ b/cmd/resetHeaderCheckCount.go @@ -64,6 +64,6 @@ func init() { func resetHeaderCount(blockNumber int64) error { blockChain := getBlockChain() db := utils.LoadPostgres(databaseConfig, blockChain.Node()) - repo := repositories.NewCheckedHeadersRepository(&db) + repo := repositories.NewCheckedHeadersRepository(&db, genConfig.Schema) return repo.MarkSingleHeaderUnchecked(blockNumber) } diff --git a/pkg/datastore/postgres/repositories/checked_headers_repository.go b/pkg/datastore/postgres/repositories/checked_headers_repository.go index b599ccb39..0bc440a39 100644 --- a/pkg/datastore/postgres/repositories/checked_headers_repository.go +++ b/pkg/datastore/postgres/repositories/checked_headers_repository.go @@ -17,31 +17,47 @@ package repositories import ( + "fmt" + "github.com/makerdao/vulcanizedb/pkg/core" "github.com/makerdao/vulcanizedb/pkg/datastore/postgres" ) const ( - insertCheckedHeaderQuery = `UPDATE public.headers SET check_count = (SELECT check_count WHERE id = $1) + 1 WHERE id = $1` + insertCheckedHeaderQuery = ` +INSERT INTO %s.checked_headers (check_count, header_id) +VALUES (1, $1) +ON CONFLICT (header_id) DO + UPDATE SET check_count = + (SELECT checked_headers.check_count WHERE checked_headers.header_id = $1) + 1 + WHERE checked_headers.header_id = $1` ) type CheckedHeadersRepository struct { - db *postgres.DB + db *postgres.DB + schemaName string } func NewCheckedHeadersRepository(db *postgres.DB, schemaName string) CheckedHeadersRepository { - return CheckedHeadersRepository{db: db} + return CheckedHeadersRepository{db: db, schemaName: schemaName} } // Increment check_count for header func (repo CheckedHeadersRepository) MarkHeaderChecked(headerID int64) error { - _, err := repo.db.Exec(insertCheckedHeaderQuery, headerID) + queryString := fmt.Sprintf(insertCheckedHeaderQuery, repo.schemaName) + _, err := repo.db.Exec(queryString, headerID) return err } // Zero out check count for header with the given block number func (repo CheckedHeadersRepository) MarkSingleHeaderUnchecked(blockNumber int64) error { - _, err := repo.db.Exec(`UPDATE public.headers SET check_count = 0 WHERE block_number = $1`, blockNumber) + queryString := fmt.Sprintf(`UPDATE %s.checked_headers ch + SET check_count = 0 + FROM public.headers h + WHERE ch.header_id = h.id + AND h.block_number = $1`, repo.schemaName) + + _, err := repo.db.Exec(queryString, blockNumber) return err } @@ -54,25 +70,25 @@ func (repo CheckedHeadersRepository) UncheckedHeaders(startingBlockNumber, endin recheckOffsetMultiplier = 15 ) + query = fmt.Sprintf(`SELECT h.id, h.block_number, h.hash + FROM public.headers h + LEFT JOIN %s.checked_headers ch + ON ch.header_id = h.id`, repo.schemaName) + if endingBlockNumber == -1 { - query = `SELECT id, block_number, hash - FROM public.headers - WHERE (check_count < 1 - AND block_number >= $1) - OR (check_count < $2 - AND block_number <= ((SELECT MAX(block_number) FROM public.headers) - ($3 * check_count * (check_count + 1) / 2)))` - err = repo.db.Select(&result, query, startingBlockNumber, checkCount, recheckOffsetMultiplier) + noEndingBlockQuery := fmt.Sprintf(`%s + WHERE ((ch.check_count IS NULL OR ch.check_count < 1) AND h.block_number >= $1) + OR ((ch.check_count IS NULL OR ch.check_count < $2) + AND h.block_number <= ((SELECT MAX(block_number) FROM public.headers) - ($3 * ch.check_count * (ch.check_count + 1) / 2)))`, query) + err = repo.db.Select(&result, noEndingBlockQuery, startingBlockNumber, checkCount, recheckOffsetMultiplier) } else { - query = `SELECT id, block_number, hash - FROM public.headers - WHERE (check_count < 1 - AND block_number >= $1 - AND block_number <= $2) - OR (check_count < $3 - AND block_number >= $1 - AND block_number <= $2 - AND block_number <= ((SELECT MAX(block_number) FROM public.headers) - ($4 * (check_count * (check_count + 1) / 2))))` - err = repo.db.Select(&result, query, startingBlockNumber, endingBlockNumber, checkCount, recheckOffsetMultiplier) + endingBlockQuery := fmt.Sprintf(`%s + WHERE ((ch.check_count IS NULL OR ch.check_count < 1) AND h.block_number >= $1 AND h.block_number <= $2) + OR ((ch.check_count IS NULL OR ch.check_count < $3) + AND h.block_number >= $1 + AND h.block_number <= $2 + AND h.block_number <= ((SELECT MAX(block_number) FROM public.headers) - ($4 * (ch.check_count * (ch.check_count + 1) / 2))))`, query) + err = repo.db.Select(&result, endingBlockQuery, startingBlockNumber, endingBlockNumber, checkCount, recheckOffsetMultiplier) } return result, err diff --git a/pkg/datastore/postgres/repositories/checked_headers_repository_test.go b/pkg/datastore/postgres/repositories/checked_headers_repository_test.go index c5049c5ec..5de6a1655 100644 --- a/pkg/datastore/postgres/repositories/checked_headers_repository_test.go +++ b/pkg/datastore/postgres/repositories/checked_headers_repository_test.go @@ -22,6 +22,7 @@ import ( "github.com/makerdao/vulcanizedb/pkg/core" "github.com/makerdao/vulcanizedb/pkg/datastore" + "github.com/makerdao/vulcanizedb/pkg/datastore/postgres" "github.com/makerdao/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/makerdao/vulcanizedb/pkg/fakes" "github.com/makerdao/vulcanizedb/test_config" @@ -29,6 +30,13 @@ import ( . "github.com/onsi/gomega" ) +func selectCheckedHeaders(db *postgres.DB, schemaName string, headerID int64) (int, error) { + var checkedCount int + queryString := fmt.Sprintf(`SELECT check_count FROM %s.checked_headers WHERE header_id = $1`, schemaName) + fetchErr := db.Get(&checkedCount, queryString, headerID) + return checkedCount, fetchErr +} + var _ = Describe("Checked Headers repository", func() { var ( db = test_config.NewTestDB(test_config.NewTestNode()) @@ -38,21 +46,21 @@ var _ = Describe("Checked Headers repository", func() { BeforeEach(func() { test_config.CleanTestDB(db) - createSchema := `CREATE SCHEMA IF NOT EXISTS %s` - _, schemaError := db.Exec(fmt.Sprintf(createSchema, pluginSchemaName)) + prepareSchema := ` +CREATE SCHEMA IF NOT EXISTS %[1]s; + +CREATE TABLE %[1]s.checked_headers ( + id SERIAL PRIMARY KEY, + check_count INTEGER NOT NULL DEFAULT 0, + header_id INTEGER NOT NULL REFERENCES public.headers(id) ON DELETE CASCADE, + UNIQUE (header_id) +); +` + _, schemaError := db.Exec(fmt.Sprintf(prepareSchema, pluginSchemaName)) if schemaError != nil { Fail(fmt.Errorf("Could not create schema %s, err %w", pluginSchemaName, schemaError).Error()) } - createTable := `CREATE TABLE %s.checked_headers ( -check_count INTEGER NOT NULL DEFAULT 0, -header_id INTEGER NOT NULL REFERENCES public.headers(id) ON DELETE CASCADE -);` - _, tableError := db.Exec(fmt.Sprintf(createTable, pluginSchemaName)) - if tableError != nil { - Fail(fmt.Errorf("Could not create table %s, err %w", pluginSchemaName, tableError).Error()) - } - repo = repositories.NewCheckedHeadersRepository(db, pluginSchemaName) }) @@ -72,8 +80,7 @@ header_id INTEGER NOT NULL REFERENCES public.headers(id) ON DELETE CASCADE err := repo.MarkHeaderChecked(headerID) Expect(err).NotTo(HaveOccurred()) - var checkedCount int - fetchErr := db.Get(&checkedCount, `SELECT check_count FROM public.headers WHERE id = $1`, headerID) + checkedCount, fetchErr := selectCheckedHeaders(db, pluginSchemaName, headerID) Expect(fetchErr).NotTo(HaveOccurred()) Expect(checkedCount).To(Equal(1)) }) @@ -89,8 +96,7 @@ header_id INTEGER NOT NULL REFERENCES public.headers(id) ON DELETE CASCADE updateErr := repo.MarkHeaderChecked(headerID) Expect(updateErr).NotTo(HaveOccurred()) - var checkedCount int - fetchErr := db.Get(&checkedCount, `SELECT check_count FROM public.headers WHERE id = $1`, headerID) + checkedCount, fetchErr := selectCheckedHeaders(db, pluginSchemaName, headerID) Expect(fetchErr).NotTo(HaveOccurred()) Expect(checkedCount).To(Equal(2)) }) @@ -124,14 +130,16 @@ header_id INTEGER NOT NULL REFERENCES public.headers(id) ON DELETE CASCADE err := repo.MarkSingleHeaderUnchecked(blockNumberTwo) Expect(err).NotTo(HaveOccurred()) - var headerOneCheckCount, headerTwoCheckCount, headerThreeCheckCount int - getHeaderOneErr := db.Get(&headerOneCheckCount, `SELECT check_count FROM public.headers WHERE id = $1`, headerIdOne) + + headerOneCheckCount, getHeaderOneErr := selectCheckedHeaders(db, pluginSchemaName, headerIdOne) Expect(getHeaderOneErr).NotTo(HaveOccurred()) Expect(headerOneCheckCount).To(Equal(1)) - getHeaderTwoErr := db.Get(&headerTwoCheckCount, `SELECT check_count FROM public.headers WHERE id = $1`, headerIdTwo) + + headerTwoCheckCount, getHeaderTwoErr := selectCheckedHeaders(db, pluginSchemaName, headerIdTwo) Expect(getHeaderTwoErr).NotTo(HaveOccurred()) Expect(headerTwoCheckCount).To(BeZero()) - getHeaderThreeErr := db.Get(&headerThreeCheckCount, `SELECT check_count FROM public.headers WHERE id = $1`, headerIdThree) + + headerThreeCheckCount, getHeaderThreeErr := selectCheckedHeaders(db, pluginSchemaName, headerIdThree) Expect(getHeaderThreeErr).NotTo(HaveOccurred()) Expect(headerThreeCheckCount).To(Equal(1)) }) @@ -184,7 +192,7 @@ header_id INTEGER NOT NULL REFERENCES public.headers(id) ON DELETE CASCADE }) It("excludes headers that have been checked more than the check count", func() { - _, err = db.Exec(`UPDATE public.headers SET check_count = 1 WHERE id = $1`, secondHeaderID) + err = repo.MarkHeaderChecked(secondHeaderID) Expect(err).NotTo(HaveOccurred()) headers, err := repo.UncheckedHeaders(firstBlock, thirdBlock, uncheckedCheckCount) @@ -197,7 +205,7 @@ header_id INTEGER NOT NULL REFERENCES public.headers(id) ON DELETE CASCADE Describe("when header has already been checked", func() { It("includes header with block number > 15 back from latest with check count of 1", func() { - _, err = db.Exec(`UPDATE public.headers SET check_count = 1 WHERE id = $1`, thirdHeaderID) + err := repo.MarkHeaderChecked(thirdHeaderID) Expect(err).NotTo(HaveOccurred()) headers, err := repo.UncheckedHeaders(firstBlock, lastBlock, recheckCheckCount) @@ -211,7 +219,7 @@ header_id INTEGER NOT NULL REFERENCES public.headers(id) ON DELETE CASCADE excludedHeader := fakes.GetFakeHeader(thirdBlock + 1) excludedHeaderID, createHeaderErr := headerRepository.CreateOrUpdateHeader(excludedHeader) Expect(createHeaderErr).NotTo(HaveOccurred()) - _, updateHeaderErr := db.Exec(`UPDATE public.headers SET check_count = 1 WHERE id = $1`, excludedHeaderID) + updateHeaderErr := repo.MarkHeaderChecked(excludedHeaderID) Expect(updateHeaderErr).NotTo(HaveOccurred()) headers, err := repo.UncheckedHeaders(firstBlock, lastBlock, recheckCheckCount) @@ -222,7 +230,7 @@ header_id INTEGER NOT NULL REFERENCES public.headers(id) ON DELETE CASCADE }) It("includes header with block number > 45 back from latest with check count of 2", func() { - _, err = db.Exec(`UPDATE public.headers SET check_count = 1 WHERE id = $1`, secondHeaderID) + err := repo.MarkHeaderChecked(secondHeaderID) Expect(err).NotTo(HaveOccurred()) headers, err := repo.UncheckedHeaders(firstBlock, lastBlock, recheckCheckCount) @@ -236,8 +244,11 @@ header_id INTEGER NOT NULL REFERENCES public.headers(id) ON DELETE CASCADE excludedHeader := fakes.GetFakeHeader(secondBlock + 1) excludedHeaderID, createHeaderErr := headerRepository.CreateOrUpdateHeader(excludedHeader) Expect(createHeaderErr).NotTo(HaveOccurred()) - _, updateHeaderErr := db.Exec(`UPDATE public.headers SET check_count = 2 WHERE id = $1`, excludedHeaderID) - Expect(updateHeaderErr).NotTo(HaveOccurred()) + + firstCheckErr := repo.MarkHeaderChecked(excludedHeaderID) + Expect(firstCheckErr).NotTo(HaveOccurred()) + secondCheckErr := repo.MarkHeaderChecked(excludedHeaderID) + Expect(secondCheckErr).NotTo(HaveOccurred()) headers, err := repo.UncheckedHeaders(firstBlock, lastBlock, 3) Expect(err).NotTo(HaveOccurred()) @@ -279,7 +290,7 @@ header_id INTEGER NOT NULL REFERENCES public.headers(id) ON DELETE CASCADE }) It("excludes headers that have been checked more than the check count", func() { - _, err = db.Exec(`UPDATE public.headers SET check_count = 1 WHERE id = $1`, headerIDs[1]) + err := repo.MarkHeaderChecked(headerIDs[1]) Expect(err).NotTo(HaveOccurred()) headers, err := repo.UncheckedHeaders(firstBlock, -1, uncheckedCheckCount) @@ -292,7 +303,7 @@ header_id INTEGER NOT NULL REFERENCES public.headers(id) ON DELETE CASCADE Describe("when header has already been checked", func() { It("includes header with block number > 15 back from latest with check count of 1", func() { - _, err = db.Exec(`UPDATE public.headers SET check_count = 1 WHERE id = $1`, thirdHeaderID) + err := repo.MarkHeaderChecked(thirdHeaderID) Expect(err).NotTo(HaveOccurred()) headers, err := repo.UncheckedHeaders(firstBlock, -1, recheckCheckCount) @@ -306,7 +317,7 @@ header_id INTEGER NOT NULL REFERENCES public.headers(id) ON DELETE CASCADE excludedHeader := fakes.GetFakeHeader(thirdBlock + 1) excludedHeaderID, createHeaderErr := headerRepository.CreateOrUpdateHeader(excludedHeader) Expect(createHeaderErr).NotTo(HaveOccurred()) - _, updateHeaderErr := db.Exec(`UPDATE public.headers SET check_count = 1 WHERE id = $1`, excludedHeaderID) + updateHeaderErr := repo.MarkHeaderChecked(excludedHeaderID) Expect(updateHeaderErr).NotTo(HaveOccurred()) headers, err := repo.UncheckedHeaders(firstBlock, -1, recheckCheckCount) @@ -316,8 +327,8 @@ header_id INTEGER NOT NULL REFERENCES public.headers(id) ON DELETE CASCADE Expect(headerBlockNumbers).NotTo(ContainElement(excludedHeader.BlockNumber)) }) - It("includes header with block number > 45 back from latest with check count of 2", func() { - _, err = db.Exec(`UPDATE public.headers SET check_count = 1 WHERE id = $1`, secondHeaderID) + It("includes header with block number > 45 back from latest with check count of 1", func() { + err := repo.MarkHeaderChecked(secondHeaderID) Expect(err).NotTo(HaveOccurred()) headers, err := repo.UncheckedHeaders(firstBlock, -1, recheckCheckCount) @@ -331,7 +342,10 @@ header_id INTEGER NOT NULL REFERENCES public.headers(id) ON DELETE CASCADE excludedHeader := fakes.GetFakeHeader(secondBlock + 1) excludedHeaderID, createHeaderErr := headerRepository.CreateOrUpdateHeader(excludedHeader) Expect(createHeaderErr).NotTo(HaveOccurred()) - _, updateHeaderErr := db.Exec(`UPDATE public.headers SET check_count = 2 WHERE id = $1`, excludedHeaderID) + + updateHeaderErr := repo.MarkHeaderChecked(excludedHeaderID) + Expect(updateHeaderErr).NotTo(HaveOccurred()) + updateHeaderErr = repo.MarkHeaderChecked(excludedHeaderID) Expect(updateHeaderErr).NotTo(HaveOccurred()) headers, err := repo.UncheckedHeaders(firstBlock, -1, 3) From 22fa90093941b16590d9ee2a17a7b06443556a73 Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Wed, 16 Sep 2020 10:17:33 -0500 Subject: [PATCH 50/66] Explicitly call the query a joinQuery Refactor - just a rename --- .../postgres/repositories/checked_headers_repository.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/datastore/postgres/repositories/checked_headers_repository.go b/pkg/datastore/postgres/repositories/checked_headers_repository.go index 0bc440a39..f56e14b58 100644 --- a/pkg/datastore/postgres/repositories/checked_headers_repository.go +++ b/pkg/datastore/postgres/repositories/checked_headers_repository.go @@ -65,12 +65,11 @@ func (repo CheckedHeadersRepository) MarkSingleHeaderUnchecked(blockNumber int64 func (repo CheckedHeadersRepository) UncheckedHeaders(startingBlockNumber, endingBlockNumber, checkCount int64) ([]core.Header, error) { var ( result []core.Header - query string err error recheckOffsetMultiplier = 15 ) - query = fmt.Sprintf(`SELECT h.id, h.block_number, h.hash + joinQuery := fmt.Sprintf(`SELECT h.id, h.block_number, h.hash FROM public.headers h LEFT JOIN %s.checked_headers ch ON ch.header_id = h.id`, repo.schemaName) @@ -79,7 +78,7 @@ func (repo CheckedHeadersRepository) UncheckedHeaders(startingBlockNumber, endin noEndingBlockQuery := fmt.Sprintf(`%s WHERE ((ch.check_count IS NULL OR ch.check_count < 1) AND h.block_number >= $1) OR ((ch.check_count IS NULL OR ch.check_count < $2) - AND h.block_number <= ((SELECT MAX(block_number) FROM public.headers) - ($3 * ch.check_count * (ch.check_count + 1) / 2)))`, query) + AND h.block_number <= ((SELECT MAX(block_number) FROM public.headers) - ($3 * ch.check_count * (ch.check_count + 1) / 2)))`, joinQuery) err = repo.db.Select(&result, noEndingBlockQuery, startingBlockNumber, checkCount, recheckOffsetMultiplier) } else { endingBlockQuery := fmt.Sprintf(`%s @@ -87,7 +86,7 @@ func (repo CheckedHeadersRepository) UncheckedHeaders(startingBlockNumber, endin OR ((ch.check_count IS NULL OR ch.check_count < $3) AND h.block_number >= $1 AND h.block_number <= $2 - AND h.block_number <= ((SELECT MAX(block_number) FROM public.headers) - ($4 * (ch.check_count * (ch.check_count + 1) / 2))))`, query) + AND h.block_number <= ((SELECT MAX(block_number) FROM public.headers) - ($4 * (ch.check_count * (ch.check_count + 1) / 2))))`, joinQuery) err = repo.db.Select(&result, endingBlockQuery, startingBlockNumber, endingBlockNumber, checkCount, recheckOffsetMultiplier) } From 39641c59198d26cb2ce2a8d685d30ad6a191071b Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Wed, 16 Sep 2020 14:57:08 -0500 Subject: [PATCH 51/66] Pass unit tests without CLIENT_IPCPATH The CLIENT_IPCPATH was being used by the contract_watcher/repository/header_repository_test on accident. A helper function it called happened to initialize a geth connection. Remove that in favor of the simpler db config, and move the integration test initialization and warning to integration_test/integration_test. --- .../contract_watcher_transformer_test.go | 8 +-- integration_test/geth_blockchain_test.go | 7 +-- integration_test/integration_test.go | 59 +++++++++++++++++++ .../helpers/test_helpers/database.go | 56 ++++++------------ .../repository/header_repository_test.go | 9 +-- test_config/test_config.go | 15 ----- 6 files changed, 90 insertions(+), 64 deletions(-) create mode 100644 integration_test/integration_test.go diff --git a/integration_test/contract_watcher_transformer_test.go b/integration_test/contract_watcher_transformer_test.go index 5fd15df00..fcff75982 100644 --- a/integration_test/contract_watcher_transformer_test.go +++ b/integration_test/contract_watcher_transformer_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -package integration +package integration_test import ( "fmt" @@ -27,15 +27,15 @@ import ( "github.com/makerdao/vulcanizedb/pkg/contract_watcher/transformer" "github.com/makerdao/vulcanizedb/pkg/core" "github.com/makerdao/vulcanizedb/pkg/datastore" - "github.com/makerdao/vulcanizedb/pkg/datastore/postgres" "github.com/makerdao/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/makerdao/vulcanizedb/test_config" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) var _ = Describe("contractWatcher transformer", func() { var ( - db *postgres.DB + db = test_config.NewTestDB(test_config.NewTestNode()) err error blockChain core.BlockChain headerRepository datastore.HeaderRepository @@ -45,7 +45,7 @@ var _ = Describe("contractWatcher transformer", func() { ) BeforeEach(func() { - db, blockChain = test_helpers.SetupDBandBC() + db, blockChain = SetupDBandBC() headerRepository = repositories.NewHeaderRepository(db) }) diff --git a/integration_test/geth_blockchain_test.go b/integration_test/geth_blockchain_test.go index 1a05778d5..528ce9fbc 100644 --- a/integration_test/geth_blockchain_test.go +++ b/integration_test/geth_blockchain_test.go @@ -25,18 +25,17 @@ import ( "github.com/makerdao/vulcanizedb/pkg/eth/client" "github.com/makerdao/vulcanizedb/pkg/eth/converters" "github.com/makerdao/vulcanizedb/pkg/eth/node" - "github.com/makerdao/vulcanizedb/test_config" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) var _ = Describe("Reading from the Geth blockchain", func() { - var blockChain *eth.BlockChain + var blockChain core.BlockChain BeforeEach(func() { - rawRpcClient, err := rpc.Dial(test_config.TestClient.IPCPath) + rawRpcClient, err := rpc.Dial(TestClient.IPCPath) Expect(err).NotTo(HaveOccurred()) - rpcClient := client.NewRpcClient(rawRpcClient, test_config.TestClient.IPCPath) + rpcClient := client.NewRpcClient(rawRpcClient, TestClient.IPCPath) ethClient := ethclient.NewClient(rawRpcClient) blockChainClient := client.NewEthClient(ethClient) node := node.MakeNode(rpcClient) diff --git a/integration_test/integration_test.go b/integration_test/integration_test.go new file mode 100644 index 000000000..1cd888e7a --- /dev/null +++ b/integration_test/integration_test.go @@ -0,0 +1,59 @@ +package integration_test + +import ( + "errors" + + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/rpc" + "github.com/makerdao/vulcanizedb/pkg/config" + "github.com/makerdao/vulcanizedb/pkg/core" + "github.com/makerdao/vulcanizedb/pkg/datastore/postgres" + "github.com/makerdao/vulcanizedb/pkg/eth" + "github.com/makerdao/vulcanizedb/pkg/eth/client" + "github.com/makerdao/vulcanizedb/pkg/eth/converters" + "github.com/makerdao/vulcanizedb/pkg/eth/node" + "github.com/makerdao/vulcanizedb/test_config" + . "github.com/onsi/gomega" + "github.com/sirupsen/logrus" +) + +var TestClient config.Client + +func init() { + ipc := test_config.TestConfig.GetString("client.ipcPath") + + // If we don't have an ipc path in the config file, check the env variable + if ipc == "" { + test_config.TestConfig.BindEnv("url", "CLIENT_IPCPATH") + ipc = test_config.TestConfig.GetString("url") + } + if ipc == "" { + logrus.Fatal(errors.New("testing.toml IPC path or $CLIENT_IPCPATH env variable need to be set")) + } + + TestClient = config.Client{ + IPCPath: ipc, + } +} + +func SetupDBandBC() (*postgres.DB, core.BlockChain) { + con := TestClient + testIPC := con.IPCPath + rawRPCClient, err := rpc.Dial(testIPC) + Expect(err).NotTo(HaveOccurred()) + rpcClient := client.NewRpcClient(rawRPCClient, testIPC) + ethClient := ethclient.NewClient(rawRPCClient) + blockChainClient := client.NewEthClient(ethClient) + madeNode := node.MakeNode(rpcClient) + transactionConverter := converters.NewTransactionConverter(ethClient) + blockChain := eth.NewBlockChain(blockChainClient, rpcClient, madeNode, transactionConverter) + + db, err := postgres.NewDB(config.Database{ + Hostname: "localhost", + Name: "vulcanize_testing", + Port: 5432, + }, blockChain.Node()) + Expect(err).NotTo(HaveOccurred()) + + return db, blockChain +} diff --git a/pkg/contract_watcher/helpers/test_helpers/database.go b/pkg/contract_watcher/helpers/test_helpers/database.go index 60c9920f9..a82172e9e 100644 --- a/pkg/contract_watcher/helpers/test_helpers/database.go +++ b/pkg/contract_watcher/helpers/test_helpers/database.go @@ -17,19 +17,12 @@ package test_helpers import ( - "github.com/ethereum/go-ethereum/ethclient" - "github.com/ethereum/go-ethereum/rpc" "github.com/makerdao/vulcanizedb/pkg/config" "github.com/makerdao/vulcanizedb/pkg/contract_watcher/constants" "github.com/makerdao/vulcanizedb/pkg/contract_watcher/contract" "github.com/makerdao/vulcanizedb/pkg/contract_watcher/helpers/test_helpers/mocks" "github.com/makerdao/vulcanizedb/pkg/core" "github.com/makerdao/vulcanizedb/pkg/datastore/postgres" - "github.com/makerdao/vulcanizedb/pkg/eth" - "github.com/makerdao/vulcanizedb/pkg/eth/client" - "github.com/makerdao/vulcanizedb/pkg/eth/converters" - "github.com/makerdao/vulcanizedb/pkg/eth/node" - "github.com/makerdao/vulcanizedb/test_config" . "github.com/onsi/gomega" ) @@ -55,28 +48,6 @@ type NewOwnerLog struct { RawLog []byte `db:"raw_log"` } -func SetupDBandBC() (*postgres.DB, core.BlockChain) { - con := test_config.TestClient - testIPC := con.IPCPath - rawRPCClient, err := rpc.Dial(testIPC) - Expect(err).NotTo(HaveOccurred()) - rpcClient := client.NewRpcClient(rawRPCClient, testIPC) - ethClient := ethclient.NewClient(rawRPCClient) - blockChainClient := client.NewEthClient(ethClient) - madeNode := node.MakeNode(rpcClient) - transactionConverter := converters.NewTransactionConverter(ethClient) - blockChain := eth.NewBlockChain(blockChainClient, rpcClient, madeNode, transactionConverter) - - db, err := postgres.NewDB(config.Database{ - Hostname: "localhost", - Name: "vulcanize_testing", - Port: 5432, - }, blockChain.Node()) - Expect(err).NotTo(HaveOccurred()) - - return db, blockChain -} - func SetupTusdRepo(wantedEvents []string) (*postgres.DB, *contract.Contract) { db, err := postgres.NewDB(config.Database{ Hostname: "localhost", @@ -167,14 +138,6 @@ func TearDown(db *postgres.DB) { _, err = tx.Exec(`DELETE FROM public.receipts`) Expect(err).NotTo(HaveOccurred()) - _, err = tx.Exec(`DROP TABLE public.checked_headers`) - Expect(err).NotTo(HaveOccurred()) - - _, err = tx.Exec(`CREATE TABLE public.checked_headers ( - id SERIAL PRIMARY KEY, - header_id INTEGER UNIQUE NOT NULL REFERENCES headers (id) ON DELETE CASCADE);`) - Expect(err).NotTo(HaveOccurred()) - _, err = tx.Exec(`DROP SCHEMA IF EXISTS cw_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e CASCADE`) Expect(err).NotTo(HaveOccurred()) @@ -184,6 +147,25 @@ func TearDown(db *postgres.DB) { err = tx.Commit() Expect(err).NotTo(HaveOccurred()) + err = ResetCheckedHeadersTableSchema(db) + Expect(err).NotTo(HaveOccurred()) + _, err = db.Exec(`VACUUM public.checked_headers`) Expect(err).NotTo(HaveOccurred()) } + +func ResetCheckedHeadersTableSchema(db *postgres.DB) error { + _, dropTableErr := db.Exec(`DROP TABLE public.checked_headers`) + if dropTableErr != nil { + return dropTableErr + } + + _, createTableErr := db.Exec(`CREATE TABLE public.checked_headers ( + id SERIAL PRIMARY KEY, + header_id INTEGER UNIQUE NOT NULL REFERENCES headers (id) ON DELETE CASCADE);`) + if createTableErr != nil { + return createTableErr + } + + return nil +} diff --git a/pkg/contract_watcher/repository/header_repository_test.go b/pkg/contract_watcher/repository/header_repository_test.go index 05d854ce7..02e1d7086 100644 --- a/pkg/contract_watcher/repository/header_repository_test.go +++ b/pkg/contract_watcher/repository/header_repository_test.go @@ -24,15 +24,15 @@ import ( "github.com/makerdao/vulcanizedb/pkg/contract_watcher/repository" "github.com/makerdao/vulcanizedb/pkg/core" "github.com/makerdao/vulcanizedb/pkg/datastore" - "github.com/makerdao/vulcanizedb/pkg/datastore/postgres" "github.com/makerdao/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/makerdao/vulcanizedb/test_config" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) var _ = Describe("Repository", func() { var ( - db *postgres.DB + db = test_config.NewTestDB(test_config.NewTestNode()) contractHeaderRepo repository.HeaderRepository // contract_watcher header repository coreHeaderRepo datastore.HeaderRepository // pkg/datastore header repository eventIDs = []string{ @@ -43,13 +43,14 @@ var _ = Describe("Repository", func() { ) BeforeEach(func() { - db, _ = test_helpers.SetupDBandBC() + test_config.CleanTestDB(db) contractHeaderRepo = repository.NewHeaderRepository(db) coreHeaderRepo = repositories.NewHeaderRepository(db) }) AfterEach(func() { - test_helpers.TearDown(db) + err := test_helpers.ResetCheckedHeadersTableSchema(db) + Expect(err).NotTo(HaveOccurred(), "Failed to reset checked headers table in teardown") }) Describe("AddCheckColumn", func() { diff --git a/test_config/test_config.go b/test_config/test_config.go index 468bf0e18..4a681c10a 100644 --- a/test_config/test_config.go +++ b/test_config/test_config.go @@ -17,7 +17,6 @@ package test_config import ( - "errors" "fmt" "os" @@ -31,7 +30,6 @@ import ( var TestConfig *viper.Viper var DBConfig config.Database -var TestClient config.Client var ABIFilePath string func init() { @@ -47,16 +45,6 @@ func setTestConfig() { if err != nil { logrus.Fatal(err) } - ipc := TestConfig.GetString("client.ipcPath") - - // If we don't have an ipc path in the config file, check the env variable - if ipc == "" { - TestConfig.BindEnv("url", "CLIENT_IPCPATH") - ipc = TestConfig.GetString("url") - } - if ipc == "" { - logrus.Fatal(errors.New("testing.toml IPC path or $CLIENT_IPCPATH env variable need to be set")) - } hn := TestConfig.GetString("database.hostname") port := TestConfig.GetInt("database.port") @@ -67,9 +55,6 @@ func setTestConfig() { Name: name, Port: port, } - TestClient = config.Client{ - IPCPath: ipc, - } } func setABIPath() { From ff6115398702899a02d12da0f623c7a4ddf775f2 Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Wed, 16 Sep 2020 15:00:00 -0500 Subject: [PATCH 52/66] Add a test for CLIENT_IPCPATH on integrationtest --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 4eb9a1598..bac547076 100644 --- a/Makefile +++ b/Makefile @@ -61,6 +61,7 @@ test: | $(GINKGO) $(LINT) .PHONY: integrationtest integrationtest: | $(GINKGO) $(LINT) + test -n "$(CLIENT_IPCPATH)" # $$(CLIENT_IPCPATH) go vet ./... go fmt ./... dropdb --if-exists $(TEST_DB) From 7a7bdbaaa4a7ee3a6adb10c3e9dd5d0b6ab15101 Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Wed, 16 Sep 2020 15:06:37 -0500 Subject: [PATCH 53/66] Extract SetupBC from SetupDBandBC Only the SetupBC code was shared. --- .../contract_watcher_transformer_test.go | 9 ++++++++- integration_test/geth_blockchain_test.go | 15 +-------------- integration_test/integration_test.go | 12 ++---------- 3 files changed, 11 insertions(+), 25 deletions(-) diff --git a/integration_test/contract_watcher_transformer_test.go b/integration_test/contract_watcher_transformer_test.go index fcff75982..76724a2a2 100644 --- a/integration_test/contract_watcher_transformer_test.go +++ b/integration_test/contract_watcher_transformer_test.go @@ -27,6 +27,7 @@ import ( "github.com/makerdao/vulcanizedb/pkg/contract_watcher/transformer" "github.com/makerdao/vulcanizedb/pkg/core" "github.com/makerdao/vulcanizedb/pkg/datastore" + "github.com/makerdao/vulcanizedb/pkg/datastore/postgres" "github.com/makerdao/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/makerdao/vulcanizedb/test_config" . "github.com/onsi/ginkgo" @@ -45,7 +46,13 @@ var _ = Describe("contractWatcher transformer", func() { ) BeforeEach(func() { - db, blockChain = SetupDBandBC() + blockChain = SetupBC() + db, err := postgres.NewDB(config.Database{ + Hostname: "localhost", + Name: "vulcanize_testing", + Port: 5432, + }, blockChain.Node()) + Expect(err).NotTo(HaveOccurred()) headerRepository = repositories.NewHeaderRepository(db) }) diff --git a/integration_test/geth_blockchain_test.go b/integration_test/geth_blockchain_test.go index 528ce9fbc..0c57cd78e 100644 --- a/integration_test/geth_blockchain_test.go +++ b/integration_test/geth_blockchain_test.go @@ -18,13 +18,7 @@ package integration_test import ( "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/ethclient" - "github.com/ethereum/go-ethereum/rpc" "github.com/makerdao/vulcanizedb/pkg/core" - "github.com/makerdao/vulcanizedb/pkg/eth" - "github.com/makerdao/vulcanizedb/pkg/eth/client" - "github.com/makerdao/vulcanizedb/pkg/eth/converters" - "github.com/makerdao/vulcanizedb/pkg/eth/node" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) @@ -33,14 +27,7 @@ var _ = Describe("Reading from the Geth blockchain", func() { var blockChain core.BlockChain BeforeEach(func() { - rawRpcClient, err := rpc.Dial(TestClient.IPCPath) - Expect(err).NotTo(HaveOccurred()) - rpcClient := client.NewRpcClient(rawRpcClient, TestClient.IPCPath) - ethClient := ethclient.NewClient(rawRpcClient) - blockChainClient := client.NewEthClient(ethClient) - node := node.MakeNode(rpcClient) - transactionConverter := converters.NewTransactionConverter(ethClient) - blockChain = eth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter) + blockChain = SetupBC() }) It("retrieves the node info", func(done Done) { diff --git a/integration_test/integration_test.go b/integration_test/integration_test.go index 1cd888e7a..5d1677f94 100644 --- a/integration_test/integration_test.go +++ b/integration_test/integration_test.go @@ -7,7 +7,6 @@ import ( "github.com/ethereum/go-ethereum/rpc" "github.com/makerdao/vulcanizedb/pkg/config" "github.com/makerdao/vulcanizedb/pkg/core" - "github.com/makerdao/vulcanizedb/pkg/datastore/postgres" "github.com/makerdao/vulcanizedb/pkg/eth" "github.com/makerdao/vulcanizedb/pkg/eth/client" "github.com/makerdao/vulcanizedb/pkg/eth/converters" @@ -36,7 +35,7 @@ func init() { } } -func SetupDBandBC() (*postgres.DB, core.BlockChain) { +func SetupBC() core.BlockChain { con := TestClient testIPC := con.IPCPath rawRPCClient, err := rpc.Dial(testIPC) @@ -48,12 +47,5 @@ func SetupDBandBC() (*postgres.DB, core.BlockChain) { transactionConverter := converters.NewTransactionConverter(ethClient) blockChain := eth.NewBlockChain(blockChainClient, rpcClient, madeNode, transactionConverter) - db, err := postgres.NewDB(config.Database{ - Hostname: "localhost", - Name: "vulcanize_testing", - Port: 5432, - }, blockChain.Node()) - Expect(err).NotTo(HaveOccurred()) - - return db, blockChain + return blockChain } From 8d03ec5ec683e54dc7c3ee9b95583f630a570ca9 Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Wed, 16 Sep 2020 15:35:31 -0500 Subject: [PATCH 54/66] Refactoring: Make the tests a little easier to read --- .../checked_headers_repository_test.go | 109 ++++++++++-------- 1 file changed, 59 insertions(+), 50 deletions(-) diff --git a/pkg/datastore/postgres/repositories/checked_headers_repository_test.go b/pkg/datastore/postgres/repositories/checked_headers_repository_test.go index 5de6a1655..3f714f47d 100644 --- a/pkg/datastore/postgres/repositories/checked_headers_repository_test.go +++ b/pkg/datastore/postgres/repositories/checked_headers_repository_test.go @@ -37,16 +37,8 @@ func selectCheckedHeaders(db *postgres.DB, schemaName string, headerID int64) (i return checkedCount, fetchErr } -var _ = Describe("Checked Headers repository", func() { - var ( - db = test_config.NewTestDB(test_config.NewTestNode()) - repo datastore.CheckedHeadersRepository - pluginSchemaName = "plugin" - ) - - BeforeEach(func() { - test_config.CleanTestDB(db) - prepareSchema := ` +func createPluginSchema(db *postgres.DB, schemaName string) error { + prepareSchema := ` CREATE SCHEMA IF NOT EXISTS %[1]s; CREATE TABLE %[1]s.checked_headers ( @@ -56,19 +48,33 @@ CREATE TABLE %[1]s.checked_headers ( UNIQUE (header_id) ); ` - _, schemaError := db.Exec(fmt.Sprintf(prepareSchema, pluginSchemaName)) - if schemaError != nil { - Fail(fmt.Errorf("Could not create schema %s, err %w", pluginSchemaName, schemaError).Error()) - } + _, schemaError := db.Exec(fmt.Sprintf(prepareSchema, schemaName)) + return schemaError +} + +func clearPluginSchema(db *postgres.DB, schemaName string) error { + _, err := db.Exec(fmt.Sprintf("DROP SCHEMA IF EXISTS %s CASCADE;", schemaName)) + return err +} + +var _ = Describe("Checked Headers repository", func() { + var ( + db = test_config.NewTestDB(test_config.NewTestNode()) + repo datastore.CheckedHeadersRepository + pluginSchemaName = "plugin" + ) + + BeforeEach(func() { + test_config.CleanTestDB(db) + schemaErr := createPluginSchema(db, pluginSchemaName) + Expect(schemaErr).NotTo(HaveOccurred()) repo = repositories.NewCheckedHeadersRepository(db, pluginSchemaName) }) AfterEach(func() { - _, err := db.Exec(fmt.Sprintf("DROP SCHEMA IF EXISTS %s CASCADE;", pluginSchemaName)) - if err != nil { - Fail(fmt.Errorf("Could not drop schema %s, err %w", pluginSchemaName, err).Error()) - } + err := clearPluginSchema(db, pluginSchemaName) + Expect(err).NotTo(HaveOccurred()) }) Describe("MarkHeaderChecked", func() { @@ -104,44 +110,47 @@ CREATE TABLE %[1]s.checked_headers ( Describe("MarkSingleHeaderUnchecked", func() { It("marks headers with matching block number as unchecked", func() { - blockNumberOne := rand.Int63() - blockNumberTwo := blockNumberOne + 1 - blockNumberThree := blockNumberOne + 2 - fakeHeaderOne := fakes.GetFakeHeader(blockNumberOne) - fakeHeaderTwo := fakes.GetFakeHeader(blockNumberTwo) - fakeHeaderThree := fakes.GetFakeHeader(blockNumberThree) + blockNumber := rand.Int63() + fakeHeader := fakes.GetFakeHeader(blockNumber) headerRepository := repositories.NewHeaderRepository(db) - // insert three headers with incrementing block number - headerIdOne, insertHeaderOneErr := headerRepository.CreateOrUpdateHeader(fakeHeaderOne) - Expect(insertHeaderOneErr).NotTo(HaveOccurred()) - headerIdTwo, insertHeaderTwoErr := headerRepository.CreateOrUpdateHeader(fakeHeaderTwo) - Expect(insertHeaderTwoErr).NotTo(HaveOccurred()) - headerIdThree, insertHeaderThreeErr := headerRepository.CreateOrUpdateHeader(fakeHeaderThree) - Expect(insertHeaderThreeErr).NotTo(HaveOccurred()) - // mark all headers checked - markHeaderOneCheckedErr := repo.MarkHeaderChecked(headerIdOne) - Expect(markHeaderOneCheckedErr).NotTo(HaveOccurred()) - markHeaderTwoCheckedErr := repo.MarkHeaderChecked(headerIdTwo) - Expect(markHeaderTwoCheckedErr).NotTo(HaveOccurred()) - markHeaderThreeCheckedErr := repo.MarkHeaderChecked(headerIdThree) - Expect(markHeaderThreeCheckedErr).NotTo(HaveOccurred()) - - // mark header from blockNumberTwo unchecked - err := repo.MarkSingleHeaderUnchecked(blockNumberTwo) + headerID, insertHeaderErr := headerRepository.CreateOrUpdateHeader(fakeHeader) + Expect(insertHeaderErr).NotTo(HaveOccurred()) + markHeaderCheckedErr := repo.MarkHeaderChecked(headerID) + Expect(markHeaderCheckedErr).NotTo(HaveOccurred()) + err := repo.MarkSingleHeaderUnchecked(blockNumber) Expect(err).NotTo(HaveOccurred()) - headerOneCheckCount, getHeaderOneErr := selectCheckedHeaders(db, pluginSchemaName, headerIdOne) - Expect(getHeaderOneErr).NotTo(HaveOccurred()) - Expect(headerOneCheckCount).To(Equal(1)) + headerCheckCount, getHeaderErr := selectCheckedHeaders(db, pluginSchemaName, headerID) + Expect(getHeaderErr).NotTo(HaveOccurred()) + Expect(headerCheckCount).To(BeZero()) + }) - headerTwoCheckCount, getHeaderTwoErr := selectCheckedHeaders(db, pluginSchemaName, headerIdTwo) - Expect(getHeaderTwoErr).NotTo(HaveOccurred()) - Expect(headerTwoCheckCount).To(BeZero()) + It("leaves headers without a matching block number alone", func() { + checkedBlockNumber := rand.Int63() + uncheckedBlockNumber := checkedBlockNumber + 1 + checkedHeader := fakes.GetFakeHeader(checkedBlockNumber) + uncheckedHeader := fakes.GetFakeHeader(uncheckedBlockNumber) + headerRepository := repositories.NewHeaderRepository(db) + checkedHeaderID, insertCheckedHeaderErr := headerRepository.CreateOrUpdateHeader(checkedHeader) + Expect(insertCheckedHeaderErr).NotTo(HaveOccurred()) + uncheckedHeaderID, insertUncheckedHeaderErr := headerRepository.CreateOrUpdateHeader(uncheckedHeader) + Expect(insertUncheckedHeaderErr).NotTo(HaveOccurred()) + + // mark both headers as checked + markCheckedHeaderErr := repo.MarkHeaderChecked(checkedHeaderID) + Expect(markCheckedHeaderErr).NotTo(HaveOccurred()) + markUncheckedHeaderErr := repo.MarkHeaderChecked(uncheckedHeaderID) + Expect(markUncheckedHeaderErr).NotTo(HaveOccurred()) + + // re-mark unchecked header as unchecked + err := repo.MarkSingleHeaderUnchecked(uncheckedBlockNumber) + Expect(err).NotTo(HaveOccurred()) - headerThreeCheckCount, getHeaderThreeErr := selectCheckedHeaders(db, pluginSchemaName, headerIdThree) - Expect(getHeaderThreeErr).NotTo(HaveOccurred()) - Expect(headerThreeCheckCount).To(Equal(1)) + // Verify the other block was ignored + checkedHeaderCount, checkedHeaderErr := selectCheckedHeaders(db, pluginSchemaName, checkedHeaderID) + Expect(checkedHeaderErr).NotTo(HaveOccurred()) + Expect(checkedHeaderCount).To(Equal(1)) }) }) From cef2f9165f16653f7a3a9ac13e08588a9054e394 Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Wed, 16 Sep 2020 15:50:06 -0500 Subject: [PATCH 55/66] Fix bugs in two tests Two tests should have been using a recheckCount of 3 - which is why they didn't make sense when I tried to update them for the new query. Updating them to a 3 means th checkCount is set to 2, and the block number must be in the new adjusted window. --- .../repositories/checked_headers_repository_test.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/datastore/postgres/repositories/checked_headers_repository_test.go b/pkg/datastore/postgres/repositories/checked_headers_repository_test.go index 3f714f47d..efd7c9c98 100644 --- a/pkg/datastore/postgres/repositories/checked_headers_repository_test.go +++ b/pkg/datastore/postgres/repositories/checked_headers_repository_test.go @@ -213,7 +213,7 @@ var _ = Describe("Checked Headers repository", func() { }) Describe("when header has already been checked", func() { - It("includes header with block number > 15 back from latest with check count of 1", func() { + It("includes header with block number >= 15 back from latest with check count of 1", func() { err := repo.MarkHeaderChecked(thirdHeaderID) Expect(err).NotTo(HaveOccurred()) @@ -241,8 +241,10 @@ var _ = Describe("Checked Headers repository", func() { It("includes header with block number > 45 back from latest with check count of 2", func() { err := repo.MarkHeaderChecked(secondHeaderID) Expect(err).NotTo(HaveOccurred()) + secondMarkErr := repo.MarkHeaderChecked(secondHeaderID) + Expect(secondMarkErr).NotTo(HaveOccurred()) - headers, err := repo.UncheckedHeaders(firstBlock, lastBlock, recheckCheckCount) + headers, err := repo.UncheckedHeaders(firstBlock, lastBlock, 3) Expect(err).NotTo(HaveOccurred()) headerBlockNumbers := getBlockNumbers(headers) @@ -336,11 +338,13 @@ var _ = Describe("Checked Headers repository", func() { Expect(headerBlockNumbers).NotTo(ContainElement(excludedHeader.BlockNumber)) }) - It("includes header with block number > 45 back from latest with check count of 1", func() { + It("includes header with block number > 45 back from latest with check count of 2", func() { err := repo.MarkHeaderChecked(secondHeaderID) Expect(err).NotTo(HaveOccurred()) + secondCheckErr := repo.MarkHeaderChecked(secondHeaderID) + Expect(secondCheckErr).NotTo(HaveOccurred()) - headers, err := repo.UncheckedHeaders(firstBlock, -1, recheckCheckCount) + headers, err := repo.UncheckedHeaders(firstBlock, -1, 3) Expect(err).NotTo(HaveOccurred()) headerBlockNumbers := getBlockNumbers(headers) From 8e931a107da32628c4bc76004625c965bdb95c71 Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Wed, 16 Sep 2020 16:19:20 -0500 Subject: [PATCH 56/66] Remove the check_count from headers It does not belong there. --- db/migrations/00003_create_headers_table.sql | 3 --- db/schema.sql | 8 -------- 2 files changed, 11 deletions(-) diff --git a/db/migrations/00003_create_headers_table.sql b/db/migrations/00003_create_headers_table.sql index b1e1d6f07..f3dc45144 100644 --- a/db/migrations/00003_create_headers_table.sql +++ b/db/migrations/00003_create_headers_table.sql @@ -6,7 +6,6 @@ CREATE TABLE public.headers block_number BIGINT NOT NULL, raw JSONB, block_timestamp NUMERIC, - check_count INTEGER NOT NULL DEFAULT 0, eth_node_id INTEGER NOT NULL REFERENCES eth_nodes (id) ON DELETE CASCADE, created TIMESTAMP NOT NULL DEFAULT NOW(), updated TIMESTAMP NOT NULL DEFAULT NOW(), @@ -33,8 +32,6 @@ CREATE INDEX headers_block_number ON public.headers (block_number); CREATE INDEX headers_block_timestamp_index ON public.headers (block_timestamp); -CREATE INDEX headers_check_count - ON public.headers (check_count); CREATE INDEX headers_eth_node ON public.headers (eth_node_id); diff --git a/db/schema.sql b/db/schema.sql index 4096d5e43..9ec16edd3 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -388,7 +388,6 @@ CREATE TABLE public.headers ( block_number bigint NOT NULL, raw jsonb, block_timestamp numeric, - check_count integer DEFAULT 0 NOT NULL, eth_node_id integer NOT NULL, created timestamp without time zone DEFAULT now() NOT NULL, updated timestamp without time zone DEFAULT now() NOT NULL @@ -815,13 +814,6 @@ CREATE INDEX headers_block_number ON public.headers USING btree (block_number); CREATE INDEX headers_block_timestamp_index ON public.headers USING btree (block_timestamp); --- --- Name: headers_check_count; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX headers_check_count ON public.headers USING btree (check_count); - - -- -- Name: headers_eth_node; Type: INDEX; Schema: public; Owner: - -- From f9fcc57af143dc04ef9ce090e72c83d0253b029b Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Thu, 17 Sep 2020 15:29:51 -0500 Subject: [PATCH 57/66] Verify the configured schema actually exists This should make SQL injection impossible, since we query for and use that schema name, rather than using the one passed in by the user. This does mean creating the new CheckedHeadersRepository can fail, so I added an error, and I did some refactoring on the checked_headers_repository_test because I learned about Expect / To(Succeed()) --- cmd/backfillEvents.go | 6 +- cmd/execute.go | 7 +- cmd/resetHeaderCheckCount.go | 7 +- .../checked_headers_repository.go | 16 +- .../checked_headers_repository_test.go | 556 +++++++++--------- 5 files changed, 308 insertions(+), 284 deletions(-) diff --git a/cmd/backfillEvents.go b/cmd/backfillEvents.go index ac91a1a09..dae8ccce7 100644 --- a/cmd/backfillEvents.go +++ b/cmd/backfillEvents.go @@ -51,7 +51,11 @@ func backFillEvents() error { blockChain := getBlockChain() db := utils.LoadPostgres(databaseConfig, blockChain.Node()) - extractor := logs.NewLogExtractor(&db, blockChain, repositories.NewCheckedHeadersRepository(&db, genConfig.Schema)) + repo, repoErr := repositories.NewCheckedHeadersRepository(&db, genConfig.Schema) + if repoErr != nil { + return fmt.Errorf("error creating checked headers repository %w", repoErr) + } + extractor := logs.NewLogExtractor(&db, blockChain, repo) for _, initializer := range ethEventInitializers { transformer := initializer(&db) diff --git a/cmd/execute.go b/cmd/execute.go index 92f9dfc82..04c80fffc 100644 --- a/cmd/execute.go +++ b/cmd/execute.go @@ -17,6 +17,7 @@ package cmd import ( + "fmt" "sync" "time" @@ -97,7 +98,11 @@ func executeTransformers() { // Use WaitGroup to wait on both goroutines var wg sync.WaitGroup if len(ethEventInitializers) > 0 { - extractor := logs.NewLogExtractor(&db, blockChain, repositories.NewCheckedHeadersRepository(&db, genConfig.Schema)) + repo, repoErr := repositories.NewCheckedHeadersRepository(&db, genConfig.Schema) + if repoErr != nil { + LogWithCommand.Fatalf(fmt.Errorf("failed to create checked headers repository: %w", repoErr).Error()) + } + extractor := logs.NewLogExtractor(&db, blockChain, repo) delegator := logs.NewLogDelegator(&db) eventHealthCheckMessage := []byte("event watcher starting\n") statusWriter := fs.NewStatusWriter(healthCheckFile, eventHealthCheckMessage) diff --git a/cmd/resetHeaderCheckCount.go b/cmd/resetHeaderCheckCount.go index 43411e2a6..c0438df89 100644 --- a/cmd/resetHeaderCheckCount.go +++ b/cmd/resetHeaderCheckCount.go @@ -64,6 +64,11 @@ func init() { func resetHeaderCount(blockNumber int64) error { blockChain := getBlockChain() db := utils.LoadPostgres(databaseConfig, blockChain.Node()) - repo := repositories.NewCheckedHeadersRepository(&db, genConfig.Schema) + repo, repoErr := repositories.NewCheckedHeadersRepository(&db, genConfig.Schema) + + if repoErr != nil { + return fmt.Errorf("error creating checked headers repository %w", repoErr) + } + return repo.MarkSingleHeaderUnchecked(blockNumber) } diff --git a/pkg/datastore/postgres/repositories/checked_headers_repository.go b/pkg/datastore/postgres/repositories/checked_headers_repository.go index f56e14b58..2f3413e2a 100644 --- a/pkg/datastore/postgres/repositories/checked_headers_repository.go +++ b/pkg/datastore/postgres/repositories/checked_headers_repository.go @@ -33,13 +33,25 @@ ON CONFLICT (header_id) DO WHERE checked_headers.header_id = $1` ) +type SchemaQuery struct { + SchemaName string `db:"schema_name"` +} + type CheckedHeadersRepository struct { db *postgres.DB schemaName string } -func NewCheckedHeadersRepository(db *postgres.DB, schemaName string) CheckedHeadersRepository { - return CheckedHeadersRepository{db: db, schemaName: schemaName} +func NewCheckedHeadersRepository(db *postgres.DB, schemaName string) (*CheckedHeadersRepository, error) { + var sq SchemaQuery + err := db.Get(&sq, "SELECT schema_name FROM information_schema.schemata WHERE schema_name = $1", schemaName) + if err != nil { + return nil, fmt.Errorf("error creating checked headers repository - invalid schema %w", err) + } + + // Use the SchemaName from the actual information_schema, just in case the + // passed in schemaName doesn't exist + return &CheckedHeadersRepository{db: db, schemaName: sq.SchemaName}, nil } // Increment check_count for header diff --git a/pkg/datastore/postgres/repositories/checked_headers_repository_test.go b/pkg/datastore/postgres/repositories/checked_headers_repository_test.go index efd7c9c98..4bfb49f6c 100644 --- a/pkg/datastore/postgres/repositories/checked_headers_repository_test.go +++ b/pkg/datastore/postgres/repositories/checked_headers_repository_test.go @@ -30,33 +30,6 @@ import ( . "github.com/onsi/gomega" ) -func selectCheckedHeaders(db *postgres.DB, schemaName string, headerID int64) (int, error) { - var checkedCount int - queryString := fmt.Sprintf(`SELECT check_count FROM %s.checked_headers WHERE header_id = $1`, schemaName) - fetchErr := db.Get(&checkedCount, queryString, headerID) - return checkedCount, fetchErr -} - -func createPluginSchema(db *postgres.DB, schemaName string) error { - prepareSchema := ` -CREATE SCHEMA IF NOT EXISTS %[1]s; - -CREATE TABLE %[1]s.checked_headers ( - id SERIAL PRIMARY KEY, - check_count INTEGER NOT NULL DEFAULT 0, - header_id INTEGER NOT NULL REFERENCES public.headers(id) ON DELETE CASCADE, - UNIQUE (header_id) -); -` - _, schemaError := db.Exec(fmt.Sprintf(prepareSchema, schemaName)) - return schemaError -} - -func clearPluginSchema(db *postgres.DB, schemaName string) error { - _, err := db.Exec(fmt.Sprintf("DROP SCHEMA IF EXISTS %s CASCADE;", schemaName)) - return err -} - var _ = Describe("Checked Headers repository", func() { var ( db = test_config.NewTestDB(test_config.NewTestNode()) @@ -64,330 +37,319 @@ var _ = Describe("Checked Headers repository", func() { pluginSchemaName = "plugin" ) - BeforeEach(func() { - test_config.CleanTestDB(db) - schemaErr := createPluginSchema(db, pluginSchemaName) - Expect(schemaErr).NotTo(HaveOccurred()) + Describe("without a valid setup", func() { + BeforeEach(func() { + test_config.CleanTestDB(db) + }) - repo = repositories.NewCheckedHeadersRepository(db, pluginSchemaName) + It("errors for a schema that isn't present", func() { + var err error + repo, err = repositories.NewCheckedHeadersRepository(db, pluginSchemaName) + Expect(err.Error()).To(ContainSubstring("invalid schema")) + Expect(repo).To(BeNil()) + }) }) - AfterEach(func() { - err := clearPluginSchema(db, pluginSchemaName) - Expect(err).NotTo(HaveOccurred()) - }) + Describe("with a valid database schema", func() { - Describe("MarkHeaderChecked", func() { - It("marks passed header as checked on insert", func() { - headerRepository := repositories.NewHeaderRepository(db) - headerID, headerErr := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) - Expect(headerErr).NotTo(HaveOccurred()) + BeforeEach(func() { + test_config.CleanTestDB(db) + Expect(createPluginCheckedHeadersTable(db, pluginSchemaName)).To(Succeed()) - err := repo.MarkHeaderChecked(headerID) + var repoErr error + repo, repoErr = repositories.NewCheckedHeadersRepository(db, pluginSchemaName) + Expect(repoErr).NotTo(HaveOccurred()) + }) - Expect(err).NotTo(HaveOccurred()) - checkedCount, fetchErr := selectCheckedHeaders(db, pluginSchemaName, headerID) - Expect(fetchErr).NotTo(HaveOccurred()) - Expect(checkedCount).To(Equal(1)) + AfterEach(func() { + Expect(clearPluginSchema(db, pluginSchemaName)).To(Succeed()) }) - It("increments check count on update", func() { - headerRepository := repositories.NewHeaderRepository(db) - headerID, headerErr := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) - Expect(headerErr).NotTo(HaveOccurred()) + Describe("MarkHeaderChecked", func() { + It("marks passed header as checked on insert", func() { + headerRepository := repositories.NewHeaderRepository(db) + headerID, headerErr := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) + Expect(headerErr).NotTo(HaveOccurred()) - insertErr := repo.MarkHeaderChecked(headerID) - Expect(insertErr).NotTo(HaveOccurred()) + Expect(repo.MarkHeaderChecked(headerID)).To(Succeed()) - updateErr := repo.MarkHeaderChecked(headerID) - Expect(updateErr).NotTo(HaveOccurred()) + Expect(selectCheckedHeaders(db, pluginSchemaName, headerID)).To(Equal(1)) + }) - checkedCount, fetchErr := selectCheckedHeaders(db, pluginSchemaName, headerID) - Expect(fetchErr).NotTo(HaveOccurred()) - Expect(checkedCount).To(Equal(2)) - }) - }) + It("increments check count on update", func() { + headerRepository := repositories.NewHeaderRepository(db) + headerID, headerErr := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) + Expect(headerErr).NotTo(HaveOccurred()) - Describe("MarkSingleHeaderUnchecked", func() { - It("marks headers with matching block number as unchecked", func() { - blockNumber := rand.Int63() - fakeHeader := fakes.GetFakeHeader(blockNumber) - headerRepository := repositories.NewHeaderRepository(db) - headerID, insertHeaderErr := headerRepository.CreateOrUpdateHeader(fakeHeader) - Expect(insertHeaderErr).NotTo(HaveOccurred()) - markHeaderCheckedErr := repo.MarkHeaderChecked(headerID) - Expect(markHeaderCheckedErr).NotTo(HaveOccurred()) - - err := repo.MarkSingleHeaderUnchecked(blockNumber) - Expect(err).NotTo(HaveOccurred()) - - headerCheckCount, getHeaderErr := selectCheckedHeaders(db, pluginSchemaName, headerID) - Expect(getHeaderErr).NotTo(HaveOccurred()) - Expect(headerCheckCount).To(BeZero()) - }) + Expect(repo.MarkHeaderChecked(headerID)).To(Succeed()) + Expect(repo.MarkHeaderChecked(headerID)).To(Succeed()) - It("leaves headers without a matching block number alone", func() { - checkedBlockNumber := rand.Int63() - uncheckedBlockNumber := checkedBlockNumber + 1 - checkedHeader := fakes.GetFakeHeader(checkedBlockNumber) - uncheckedHeader := fakes.GetFakeHeader(uncheckedBlockNumber) - headerRepository := repositories.NewHeaderRepository(db) - checkedHeaderID, insertCheckedHeaderErr := headerRepository.CreateOrUpdateHeader(checkedHeader) - Expect(insertCheckedHeaderErr).NotTo(HaveOccurred()) - uncheckedHeaderID, insertUncheckedHeaderErr := headerRepository.CreateOrUpdateHeader(uncheckedHeader) - Expect(insertUncheckedHeaderErr).NotTo(HaveOccurred()) - - // mark both headers as checked - markCheckedHeaderErr := repo.MarkHeaderChecked(checkedHeaderID) - Expect(markCheckedHeaderErr).NotTo(HaveOccurred()) - markUncheckedHeaderErr := repo.MarkHeaderChecked(uncheckedHeaderID) - Expect(markUncheckedHeaderErr).NotTo(HaveOccurred()) - - // re-mark unchecked header as unchecked - err := repo.MarkSingleHeaderUnchecked(uncheckedBlockNumber) - Expect(err).NotTo(HaveOccurred()) - - // Verify the other block was ignored - checkedHeaderCount, checkedHeaderErr := selectCheckedHeaders(db, pluginSchemaName, checkedHeaderID) - Expect(checkedHeaderErr).NotTo(HaveOccurred()) - Expect(checkedHeaderCount).To(Equal(1)) + Expect(selectCheckedHeaders(db, pluginSchemaName, headerID)).To(Equal(2)) + }) }) - }) - Describe("UncheckedHeaders", func() { - var ( - headerRepository datastore.HeaderRepository - firstBlock, - secondBlock, - thirdBlock, - lastBlock, - secondHeaderID, - thirdHeaderID int64 - blockNumbers []int64 - headerIDs []int64 - err error - uncheckedCheckCount = int64(1) - recheckCheckCount = int64(2) - ) + Describe("MarkSingleHeaderUnchecked", func() { + It("marks headers with matching block number as unchecked", func() { + blockNumber := rand.Int63() + fakeHeader := fakes.GetFakeHeader(blockNumber) + headerRepository := repositories.NewHeaderRepository(db) + headerID, insertHeaderErr := headerRepository.CreateOrUpdateHeader(fakeHeader) + Expect(insertHeaderErr).NotTo(HaveOccurred()) + Expect(repo.MarkHeaderChecked(headerID)).To(Succeed()) - BeforeEach(func() { - headerRepository = repositories.NewHeaderRepository(db) - - lastBlock = rand.Int63() - thirdBlock = lastBlock - 15 - secondBlock = lastBlock - (15 + 30) - firstBlock = lastBlock - (15 + 30 + 45) - - blockNumbers = []int64{firstBlock, secondBlock, thirdBlock, lastBlock} - - headerIDs = []int64{} - for _, n := range blockNumbers { - headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(n)) - headerIDs = append(headerIDs, headerID) - Expect(err).NotTo(HaveOccurred()) - } - secondHeaderID = headerIDs[1] - thirdHeaderID = headerIDs[2] - }) + Expect(repo.MarkSingleHeaderUnchecked(blockNumber)).To(Succeed()) - Describe("when ending block is specified", func() { - It("excludes headers that are out of range", func() { - headers, err := repo.UncheckedHeaders(firstBlock, thirdBlock, uncheckedCheckCount) - Expect(err).NotTo(HaveOccurred()) - - headerBlockNumbers := getBlockNumbers(headers) - Expect(headerBlockNumbers).To(ConsistOf(firstBlock, secondBlock, thirdBlock)) - Expect(headerBlockNumbers).NotTo(ContainElement(lastBlock)) + Expect(selectCheckedHeaders(db, pluginSchemaName, headerID)).To(BeZero()) }) - It("excludes headers that have been checked more than the check count", func() { - err = repo.MarkHeaderChecked(secondHeaderID) - Expect(err).NotTo(HaveOccurred()) - - headers, err := repo.UncheckedHeaders(firstBlock, thirdBlock, uncheckedCheckCount) - Expect(err).NotTo(HaveOccurred()) - - headerBlockNumbers := getBlockNumbers(headers) - Expect(headerBlockNumbers).To(ConsistOf(firstBlock, thirdBlock)) - Expect(headerBlockNumbers).NotTo(ContainElement(secondBlock)) + It("leaves headers without a matching block number alone", func() { + checkedBlockNumber := rand.Int63() + uncheckedBlockNumber := checkedBlockNumber + 1 + checkedHeader := fakes.GetFakeHeader(checkedBlockNumber) + uncheckedHeader := fakes.GetFakeHeader(uncheckedBlockNumber) + headerRepository := repositories.NewHeaderRepository(db) + checkedHeaderID, insertCheckedHeaderErr := headerRepository.CreateOrUpdateHeader(checkedHeader) + Expect(insertCheckedHeaderErr).NotTo(HaveOccurred()) + uncheckedHeaderID, insertUncheckedHeaderErr := headerRepository.CreateOrUpdateHeader(uncheckedHeader) + Expect(insertUncheckedHeaderErr).NotTo(HaveOccurred()) + + // mark both headers as checked + Expect(repo.MarkHeaderChecked(checkedHeaderID)).To(Succeed()) + Expect(repo.MarkHeaderChecked(uncheckedHeaderID)).To(Succeed()) + + // re-mark unchecked header as unchecked + Expect(repo.MarkSingleHeaderUnchecked(uncheckedBlockNumber)).To(Succeed()) + + // Verify the other block was not checked (1 checked header) + Expect(selectCheckedHeaders(db, pluginSchemaName, checkedHeaderID)).To(Equal(1)) }) + }) - Describe("when header has already been checked", func() { - It("includes header with block number >= 15 back from latest with check count of 1", func() { - err := repo.MarkHeaderChecked(thirdHeaderID) + Describe("UncheckedHeaders", func() { + var ( + headerRepository datastore.HeaderRepository + firstBlock, + secondBlock, + thirdBlock, + lastBlock, + secondHeaderID, + thirdHeaderID int64 + blockNumbers []int64 + headerIDs []int64 + err error + uncheckedCheckCount = int64(1) + recheckCheckCount = int64(2) + ) + + BeforeEach(func() { + headerRepository = repositories.NewHeaderRepository(db) + + lastBlock = rand.Int63() + thirdBlock = lastBlock - 15 + secondBlock = lastBlock - (15 + 30) + firstBlock = lastBlock - (15 + 30 + 45) + + blockNumbers = []int64{firstBlock, secondBlock, thirdBlock, lastBlock} + + headerIDs = []int64{} + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(n)) Expect(err).NotTo(HaveOccurred()) + headerIDs = append(headerIDs, headerID) + } + secondHeaderID = headerIDs[1] + thirdHeaderID = headerIDs[2] + }) - headers, err := repo.UncheckedHeaders(firstBlock, lastBlock, recheckCheckCount) + Describe("when ending block is specified", func() { + It("excludes headers that are out of range", func() { + headers, err := repo.UncheckedHeaders(firstBlock, thirdBlock, uncheckedCheckCount) Expect(err).NotTo(HaveOccurred()) headerBlockNumbers := getBlockNumbers(headers) - Expect(headerBlockNumbers).To(ContainElement(thirdBlock)) + Expect(headerBlockNumbers).To(ConsistOf(firstBlock, secondBlock, thirdBlock)) + Expect(headerBlockNumbers).NotTo(ContainElement(lastBlock)) }) - It("excludes header with block number < 15 back from latest with check count of 1", func() { - excludedHeader := fakes.GetFakeHeader(thirdBlock + 1) - excludedHeaderID, createHeaderErr := headerRepository.CreateOrUpdateHeader(excludedHeader) - Expect(createHeaderErr).NotTo(HaveOccurred()) - updateHeaderErr := repo.MarkHeaderChecked(excludedHeaderID) - Expect(updateHeaderErr).NotTo(HaveOccurred()) + It("excludes headers that have been checked more than the check count", func() { + Expect(repo.MarkHeaderChecked(secondHeaderID)).To(Succeed()) - headers, err := repo.UncheckedHeaders(firstBlock, lastBlock, recheckCheckCount) + headers, err := repo.UncheckedHeaders(firstBlock, thirdBlock, uncheckedCheckCount) Expect(err).NotTo(HaveOccurred()) headerBlockNumbers := getBlockNumbers(headers) - Expect(headerBlockNumbers).NotTo(ContainElement(excludedHeader.BlockNumber)) + Expect(headerBlockNumbers).To(ConsistOf(firstBlock, thirdBlock)) + Expect(headerBlockNumbers).NotTo(ContainElement(secondBlock)) }) - It("includes header with block number > 45 back from latest with check count of 2", func() { - err := repo.MarkHeaderChecked(secondHeaderID) - Expect(err).NotTo(HaveOccurred()) - secondMarkErr := repo.MarkHeaderChecked(secondHeaderID) - Expect(secondMarkErr).NotTo(HaveOccurred()) + Describe("when header has already been checked", func() { + It("includes header with block number >= 15 back from latest with check count of 1", func() { + Expect(repo.MarkHeaderChecked(thirdHeaderID)).To(Succeed()) - headers, err := repo.UncheckedHeaders(firstBlock, lastBlock, 3) - Expect(err).NotTo(HaveOccurred()) + headers, err := repo.UncheckedHeaders(firstBlock, lastBlock, recheckCheckCount) + Expect(err).NotTo(HaveOccurred()) - headerBlockNumbers := getBlockNumbers(headers) - Expect(headerBlockNumbers).To(ContainElement(secondBlock)) - }) + headerBlockNumbers := getBlockNumbers(headers) + Expect(headerBlockNumbers).To(ContainElement(thirdBlock)) + }) - It("excludes header with block number < 45 back from latest with check count of 2", func() { - excludedHeader := fakes.GetFakeHeader(secondBlock + 1) - excludedHeaderID, createHeaderErr := headerRepository.CreateOrUpdateHeader(excludedHeader) - Expect(createHeaderErr).NotTo(HaveOccurred()) + It("excludes header with block number < 15 back from latest with check count of 1", func() { + excludedHeader := fakes.GetFakeHeader(thirdBlock + 1) + excludedHeaderID, createHeaderErr := headerRepository.CreateOrUpdateHeader(excludedHeader) + Expect(createHeaderErr).NotTo(HaveOccurred()) + Expect(repo.MarkHeaderChecked(excludedHeaderID)).To(Succeed()) - firstCheckErr := repo.MarkHeaderChecked(excludedHeaderID) - Expect(firstCheckErr).NotTo(HaveOccurred()) - secondCheckErr := repo.MarkHeaderChecked(excludedHeaderID) - Expect(secondCheckErr).NotTo(HaveOccurred()) + headers, err := repo.UncheckedHeaders(firstBlock, lastBlock, recheckCheckCount) + Expect(err).NotTo(HaveOccurred()) - headers, err := repo.UncheckedHeaders(firstBlock, lastBlock, 3) - Expect(err).NotTo(HaveOccurred()) + headerBlockNumbers := getBlockNumbers(headers) + Expect(headerBlockNumbers).NotTo(ContainElement(excludedHeader.BlockNumber)) + }) - headerBlockNumbers := getBlockNumbers(headers) - Expect(headerBlockNumbers).NotTo(ContainElement(excludedHeader.BlockNumber)) - }) - }) + It("includes header with block number > 45 back from latest with check count of 2", func() { + Expect(repo.MarkHeaderChecked(secondHeaderID)).To(Succeed()) + Expect(repo.MarkHeaderChecked(secondHeaderID)).To(Succeed()) - It("only returns headers associated with any node", func() { - dbTwo := test_config.NewTestDB(core.Node{ID: "second"}) - headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo) - repoTwo := repositories.NewCheckedHeadersRepository(dbTwo, pluginSchemaName) - for _, n := range blockNumbers { - _, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n + 10)) - Expect(err).NotTo(HaveOccurred()) - } - allHeaders := []int64{firstBlock, firstBlock + 10, secondBlock, secondBlock + 10, thirdBlock, thirdBlock + 10} + headers, err := repo.UncheckedHeaders(firstBlock, lastBlock, 3) + Expect(err).NotTo(HaveOccurred()) - nodeOneMissingHeaders, err := repo.UncheckedHeaders(firstBlock, thirdBlock+10, uncheckedCheckCount) - Expect(err).NotTo(HaveOccurred()) - nodeOneHeaderBlockNumbers := getBlockNumbers(nodeOneMissingHeaders) - Expect(nodeOneHeaderBlockNumbers).To(ConsistOf(allHeaders)) + headerBlockNumbers := getBlockNumbers(headers) + Expect(headerBlockNumbers).To(ContainElement(secondBlock)) + }) - nodeTwoMissingHeaders, err := repoTwo.UncheckedHeaders(firstBlock, thirdBlock+10, uncheckedCheckCount) - Expect(err).NotTo(HaveOccurred()) - nodeTwoHeaderBlockNumbers := getBlockNumbers(nodeTwoMissingHeaders) - Expect(nodeTwoHeaderBlockNumbers).To(ConsistOf(allHeaders)) - }) - }) + It("excludes header with block number < 45 back from latest with check count of 2", func() { + excludedHeader := fakes.GetFakeHeader(secondBlock + 1) + excludedHeaderID, createHeaderErr := headerRepository.CreateOrUpdateHeader(excludedHeader) + Expect(createHeaderErr).NotTo(HaveOccurred()) - Describe("when ending block is -1", func() { - It("includes all non-checked headers when ending block is -1 ", func() { - headers, err := repo.UncheckedHeaders(firstBlock, -1, uncheckedCheckCount) - Expect(err).NotTo(HaveOccurred()) + Expect(repo.MarkHeaderChecked(excludedHeaderID)).To(Succeed()) + Expect(repo.MarkHeaderChecked(excludedHeaderID)).To(Succeed()) - headerBlockNumbers := getBlockNumbers(headers) - Expect(headerBlockNumbers).To(ConsistOf(firstBlock, secondBlock, thirdBlock, lastBlock)) - }) + headers, err := repo.UncheckedHeaders(firstBlock, lastBlock, 3) + Expect(err).NotTo(HaveOccurred()) - It("excludes headers that have been checked more than the check count", func() { - err := repo.MarkHeaderChecked(headerIDs[1]) - Expect(err).NotTo(HaveOccurred()) + headerBlockNumbers := getBlockNumbers(headers) + Expect(headerBlockNumbers).NotTo(ContainElement(excludedHeader.BlockNumber)) + }) + }) - headers, err := repo.UncheckedHeaders(firstBlock, -1, uncheckedCheckCount) - Expect(err).NotTo(HaveOccurred()) + It("only returns headers associated with any node", func() { + dbTwo := test_config.NewTestDB(core.Node{ID: "second"}) + headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo) + repoTwo, repoErr := repositories.NewCheckedHeadersRepository(dbTwo, pluginSchemaName) + Expect(repoErr).NotTo(HaveOccurred()) - headerBlockNumbers := getBlockNumbers(headers) - Expect(headerBlockNumbers).To(ConsistOf(firstBlock, thirdBlock, lastBlock)) - Expect(headerBlockNumbers).NotTo(ContainElement(secondBlock)) - }) + for _, n := range blockNumbers { + _, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n + 10)) + Expect(err).NotTo(HaveOccurred()) + } + allHeaders := []int64{firstBlock, firstBlock + 10, secondBlock, secondBlock + 10, thirdBlock, thirdBlock + 10} - Describe("when header has already been checked", func() { - It("includes header with block number > 15 back from latest with check count of 1", func() { - err := repo.MarkHeaderChecked(thirdHeaderID) + nodeOneMissingHeaders, err := repo.UncheckedHeaders(firstBlock, thirdBlock+10, uncheckedCheckCount) Expect(err).NotTo(HaveOccurred()) + nodeOneHeaderBlockNumbers := getBlockNumbers(nodeOneMissingHeaders) + Expect(nodeOneHeaderBlockNumbers).To(ConsistOf(allHeaders)) - headers, err := repo.UncheckedHeaders(firstBlock, -1, recheckCheckCount) + nodeTwoMissingHeaders, err := repoTwo.UncheckedHeaders(firstBlock, thirdBlock+10, uncheckedCheckCount) Expect(err).NotTo(HaveOccurred()) - - headerBlockNumbers := getBlockNumbers(headers) - Expect(headerBlockNumbers).To(ContainElement(thirdBlock)) + nodeTwoHeaderBlockNumbers := getBlockNumbers(nodeTwoMissingHeaders) + Expect(nodeTwoHeaderBlockNumbers).To(ConsistOf(allHeaders)) }) + }) - It("excludes header with block number < 15 back from latest with check count of 1", func() { - excludedHeader := fakes.GetFakeHeader(thirdBlock + 1) - excludedHeaderID, createHeaderErr := headerRepository.CreateOrUpdateHeader(excludedHeader) - Expect(createHeaderErr).NotTo(HaveOccurred()) - updateHeaderErr := repo.MarkHeaderChecked(excludedHeaderID) - Expect(updateHeaderErr).NotTo(HaveOccurred()) - - headers, err := repo.UncheckedHeaders(firstBlock, -1, recheckCheckCount) + Describe("when ending block is -1", func() { + It("includes all non-checked headers when ending block is -1 ", func() { + headers, err := repo.UncheckedHeaders(firstBlock, -1, uncheckedCheckCount) Expect(err).NotTo(HaveOccurred()) headerBlockNumbers := getBlockNumbers(headers) - Expect(headerBlockNumbers).NotTo(ContainElement(excludedHeader.BlockNumber)) + Expect(headerBlockNumbers).To(ConsistOf(firstBlock, secondBlock, thirdBlock, lastBlock)) }) - It("includes header with block number > 45 back from latest with check count of 2", func() { - err := repo.MarkHeaderChecked(secondHeaderID) - Expect(err).NotTo(HaveOccurred()) - secondCheckErr := repo.MarkHeaderChecked(secondHeaderID) - Expect(secondCheckErr).NotTo(HaveOccurred()) + It("excludes headers that have been checked more than the check count", func() { + Expect(repo.MarkHeaderChecked(headerIDs[1])).To(Succeed()) - headers, err := repo.UncheckedHeaders(firstBlock, -1, 3) + headers, err := repo.UncheckedHeaders(firstBlock, -1, uncheckedCheckCount) Expect(err).NotTo(HaveOccurred()) headerBlockNumbers := getBlockNumbers(headers) - Expect(headerBlockNumbers).To(ContainElement(secondBlock)) + Expect(headerBlockNumbers).To(ConsistOf(firstBlock, thirdBlock, lastBlock)) + Expect(headerBlockNumbers).NotTo(ContainElement(secondBlock)) }) - It("excludes header with block number < 45 back from latest with check count of 2", func() { - excludedHeader := fakes.GetFakeHeader(secondBlock + 1) - excludedHeaderID, createHeaderErr := headerRepository.CreateOrUpdateHeader(excludedHeader) - Expect(createHeaderErr).NotTo(HaveOccurred()) + Describe("when header has already been checked", func() { + It("includes header with block number > 15 back from latest with check count of 1", func() { + Expect(repo.MarkHeaderChecked(thirdHeaderID)).To(Succeed()) - updateHeaderErr := repo.MarkHeaderChecked(excludedHeaderID) - Expect(updateHeaderErr).NotTo(HaveOccurred()) - updateHeaderErr = repo.MarkHeaderChecked(excludedHeaderID) - Expect(updateHeaderErr).NotTo(HaveOccurred()) + headers, err := repo.UncheckedHeaders(firstBlock, -1, recheckCheckCount) + Expect(err).NotTo(HaveOccurred()) - headers, err := repo.UncheckedHeaders(firstBlock, -1, 3) - Expect(err).NotTo(HaveOccurred()) + headerBlockNumbers := getBlockNumbers(headers) + Expect(headerBlockNumbers).To(ContainElement(thirdBlock)) + }) - headerBlockNumbers := getBlockNumbers(headers) - Expect(headerBlockNumbers).NotTo(ContainElement(excludedHeader.BlockNumber)) + It("excludes header with block number < 15 back from latest with check count of 1", func() { + excludedHeader := fakes.GetFakeHeader(thirdBlock + 1) + excludedHeaderID, createHeaderErr := headerRepository.CreateOrUpdateHeader(excludedHeader) + Expect(createHeaderErr).NotTo(HaveOccurred()) + Expect(repo.MarkHeaderChecked(excludedHeaderID)).To(Succeed()) + + headers, err := repo.UncheckedHeaders(firstBlock, -1, recheckCheckCount) + Expect(err).NotTo(HaveOccurred()) + + headerBlockNumbers := getBlockNumbers(headers) + Expect(headerBlockNumbers).NotTo(ContainElement(excludedHeader.BlockNumber)) + }) + + It("includes header with block number > 45 back from latest with check count of 2", func() { + Expect(repo.MarkHeaderChecked(secondHeaderID)).To(Succeed()) + Expect(repo.MarkHeaderChecked(secondHeaderID)).To(Succeed()) + + headers, err := repo.UncheckedHeaders(firstBlock, -1, 3) + Expect(err).NotTo(HaveOccurred()) + + headerBlockNumbers := getBlockNumbers(headers) + Expect(headerBlockNumbers).To(ContainElement(secondBlock)) + }) + + It("excludes header with block number < 45 back from latest with check count of 2", func() { + excludedHeader := fakes.GetFakeHeader(secondBlock + 1) + excludedHeaderID, createHeaderErr := headerRepository.CreateOrUpdateHeader(excludedHeader) + Expect(createHeaderErr).NotTo(HaveOccurred()) + + Expect(repo.MarkHeaderChecked(excludedHeaderID)).To(Succeed()) + Expect(repo.MarkHeaderChecked(excludedHeaderID)).To(Succeed()) + + headers, err := repo.UncheckedHeaders(firstBlock, -1, 3) + Expect(err).NotTo(HaveOccurred()) + + headerBlockNumbers := getBlockNumbers(headers) + Expect(headerBlockNumbers).NotTo(ContainElement(excludedHeader.BlockNumber)) + }) }) - }) - It("returns headers associated with any node", func() { - dbTwo := test_config.NewTestDB(core.Node{ID: "second"}) - headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo) - repoTwo := repositories.NewCheckedHeadersRepository(dbTwo, pluginSchemaName) - for _, n := range blockNumbers { - _, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n + 10)) - Expect(err).NotTo(HaveOccurred()) - } - allHeaders := []int64{firstBlock, firstBlock + 10, secondBlock, secondBlock + 10, thirdBlock, thirdBlock + 10, lastBlock, lastBlock + 10} + It("returns headers associated with any node", func() { + dbTwo := test_config.NewTestDB(core.Node{ID: "second"}) + headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo) + repoTwo, repoErr := repositories.NewCheckedHeadersRepository(dbTwo, pluginSchemaName) + Expect(repoErr).NotTo(HaveOccurred()) - nodeOneMissingHeaders, err := repo.UncheckedHeaders(firstBlock, -1, uncheckedCheckCount) - Expect(err).NotTo(HaveOccurred()) - nodeOneBlockNumbers := getBlockNumbers(nodeOneMissingHeaders) - Expect(nodeOneBlockNumbers).To(ConsistOf(allHeaders)) + for _, n := range blockNumbers { + _, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n + 10)) + Expect(err).NotTo(HaveOccurred()) + } + allHeaders := []int64{firstBlock, firstBlock + 10, secondBlock, secondBlock + 10, thirdBlock, thirdBlock + 10, lastBlock, lastBlock + 10} + + nodeOneMissingHeaders, err := repo.UncheckedHeaders(firstBlock, -1, uncheckedCheckCount) + Expect(err).NotTo(HaveOccurred()) + nodeOneBlockNumbers := getBlockNumbers(nodeOneMissingHeaders) + Expect(nodeOneBlockNumbers).To(ConsistOf(allHeaders)) - nodeTwoMissingHeaders, err := repoTwo.UncheckedHeaders(firstBlock, -1, uncheckedCheckCount) - Expect(err).NotTo(HaveOccurred()) - nodeTwoBlockNumbers := getBlockNumbers(nodeTwoMissingHeaders) - Expect(nodeTwoBlockNumbers).To(ConsistOf(allHeaders)) + nodeTwoMissingHeaders, err := repoTwo.UncheckedHeaders(firstBlock, -1, uncheckedCheckCount) + Expect(err).NotTo(HaveOccurred()) + nodeTwoBlockNumbers := getBlockNumbers(nodeTwoMissingHeaders) + Expect(nodeTwoBlockNumbers).To(ConsistOf(allHeaders)) + }) }) }) }) @@ -400,3 +362,39 @@ func getBlockNumbers(headers []core.Header) []int64 { } return headerBlockNumbers } + +func selectCheckedHeaders(db *postgres.DB, schemaName string, headerID int64) (int, error) { + var checkedCount int + queryString := fmt.Sprintf(`SELECT check_count FROM %s.checked_headers WHERE header_id = $1`, schemaName) + fetchErr := db.Get(&checkedCount, queryString, headerID) + return checkedCount, fetchErr +} + +func createPluginSchema(db *postgres.DB, schemaName string) error { + prepareSchema := ` +CREATE SCHEMA IF NOT EXISTS %[1]s; +` + + _, schemaError := db.Exec(fmt.Sprintf(prepareSchema, schemaName)) + return schemaError +} + +func createPluginCheckedHeadersTable(db *postgres.DB, schemaName string) error { + prepareSchema := ` +CREATE SCHEMA IF NOT EXISTS %[1]s; + +CREATE TABLE %[1]s.checked_headers ( + id SERIAL PRIMARY KEY, + check_count INTEGER NOT NULL DEFAULT 0, + header_id INTEGER NOT NULL REFERENCES public.headers(id) ON DELETE CASCADE, + UNIQUE (header_id) +); +` + _, schemaError := db.Exec(fmt.Sprintf(prepareSchema, schemaName)) + return schemaError +} + +func clearPluginSchema(db *postgres.DB, schemaName string) error { + _, err := db.Exec(fmt.Sprintf("DROP SCHEMA IF EXISTS %s CASCADE;", schemaName)) + return err +} From 538a401df195772c62b810a22053af38abc59b46 Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Thu, 17 Sep 2020 16:21:39 -0500 Subject: [PATCH 58/66] Provide a helper for testing checked_headers This allows the plugin writer to create a test with only minimal setup validating that they have created the write table in the right schema for their plugin. --- libraries/shared/test_data/test_helpers.go | 28 ++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/libraries/shared/test_data/test_helpers.go b/libraries/shared/test_data/test_helpers.go index 254f70268..b18e549db 100644 --- a/libraries/shared/test_data/test_helpers.go +++ b/libraries/shared/test_data/test_helpers.go @@ -11,6 +11,7 @@ import ( "github.com/makerdao/vulcanizedb/pkg/datastore" "github.com/makerdao/vulcanizedb/pkg/datastore/postgres" "github.com/makerdao/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/makerdao/vulcanizedb/pkg/fakes" . "github.com/onsi/gomega" ) @@ -88,3 +89,30 @@ func getRandomAddress() string { stringHash := hashToPrefixedString(hash) return stringHash[:42] } + +/// DescribeAValidCheckedHeadersModelWithSchema is provided so that +/// plugins can easily validate that they have the necessary checked_headers +/// table in their schema for the execute process. +/// +/// Use like so in your tests: +/// var _ = test_data.DescribeAValidCheckedHeadersModelWithSchema("schemaName") +func ExpectCheckedHeadersInThisSchema(db *postgres.DB, schema string) { + Expect(db).NotTo(BeNil()) + // insert header + blockNumber := rand.Int63() + fakeHeader := fakes.GetFakeHeader(blockNumber) + headerRepo := repositories.NewHeaderRepository(db) + headerID, headerErr := headerRepo.CreateOrUpdateHeader(fakeHeader) + Expect(headerErr).NotTo(HaveOccurred()) + + checkedHeaderRepo, repoErr := repositories.NewCheckedHeadersRepository(db, schema) + Expect(repoErr).NotTo(HaveOccurred()) + uncheckedHeaders, uncheckedHeaderErr := checkedHeaderRepo.UncheckedHeaders(0, -1, 1) + Expect(uncheckedHeaderErr).NotTo(HaveOccurred()) + Expect(len(uncheckedHeaders)).To(Equal(1)) + Expect(uncheckedHeaders[0].BlockNumber).To(Equal(blockNumber)) + + Expect(checkedHeaderRepo.MarkHeaderChecked(headerID)).To(Succeed()) + + Expect(checkedHeaderRepo.UncheckedHeaders(0, -1, 1)).To(BeEmpty()) +} From 17625ffc8208503882b62b1370d71f18e1c895d6 Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Mon, 21 Sep 2020 17:45:54 -0500 Subject: [PATCH 59/66] Remove unused NewTestDB call --- integration_test/contract_watcher_transformer_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/integration_test/contract_watcher_transformer_test.go b/integration_test/contract_watcher_transformer_test.go index 76724a2a2..ce648e623 100644 --- a/integration_test/contract_watcher_transformer_test.go +++ b/integration_test/contract_watcher_transformer_test.go @@ -29,14 +29,13 @@ import ( "github.com/makerdao/vulcanizedb/pkg/datastore" "github.com/makerdao/vulcanizedb/pkg/datastore/postgres" "github.com/makerdao/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/makerdao/vulcanizedb/test_config" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) var _ = Describe("contractWatcher transformer", func() { var ( - db = test_config.NewTestDB(test_config.NewTestNode()) + db *postgres.DB err error blockChain core.BlockChain headerRepository datastore.HeaderRepository From db96a97268749f2ff8c473411471125923d4340a Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Tue, 22 Sep 2020 15:03:03 -0500 Subject: [PATCH 60/66] Don't include any headers before starting value This was ignored on one half of the constructed query. I fixed it by adding a few tests and pulling most of the query into a CTE. --- .../checked_headers_repository.go | 33 ++++++------ .../checked_headers_repository_test.go | 54 +++++++++++++++++++ 2 files changed, 71 insertions(+), 16 deletions(-) diff --git a/pkg/datastore/postgres/repositories/checked_headers_repository.go b/pkg/datastore/postgres/repositories/checked_headers_repository.go index 2f3413e2a..d51604a1a 100644 --- a/pkg/datastore/postgres/repositories/checked_headers_repository.go +++ b/pkg/datastore/postgres/repositories/checked_headers_repository.go @@ -81,25 +81,26 @@ func (repo CheckedHeadersRepository) UncheckedHeaders(startingBlockNumber, endin recheckOffsetMultiplier = 15 ) - joinQuery := fmt.Sprintf(`SELECT h.id, h.block_number, h.hash - FROM public.headers h - LEFT JOIN %s.checked_headers ch - ON ch.header_id = h.id`, repo.schemaName) + joinQuery := fmt.Sprintf(` +WITH checked_headers AS ( + SELECT h.id, h.block_number, h.hash, COALESCE(ch.check_count, 0) AS check_count + FROM public.headers h + LEFT JOIN %s.checked_headers ch + ON ch.header_id = h.id + WHERE h.block_number >= $1 +) +SELECT id, block_number, hash +FROM checked_headers +WHERE ( check_count < 1 + OR (check_count < $2 + AND block_number <= ((SELECT MAX(block_number) FROM public.headers) - ($3 * check_count * (check_count + 1) / 2)))) +`, repo.schemaName) if endingBlockNumber == -1 { - noEndingBlockQuery := fmt.Sprintf(`%s - WHERE ((ch.check_count IS NULL OR ch.check_count < 1) AND h.block_number >= $1) - OR ((ch.check_count IS NULL OR ch.check_count < $2) - AND h.block_number <= ((SELECT MAX(block_number) FROM public.headers) - ($3 * ch.check_count * (ch.check_count + 1) / 2)))`, joinQuery) - err = repo.db.Select(&result, noEndingBlockQuery, startingBlockNumber, checkCount, recheckOffsetMultiplier) + err = repo.db.Select(&result, joinQuery, startingBlockNumber, checkCount, recheckOffsetMultiplier) } else { - endingBlockQuery := fmt.Sprintf(`%s - WHERE ((ch.check_count IS NULL OR ch.check_count < 1) AND h.block_number >= $1 AND h.block_number <= $2) - OR ((ch.check_count IS NULL OR ch.check_count < $3) - AND h.block_number >= $1 - AND h.block_number <= $2 - AND h.block_number <= ((SELECT MAX(block_number) FROM public.headers) - ($4 * (ch.check_count * (ch.check_count + 1) / 2))))`, joinQuery) - err = repo.db.Select(&result, endingBlockQuery, startingBlockNumber, endingBlockNumber, checkCount, recheckOffsetMultiplier) + endingBlockQuery := fmt.Sprintf(`%s AND block_number <= $4`, joinQuery) + err = repo.db.Select(&result, endingBlockQuery, startingBlockNumber, checkCount, recheckOffsetMultiplier, endingBlockNumber) } return result, err diff --git a/pkg/datastore/postgres/repositories/checked_headers_repository_test.go b/pkg/datastore/postgres/repositories/checked_headers_repository_test.go index 4bfb49f6c..d3b748db0 100644 --- a/pkg/datastore/postgres/repositories/checked_headers_repository_test.go +++ b/pkg/datastore/postgres/repositories/checked_headers_repository_test.go @@ -171,6 +171,28 @@ var _ = Describe("Checked Headers repository", func() { Expect(headerBlockNumbers).NotTo(ContainElement(lastBlock)) }) + It("includes headers that have had their check-count reset", func() { + Expect(repo.MarkHeaderChecked(secondHeaderID)).To(Succeed()) + Expect(repo.MarkSingleHeaderUnchecked(secondBlock)).To(Succeed()) + + headers, err := repo.UncheckedHeaders(firstBlock, thirdBlock, uncheckedCheckCount) + Expect(err).NotTo(HaveOccurred()) + + headerBlockNumbers := getBlockNumbers(headers) + Expect(headerBlockNumbers).To(ConsistOf(firstBlock, secondBlock, thirdBlock)) + Expect(headerBlockNumbers).NotTo(ContainElement(lastBlock)) + }) + + It("excludes headers that are before the first block number", func() { + headers, err := repo.UncheckedHeaders(firstBlock+1, thirdBlock, uncheckedCheckCount) + Expect(err).NotTo(HaveOccurred()) + + headerBlockNumbers := getBlockNumbers(headers) + Expect(headerBlockNumbers).To(ConsistOf(secondBlock, thirdBlock)) + Expect(headerBlockNumbers).NotTo(ContainElement(firstBlock)) + Expect(headerBlockNumbers).NotTo(ContainElement(lastBlock)) + }) + It("excludes headers that have been checked more than the check count", func() { Expect(repo.MarkHeaderChecked(secondHeaderID)).To(Succeed()) @@ -183,6 +205,16 @@ var _ = Describe("Checked Headers repository", func() { }) Describe("when header has already been checked", func() { + It("excludes headers that are before the first block number", func() { + Expect(repo.MarkHeaderChecked(thirdHeaderID)).To(Succeed()) + + headers, err := repo.UncheckedHeaders(thirdBlock+1, lastBlock, recheckCheckCount) + Expect(err).NotTo(HaveOccurred()) + + headerBlockNumbers := getBlockNumbers(headers) + Expect(headerBlockNumbers).NotTo(ContainElement(thirdBlock)) + }) + It("includes header with block number >= 15 back from latest with check count of 1", func() { Expect(repo.MarkHeaderChecked(thirdHeaderID)).To(Succeed()) @@ -266,6 +298,18 @@ var _ = Describe("Checked Headers repository", func() { Expect(headerBlockNumbers).To(ConsistOf(firstBlock, secondBlock, thirdBlock, lastBlock)) }) + It("includes headers that have had their check-count reset", func() { + Expect(repo.MarkHeaderChecked(secondHeaderID)).To(Succeed()) + Expect(repo.MarkSingleHeaderUnchecked(secondBlock)).To(Succeed()) + + headers, err := repo.UncheckedHeaders(firstBlock, thirdBlock, uncheckedCheckCount) + Expect(err).NotTo(HaveOccurred()) + + headerBlockNumbers := getBlockNumbers(headers) + Expect(headerBlockNumbers).To(ConsistOf(firstBlock, secondBlock, thirdBlock)) + Expect(headerBlockNumbers).NotTo(ContainElement(lastBlock)) + }) + It("excludes headers that have been checked more than the check count", func() { Expect(repo.MarkHeaderChecked(headerIDs[1])).To(Succeed()) @@ -277,6 +321,16 @@ var _ = Describe("Checked Headers repository", func() { Expect(headerBlockNumbers).NotTo(ContainElement(secondBlock)) }) + It("excludes headers that are before the first block number", func() { + repo.MarkSingleHeaderUnchecked(secondBlock) + headers, err := repo.UncheckedHeaders(firstBlock+1, -1, uncheckedCheckCount) + Expect(err).NotTo(HaveOccurred()) + + headerBlockNumbers := getBlockNumbers(headers) + Expect(headerBlockNumbers).To(ConsistOf(secondBlock, thirdBlock, lastBlock)) + Expect(headerBlockNumbers).NotTo(ContainElement(firstBlock)) + }) + Describe("when header has already been checked", func() { It("includes header with block number > 15 back from latest with check count of 1", func() { Expect(repo.MarkHeaderChecked(thirdHeaderID)).To(Succeed()) From 04568fb312476bbf478100a130d8064b0998ebb6 Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Tue, 22 Sep 2020 15:37:01 -0500 Subject: [PATCH 61/66] Fix the contract_watcher integration test The DB setup was broken, unintentionally. --- integration_test/contract_watcher_transformer_test.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/integration_test/contract_watcher_transformer_test.go b/integration_test/contract_watcher_transformer_test.go index ce648e623..4e40b5b63 100644 --- a/integration_test/contract_watcher_transformer_test.go +++ b/integration_test/contract_watcher_transformer_test.go @@ -27,15 +27,15 @@ import ( "github.com/makerdao/vulcanizedb/pkg/contract_watcher/transformer" "github.com/makerdao/vulcanizedb/pkg/core" "github.com/makerdao/vulcanizedb/pkg/datastore" - "github.com/makerdao/vulcanizedb/pkg/datastore/postgres" "github.com/makerdao/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/makerdao/vulcanizedb/test_config" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) var _ = Describe("contractWatcher transformer", func() { var ( - db *postgres.DB + db = test_config.NewTestDB(test_config.NewTestNode()) err error blockChain core.BlockChain headerRepository datastore.HeaderRepository @@ -46,12 +46,6 @@ var _ = Describe("contractWatcher transformer", func() { BeforeEach(func() { blockChain = SetupBC() - db, err := postgres.NewDB(config.Database{ - Hostname: "localhost", - Name: "vulcanize_testing", - Port: 5432, - }, blockChain.Node()) - Expect(err).NotTo(HaveOccurred()) headerRepository = repositories.NewHeaderRepository(db) }) From e69707cc9f750a86f007cc77bfecb87e17b32824 Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Wed, 23 Sep 2020 10:44:21 -0500 Subject: [PATCH 62/66] Put schema names in the log errors --- cmd/backfillEvents.go | 2 +- cmd/execute.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cmd/backfillEvents.go b/cmd/backfillEvents.go index dae8ccce7..632cc1e83 100644 --- a/cmd/backfillEvents.go +++ b/cmd/backfillEvents.go @@ -53,7 +53,7 @@ func backFillEvents() error { repo, repoErr := repositories.NewCheckedHeadersRepository(&db, genConfig.Schema) if repoErr != nil { - return fmt.Errorf("error creating checked headers repository %w", repoErr) + return fmt.Errorf("error creating checked headers repository %w for schema %s", repoErr, genConfig.Schema) } extractor := logs.NewLogExtractor(&db, blockChain, repo) diff --git a/cmd/execute.go b/cmd/execute.go index 04c80fffc..46fc0eb72 100644 --- a/cmd/execute.go +++ b/cmd/execute.go @@ -17,7 +17,6 @@ package cmd import ( - "fmt" "sync" "time" @@ -100,7 +99,7 @@ func executeTransformers() { if len(ethEventInitializers) > 0 { repo, repoErr := repositories.NewCheckedHeadersRepository(&db, genConfig.Schema) if repoErr != nil { - LogWithCommand.Fatalf(fmt.Errorf("failed to create checked headers repository: %w", repoErr).Error()) + LogWithCommand.Fatalf("failed to create checked headers repository %s for schema %s", repoErr.Error(), genConfig.Schema) } extractor := logs.NewLogExtractor(&db, blockChain, repo) delegator := logs.NewLogDelegator(&db) From 89f31793b1bc08d21b702c71e3437fb72f69430b Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Wed, 23 Sep 2020 10:44:34 -0500 Subject: [PATCH 63/66] Fix comment and doc --- libraries/shared/test_data/test_helpers.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/libraries/shared/test_data/test_helpers.go b/libraries/shared/test_data/test_helpers.go index b18e549db..1abde2548 100644 --- a/libraries/shared/test_data/test_helpers.go +++ b/libraries/shared/test_data/test_helpers.go @@ -90,12 +90,20 @@ func getRandomAddress() string { return stringHash[:42] } -/// DescribeAValidCheckedHeadersModelWithSchema is provided so that +/// ExpectCheckedHeadersInThisSchema is provided so that /// plugins can easily validate that they have the necessary checked_headers /// table in their schema for the execute process. /// /// Use like so in your tests: -/// var _ = test_data.DescribeAValidCheckedHeadersModelWithSchema("schemaName") +/// var _ = Describe("Your Schema", func() { +/// BeforeEach(func() { +/// test_config.CleanTestDB(db) +/// }) +/// +/// It("has a proper checked headers setup in the schema", func() { +/// test_data.ExpectCheckedHeadersInThisSchema(db, "yourschemaname") +/// }) +/// }) func ExpectCheckedHeadersInThisSchema(db *postgres.DB, schema string) { Expect(db).NotTo(BeNil()) // insert header From c784a9f6f26f15ae999aff18d2726294f738af58 Mon Sep 17 00:00:00 2001 From: Rob Mulholand Date: Tue, 29 Sep 2020 09:00:35 -0500 Subject: [PATCH 64/66] Run go mod tidy --- go.mod | 1 - go.sum | 5 ----- 2 files changed, 6 deletions(-) diff --git a/go.mod b/go.mod index aec8278d4..ee7cd0ba2 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/makerdao/vulcanizedb go 1.12 require ( - github.com/ClickHouse/clickhouse-go v1.4.3 // indirect github.com/dave/jennifer v1.3.0 github.com/ethereum/go-ethereum v1.9.21 github.com/hashicorp/golang-lru v0.5.4 diff --git a/go.sum b/go.sum index c78295b51..36dd7794c 100644 --- a/go.sum +++ b/go.sum @@ -13,8 +13,6 @@ github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6L github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/ClickHouse/clickhouse-go v1.4.3 h1:iAFMa2UrQdR5bHJ2/yaSLffZkxpcOYQMCUuKeNXGdqc= -github.com/ClickHouse/clickhouse-go v1.4.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= @@ -29,7 +27,6 @@ github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847/go.mod h1: github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6 h1:Eey/GGQ/E5Xp1P2Lyx1qj007hLZfbi0+CoVeJruGCtI= github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= @@ -39,7 +36,6 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= -github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -180,7 +176,6 @@ github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= From 3db064f055b6aeee6cb44fd59031c3bbdcf84b32 Mon Sep 17 00:00:00 2001 From: Rob Mulholand Date: Tue, 29 Sep 2020 10:57:16 -0500 Subject: [PATCH 65/66] Update go version to 1.15 --- .travis.yml | 2 +- go.mod | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 320dd1f7f..9bf1dc7ad 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ dist: trusty language: go go: -- 1.14 +- 1.15 services: - postgresql - docker diff --git a/go.mod b/go.mod index ee7cd0ba2..e66571eac 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/makerdao/vulcanizedb -go 1.12 +go 1.15 require ( github.com/dave/jennifer v1.3.0 From 7adef1042ead47ce48a157cc61ba7d4ee45867a5 Mon Sep 17 00:00:00 2001 From: Rob Mulholand Date: Tue, 29 Sep 2020 14:41:43 -0500 Subject: [PATCH 66/66] Build plugin with `-mod=mod` - Required for build to succeed when vendor/ is present, as is the case when the plugin is being built --- pkg/plugin/builder/builder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/plugin/builder/builder.go b/pkg/plugin/builder/builder.go index 67157fa4d..0b2030539 100644 --- a/pkg/plugin/builder/builder.go +++ b/pkg/plugin/builder/builder.go @@ -66,7 +66,7 @@ func (b *builder) BuildPlugin() error { } // Build the .go file into a .so plugin - execErr := exec.Command("go", "build", "-buildmode=plugin", "-o", soFile, b.goFile).Run() + execErr := exec.Command("go", "build", "-mod=mod", "-buildmode=plugin", "-o", soFile, b.goFile).Run() if execErr != nil { return errors.New(fmt.Sprintf("unable to build .so file: %s", execErr.Error())) }