Merge pull request #74 from deep-stack/pm-cleanup-migrations

Create distributed hypertables directly, skipping hypertables
cerc-io · May 10, 2022 · 7312b33 · 7312b33
2 parents 045120c + 0ccb54c
commit 7312b33
Show file tree

Hide file tree

Showing 11 changed files with 11 additions and 86 deletions.
diff --git a/README.md b/README.md
@@ -27,7 +27,7 @@ Schemas and utils for IPLD ETH Postgres database
 * Edit [startup_script.sh](./scripts/startup_script.sh) to change the number of migrations to be run:
 
   ```bash
-  ./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up-to 23
+  ./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up-to 21
   ```
 
 * In another `ipld-eth-db` terminal window, build an image `migrations-test` using [Dockerfile](./db/Dockerfile):

diff --git a/...ns/00016_create_postgraphile_triggers.sql → ...ns/00015_create_postgraphile_triggers.sql b/...ns/00016_create_postgraphile_triggers.sql → ...ns/00015_create_postgraphile_triggers.sql
diff --git a/...rations/00017_create_db_version_table.sql → ...rations/00016_create_db_version_table.sql b/...rations/00017_create_db_version_table.sql → ...rations/00016_create_db_version_table.sql
diff --git a/db/migrations/00018_update_db_version.sql → db/migrations/00017_update_db_version.sql b/db/migrations/00018_update_db_version.sql → db/migrations/00017_update_db_version.sql
diff --git a/db/migrations/00020_add_data_nodes.sql → db/migrations/00018_add_data_nodes.sql b/db/migrations/00020_add_data_nodes.sql → db/migrations/00018_add_data_nodes.sql
diff --git a/...21_convert_to_distributed_hypertables.sql → ...19_convert_to_distributed_hypertables.sql b/...21_convert_to_distributed_hypertables.sql → ...19_convert_to_distributed_hypertables.sql
@@ -1,4 +1,8 @@
 -- +goose Up
+-- creating distributed hypertables from duplicate tables for now as we are getting the following error (while running geth
+-- unit tests) if regular tables are directly converted to distributed hypertables
+-- error: "cannot PREPARE a transaction that has executed LISTEN, UNLISTEN, or NOTIFY"
+
 -- create new regular tables
 CREATE TABLE eth.log_cids_i (LIKE eth.log_cids INCLUDING ALL);
 CREATE TABLE eth.access_list_elements_i (LIKE eth.access_list_elements INCLUDING ALL);
@@ -35,7 +39,7 @@ INSERT INTO eth.uncle_cids_i (SELECT * FROM eth.uncle_cids);
 INSERT INTO eth.header_cids_i (SELECT * FROM eth.header_cids);
 INSERT INTO public.blocks_i (SELECT * FROM public.blocks);
 
--- drops hypertables
+-- drop tables
 DROP TABLE eth.log_cids;
 DROP TABLE eth.access_list_elements;
 DROP TABLE eth.state_accounts;
@@ -74,9 +78,9 @@ INSERT INTO public.db_version (singleton, version) VALUES (true, 'v4.0.00-dh')
     ON CONFLICT (singleton) DO UPDATE SET (version, tstamp) = ('v4.0.0-dh', NOW());
 
 -- +goose Down
-INSERT INTO public.db_version (singleton, version) VALUES (true, 'v4.0.0-h')
-    ON CONFLICT (singleton) DO UPDATE SET (version, tstamp) = ('v4.0.0-h', NOW());
--- reversing conversion to hypertable requires migrating all data from every chunk back to a single table
+INSERT INTO public.db_version (singleton, version) VALUES (true, 'v4.0.0')
+    ON CONFLICT (singleton) DO UPDATE SET (version, tstamp) = ('v4.0.0', NOW());
+-- reversing conversion to hypertables requires migrating all data from every chunk back to a single table
 -- create new regular tables
 CREATE TABLE eth.log_cids_i (LIKE eth.log_cids INCLUDING ALL);
 CREATE TABLE eth.access_list_elements_i (LIKE eth.access_list_elements INCLUDING ALL);
@@ -89,18 +93,6 @@ CREATE TABLE eth.uncle_cids_i (LIKE eth.uncle_cids INCLUDING ALL);
 CREATE TABLE eth.header_cids_i (LIKE eth.header_cids INCLUDING ALL);
 CREATE TABLE public.blocks_i (LIKE public.blocks INCLUDING ALL);
 
--- turn them into hypertables
-SELECT create_hypertable('public.blocks_i', 'block_number', migrate_data => true, chunk_time_interval => 32768);
-SELECT create_hypertable('eth.header_cids_i', 'block_number', migrate_data => true, chunk_time_interval => 32768);
-SELECT create_hypertable('eth.uncle_cids_i', 'block_number', migrate_data => true, chunk_time_interval => 32768);
-SELECT create_hypertable('eth.transaction_cids_i', 'block_number', migrate_data => true, chunk_time_interval => 32768);
-SELECT create_hypertable('eth.receipt_cids_i', 'block_number', migrate_data => true, chunk_time_interval => 32768);
-SELECT create_hypertable('eth.state_cids_i', 'block_number', migrate_data => true, chunk_time_interval => 32768);
-SELECT create_hypertable('eth.storage_cids_i', 'block_number', migrate_data => true, chunk_time_interval => 32768);
-SELECT create_hypertable('eth.state_accounts_i', 'block_number', migrate_data => true, chunk_time_interval => 32768);
-SELECT create_hypertable('eth.access_list_elements_i', 'block_number', migrate_data => true, chunk_time_interval => 32768);
-SELECT create_hypertable('eth.log_cids_i', 'block_number', migrate_data => true, chunk_time_interval => 32768);
-
 -- migrate data
 INSERT INTO eth.log_cids_i (SELECT * FROM eth.log_cids);
 INSERT INTO eth.access_list_elements_i (SELECT * FROM eth.access_list_elements);
@@ -113,7 +105,7 @@ INSERT INTO eth.uncle_cids_i (SELECT * FROM eth.uncle_cids);
 INSERT INTO eth.header_cids_i (SELECT * FROM eth.header_cids);
 INSERT INTO public.blocks_i (SELECT * FROM public.blocks);
 
--- drops distributed hypertables
+-- drop distributed hypertables
 DROP TABLE eth.log_cids;
 DROP TABLE eth.access_list_elements;
 DROP TABLE eth.state_accounts;

diff --git a/db/migrations/00019_convert_to_hypertables.sql b/db/migrations/00019_convert_to_hypertables.sql
diff --git a/db/migrations/00022_attach_data_nodes.sql → db/migrations/00020_attach_data_nodes.sql b/db/migrations/00022_attach_data_nodes.sql → db/migrations/00020_attach_data_nodes.sql
diff --git a/...rations/00023_create_stored_functions.sql → ...rations/00021_create_stored_functions.sql b/...rations/00023_create_stored_functions.sql → ...rations/00021_create_stored_functions.sql
diff --git a/...grations/00024_set_replication_factor.sql → ...grations/00022_set_replication_factor.sql b/...grations/00024_set_replication_factor.sql → ...grations/00022_set_replication_factor.sql
diff --git a/scripts/startup_script.sh b/scripts/startup_script.sh
@@ -8,7 +8,7 @@ VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME
 # Run the DB migrations
 echo "Connecting with: $VDB_PG_CONNECT"
 echo "Running database migrations"
-./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up-to 23
+./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up-to 21
 
 # If the db migrations ran without err
 if [[ $? -eq 0 ]]; then