From 057cf91f455d4b14d1d82d84a18ee824c8a31b90 Mon Sep 17 00:00:00 2001 From: jurriaan Date: Thu, 12 Sep 2024 17:45:05 +0200 Subject: [PATCH] fix(hubble): improve reorg handling - use indexer_id --- ...f7fefc27655e8c4f347b5ab79a33f6739b6f31d.json} | 4 ++-- ...d4ed6627f1d79b682ba4c0f5f47732df3bf2b0c.json} | 4 ++-- ...149c29178aa7ba7c4bbd704cb9c4080240be4c4.json} | 4 ++-- hubble/hubble.nix | 12 ++++++------ hubble/src/fetcher/eth/eth.rs | 4 ++-- hubble/src/fetcher/fetcher.rs | 2 +- hubble/src/postgres.rs | 16 ++++++++-------- 7 files changed, 23 insertions(+), 23 deletions(-) rename hubble/.sqlx/{query-81764c1ecadf184dce2b20df496d4cd08545e1c0f021367ef7172294150a1e8d.json => query-28002e29c0654545286e0c1aff7fefc27655e8c4f347b5ab79a33f6739b6f31d.json} (75%) rename hubble/.sqlx/{query-429b9d7c44dcb7ee01e555babe73b6cecf54ca5713262f8ad1a793fa9676423f.json => query-6b41e53bdcfd1235f3cdc46a8d4ed6627f1d79b682ba4c0f5f47732df3bf2b0c.json} (57%) rename hubble/.sqlx/{query-cc0f47f826996e606b6e40f5215c54af415447cd064a047f701255d88fca2934.json => query-c6e7eceed358235902b07e91e149c29178aa7ba7c4bbd704cb9c4080240be4c4.json} (60%) diff --git a/hubble/.sqlx/query-81764c1ecadf184dce2b20df496d4cd08545e1c0f021367ef7172294150a1e8d.json b/hubble/.sqlx/query-28002e29c0654545286e0c1aff7fefc27655e8c4f347b5ab79a33f6739b6f31d.json similarity index 75% rename from hubble/.sqlx/query-81764c1ecadf184dce2b20df496d4cd08545e1c0f021367ef7172294150a1e8d.json rename to hubble/.sqlx/query-28002e29c0654545286e0c1aff7fefc27655e8c4f347b5ab79a33f6739b6f31d.json index fa854ad0a1..694969f7a2 100644 --- a/hubble/.sqlx/query-81764c1ecadf184dce2b20df496d4cd08545e1c0f021367ef7172294150a1e8d.json +++ b/hubble/.sqlx/query-28002e29c0654545286e0c1aff7fefc27655e8c4f347b5ab79a33f6739b6f31d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT start_height, max(end_height) as end_height\n FROM hubble.block_fix\n WHERE start_height = (\n SELECT min(start_height)\n FROM hubble.block_fix\n WHERE label = $1\n )\n GROUP BY start_height\n ", + "query": "\n SELECT start_height, max(end_height) as end_height\n FROM hubble.block_fix\n WHERE start_height = (\n SELECT min(start_height)\n FROM hubble.block_fix\n WHERE indexer_id = $1\n )\n GROUP BY start_height\n ", "describe": { "columns": [ { @@ -24,5 +24,5 @@ null ] }, - "hash": "81764c1ecadf184dce2b20df496d4cd08545e1c0f021367ef7172294150a1e8d" + "hash": "28002e29c0654545286e0c1aff7fefc27655e8c4f347b5ab79a33f6739b6f31d" } diff --git a/hubble/.sqlx/query-429b9d7c44dcb7ee01e555babe73b6cecf54ca5713262f8ad1a793fa9676423f.json b/hubble/.sqlx/query-6b41e53bdcfd1235f3cdc46a8d4ed6627f1d79b682ba4c0f5f47732df3bf2b0c.json similarity index 57% rename from hubble/.sqlx/query-429b9d7c44dcb7ee01e555babe73b6cecf54ca5713262f8ad1a793fa9676423f.json rename to hubble/.sqlx/query-6b41e53bdcfd1235f3cdc46a8d4ed6627f1d79b682ba4c0f5f47732df3bf2b0c.json index 7319552a43..91b951a388 100644 --- a/hubble/.sqlx/query-429b9d7c44dcb7ee01e555babe73b6cecf54ca5713262f8ad1a793fa9676423f.json +++ b/hubble/.sqlx/query-6b41e53bdcfd1235f3cdc46a8d4ed6627f1d79b682ba4c0f5f47732df3bf2b0c.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n DELETE FROM hubble.block_fix\n WHERE label = $1\n AND start_height = $2\n AND end_height <= $2\n ", + "query": "\n DELETE FROM hubble.block_fix\n WHERE indexer_id = $1\n AND start_height = $2\n AND end_height <= $2\n ", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "429b9d7c44dcb7ee01e555babe73b6cecf54ca5713262f8ad1a793fa9676423f" + "hash": "6b41e53bdcfd1235f3cdc46a8d4ed6627f1d79b682ba4c0f5f47732df3bf2b0c" } diff --git a/hubble/.sqlx/query-cc0f47f826996e606b6e40f5215c54af415447cd064a047f701255d88fca2934.json b/hubble/.sqlx/query-c6e7eceed358235902b07e91e149c29178aa7ba7c4bbd704cb9c4080240be4c4.json similarity index 60% rename from hubble/.sqlx/query-cc0f47f826996e606b6e40f5215c54af415447cd064a047f701255d88fca2934.json rename to hubble/.sqlx/query-c6e7eceed358235902b07e91e149c29178aa7ba7c4bbd704cb9c4080240be4c4.json index df7f67cb84..727094e927 100644 --- a/hubble/.sqlx/query-cc0f47f826996e606b6e40f5215c54af415447cd064a047f701255d88fca2934.json +++ b/hubble/.sqlx/query-c6e7eceed358235902b07e91e149c29178aa7ba7c4bbd704cb9c4080240be4c4.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE hubble.block_fix\n SET start_height = $3\n WHERE label = $1\n AND start_height = $2\n ", + "query": "\n UPDATE hubble.block_fix\n SET start_height = $3\n WHERE indexer_id = $1\n AND start_height = $2\n ", "describe": { "columns": [], "parameters": { @@ -12,5 +12,5 @@ }, "nullable": [] }, - "hash": "cc0f47f826996e606b6e40f5215c54af415447cd064a047f701255d88fca2934" + "hash": "c6e7eceed358235902b07e91e149c29178aa7ba7c4bbd704cb9c4080240be4c4" } diff --git a/hubble/hubble.nix b/hubble/hubble.nix index b8ffcb54c6..92814f4b73 100644 --- a/hubble/hubble.nix +++ b/hubble/hubble.nix @@ -55,20 +55,20 @@ indexers = mkOption { type = types.listOf ( types.submodule { - options.indexer_id = mkOption { - type = types.nullOr types.str; + options.indexer_id = mkOption { + type = types.nullOr types.str; description = "Id of the indexer which is used by the internal administration of Hubble. Should never change."; example = "amazing-testnet"; default = null; }; - options.internal_chain_id = mkOption { - type = types.nullOr types.number; + options.internal_chain_id = mkOption { + type = types.nullOr types.number; description = "Hubble internal chain id, used to fetch the current height when migrating to fetchers."; example = "4"; default = null; }; - options.new_chain_override = mkOption { - type = types.nullOr types.bool; + options.new_chain_override = mkOption { + type = types.nullOr types.bool; description = "Indicator that this is a new chain, so the current height must not be used when migrating to fetchers."; example = "false"; default = null; diff --git a/hubble/src/fetcher/eth/eth.rs b/hubble/src/fetcher/eth/eth.rs index ccbfec39c6..1e15399245 100644 --- a/hubble/src/fetcher/eth/eth.rs +++ b/hubble/src/fetcher/eth/eth.rs @@ -259,10 +259,10 @@ impl BlockHandle for EthBlock { let block_to_insert = self.fetch_block().await?; if let Some(block_to_insert) = block_to_insert { - debug!("{}: exists", block_height); + debug!("{}: exists => upsert", block_height); insert_batch_logs(tx, vec![block_to_insert.into()], InsertMode::Upsert).await?; } else { - debug!("{}: empty", block_height); + debug!("{}: empty => delete", block_height); delete_eth_log(tx, self.eth_client.chain_id.db, block_height).await?; } diff --git a/hubble/src/fetcher/fetcher.rs b/hubble/src/fetcher/fetcher.rs index afd44348f7..4e8a020d31 100644 --- a/hubble/src/fetcher/fetcher.rs +++ b/hubble/src/fetcher/fetcher.rs @@ -440,7 +440,7 @@ impl Indexer { if block_range_to_finalize.end - block_range_to_finalize.start < self.chunk_size as u64 { - info!("not much to finalize => sleep a bit"); + info!("not much to finalize => retry later"); sleep(Duration::from_millis(1000)).await; } } else { diff --git a/hubble/src/postgres.rs b/hubble/src/postgres.rs index 6519d1efbc..a2fb6e498c 100644 --- a/hubble/src/postgres.rs +++ b/hubble/src/postgres.rs @@ -689,7 +689,7 @@ pub async fn get_next_block_to_refresh( pub async fn get_block_range_to_fix( tx: &mut sqlx::Transaction<'_, Postgres>, - label: String, + indexer_id: IndexerId, ) -> sqlx::Result> { let record = sqlx::query!( " @@ -698,11 +698,11 @@ pub async fn get_block_range_to_fix( WHERE start_height = ( SELECT min(start_height) FROM hubble.block_fix - WHERE label = $1 + WHERE indexer_id = $1 ) GROUP BY start_height ", - label, + indexer_id, ) .fetch_optional(tx.as_mut()) .await?; @@ -715,7 +715,7 @@ pub async fn get_block_range_to_fix( pub async fn update_block_range_to_fix( tx: &mut sqlx::Transaction<'_, Postgres>, - label: String, + indexer_id: IndexerId, range: BlockRange, ) -> sqlx::Result<()> { // update start of ranges @@ -723,10 +723,10 @@ pub async fn update_block_range_to_fix( " UPDATE hubble.block_fix SET start_height = $3 - WHERE label = $1 + WHERE indexer_id = $1 AND start_height = $2 ", - label, + indexer_id, range.start as i64, range.end as i64, ) @@ -737,11 +737,11 @@ pub async fn update_block_range_to_fix( sqlx::query!( " DELETE FROM hubble.block_fix - WHERE label = $1 + WHERE indexer_id = $1 AND start_height = $2 AND end_height <= $2 ", - label, + indexer_id, range.end as i64, ) .execute(tx.as_mut())