From ecb7065d5831d1d9df88cd518b122493a363b544 Mon Sep 17 00:00:00 2001 From: invis-bitfly <162128378+invis-bitfly@users.noreply.github.com> Date: Mon, 20 Jan 2025 13:57:10 +0100 Subject: [PATCH 1/2] BEDS-140: feat(exporter): clickhouse cl dashboard data exporter --- backend/cmd/exporter/main.go | 95 +- backend/go.mod | 25 +- backend/go.sum | 70 +- backend/pkg/commons/db/clickhouse.go | 167 +- backend/pkg/commons/db/db.go | 42 +- .../20240528095700_init_database.sql | 878 +++--- .../20240702103057_add_materialized_views.sql | 286 ++ .../clickhouse/20240827153322_views.sql | 55 + .../20241019135256_add_rollings.sql | 78 + ...1203144505_fix_flaky_primary_key_index.sql | 104 + .../20241203175153_add_missing_views.sql | 51 + ...5334_add_attestation_reward_sum_column.sql | 337 +++ .../20241204121509_rename_rolling_views.sql | 43 + ...41206114911_add_more_shortcuts_to_view.sql | 645 +++++ .../20241210181608_add_ttl_to_unsafe.sql | 15 + ...4219_add_missing_helper_column_to_view.sql | 46 + ....sql => 20241212132011_status_reports.sql} | 2 +- .../20241212144649_rename_max_ts_views.sql | 33 + .../20241212161354_fix_epoch_max_ts.sql | 11 + ...5758_fix_missing_final_in_rolling_view.sql | 403 +++ ...2_update_deprecated_snowflake_function.sql | 9 + .../20241218131633_fix_all_max_ts.sql | 26 + ...8143509_materialize_shortcuts_in_views.sql | 220 ++ ...23338_temporarily_disable_optimization.sql | 170 ++ ...50102125004_fix_materialized_shortcuts.sql | 70 + ...250102131247_enable_optimization_again.sql | 141 + ...fix_missing_final_in_non_rolling_views.sql | 141 + ...30854_fix_materialized_shortcuts_again.sql | 70 + ...140906_rollings_materialized_shortcuts.sql | 293 ++ ...20250117152735_rollings_no_compression.sql | 773 ++++++ backend/pkg/commons/metrics/metrics.go | 4 + backend/pkg/commons/types/chain.go | 8 +- backend/pkg/commons/types/config.go | 9 + backend/pkg/commons/utils/config.go | 28 +- backend/pkg/commons/utils/eth.go | 4 +- backend/pkg/commons/utils/utils.go | 20 +- backend/pkg/consapi/client_node.go | 33 +- backend/pkg/consapi/network/network.go | 64 +- backend/pkg/consapi/types/slot.go | 17 + backend/pkg/consapi/types/validator.go | 23 + backend/pkg/consapi/utils/marshal.go | 2 +- backend/pkg/exporter/db/db.go | 647 ++++- backend/pkg/exporter/modules/base.go | 19 +- .../pkg/exporter/modules/dashboard_data.go | 2417 +++++------------ .../exporter/modules/dashboard_data_insert.go | 226 ++ .../modules/dashboard_data_maintenance.go | 140 + .../modules/dashboard_data_process.go | 1065 ++++++++ .../modules/dashboard_data_rollings.go | 189 ++ .../modules/dashboard_data_w_1_epoch.go | 276 -- .../modules/dashboard_data_w_2_epoch_hour.go | 253 -- .../modules/dashboard_data_w_2_epoch_total.go | 105 - .../modules/dashboard_data_w_3_epoch_day.go | 510 ---- ...dashboard_data_w_4_day_weekly-month-90d.go | 333 --- .../modules/dashboard_data_w_rolling.go | 794 ------ backend/pkg/exporter/modules/slot_exporter.go | 2 +- backend/pkg/exporter/types/dashboard_data.go | 247 ++ backend/pkg/nodejobs/node_jobs.go | 7 +- 57 files changed, 7949 insertions(+), 4792 deletions(-) create mode 100644 backend/pkg/commons/db/migrations/clickhouse/20240702103057_add_materialized_views.sql create mode 100644 backend/pkg/commons/db/migrations/clickhouse/20240827153322_views.sql create mode 100644 backend/pkg/commons/db/migrations/clickhouse/20241019135256_add_rollings.sql create mode 100644 backend/pkg/commons/db/migrations/clickhouse/20241203144505_fix_flaky_primary_key_index.sql create mode 100644 backend/pkg/commons/db/migrations/clickhouse/20241203175153_add_missing_views.sql create mode 100644 backend/pkg/commons/db/migrations/clickhouse/20241203175334_add_attestation_reward_sum_column.sql create mode 100644 backend/pkg/commons/db/migrations/clickhouse/20241204121509_rename_rolling_views.sql create mode 100644 backend/pkg/commons/db/migrations/clickhouse/20241206114911_add_more_shortcuts_to_view.sql create mode 100644 backend/pkg/commons/db/migrations/clickhouse/20241210181608_add_ttl_to_unsafe.sql create mode 100644 backend/pkg/commons/db/migrations/clickhouse/20241211154219_add_missing_helper_column_to_view.sql rename backend/pkg/commons/db/migrations/clickhouse/{20240821140310_status_reports.sql => 20241212132011_status_reports.sql} (97%) create mode 100644 backend/pkg/commons/db/migrations/clickhouse/20241212144649_rename_max_ts_views.sql create mode 100644 backend/pkg/commons/db/migrations/clickhouse/20241212161354_fix_epoch_max_ts.sql create mode 100644 backend/pkg/commons/db/migrations/clickhouse/20241217145758_fix_missing_final_in_rolling_view.sql create mode 100644 backend/pkg/commons/db/migrations/clickhouse/20241218095302_update_deprecated_snowflake_function.sql create mode 100644 backend/pkg/commons/db/migrations/clickhouse/20241218131633_fix_all_max_ts.sql create mode 100644 backend/pkg/commons/db/migrations/clickhouse/20241218143509_materialize_shortcuts_in_views.sql create mode 100644 backend/pkg/commons/db/migrations/clickhouse/20250102123338_temporarily_disable_optimization.sql create mode 100644 backend/pkg/commons/db/migrations/clickhouse/20250102125004_fix_materialized_shortcuts.sql create mode 100644 backend/pkg/commons/db/migrations/clickhouse/20250102131247_enable_optimization_again.sql create mode 100644 backend/pkg/commons/db/migrations/clickhouse/20250115123428_fix_missing_final_in_non_rolling_views.sql create mode 100644 backend/pkg/commons/db/migrations/clickhouse/20250115130854_fix_materialized_shortcuts_again.sql create mode 100644 backend/pkg/commons/db/migrations/clickhouse/20250117140906_rollings_materialized_shortcuts.sql create mode 100644 backend/pkg/commons/db/migrations/clickhouse/20250117152735_rollings_no_compression.sql create mode 100644 backend/pkg/exporter/modules/dashboard_data_insert.go create mode 100644 backend/pkg/exporter/modules/dashboard_data_maintenance.go create mode 100644 backend/pkg/exporter/modules/dashboard_data_process.go create mode 100644 backend/pkg/exporter/modules/dashboard_data_rollings.go delete mode 100644 backend/pkg/exporter/modules/dashboard_data_w_1_epoch.go delete mode 100644 backend/pkg/exporter/modules/dashboard_data_w_2_epoch_hour.go delete mode 100644 backend/pkg/exporter/modules/dashboard_data_w_2_epoch_total.go delete mode 100644 backend/pkg/exporter/modules/dashboard_data_w_3_epoch_day.go delete mode 100644 backend/pkg/exporter/modules/dashboard_data_w_4_day_weekly-month-90d.go delete mode 100644 backend/pkg/exporter/modules/dashboard_data_w_rolling.go create mode 100644 backend/pkg/exporter/types/dashboard_data.go diff --git a/backend/cmd/exporter/main.go b/backend/cmd/exporter/main.go index 8be9c7fc4..5f3683dee 100644 --- a/backend/cmd/exporter/main.go +++ b/backend/cmd/exporter/main.go @@ -60,6 +60,45 @@ func Run() { defer wg.Done() db.WriterDb, db.ReaderDb = db.MustInitDB(&cfg.WriterDatabase, &cfg.ReaderDatabase, "pgx", "postgres") }() + wg.Add(1) + go func() { + defer wg.Done() + db.AlloyWriter, db.AlloyReader = db.MustInitDB(&cfg.AlloyWriter, &cfg.AlloyReader, "pgx", "postgres") + }() + wg.Add(1) + go func() { + defer wg.Done() + bt, err := db.InitBigtable(utils.Config.Bigtable.Project, utils.Config.Bigtable.Instance, fmt.Sprintf("%d", utils.Config.Chain.ClConfig.DepositChainID), utils.Config.RedisCacheEndpoint) + if err != nil { + log.Fatal(err, "error connecting to bigtable", 0) + } + db.BigtableClient = bt + }() + if utils.Config.TieredCacheProvider != "redis" { + log.Fatal(fmt.Errorf("no cache provider set, please set TierdCacheProvider (example redis)"), "", 0) + } + if utils.Config.TieredCacheProvider == "redis" || len(utils.Config.RedisCacheEndpoint) != 0 { + wg.Add(1) + go func() { + defer wg.Done() + cache.MustInitTieredCache(utils.Config.RedisCacheEndpoint) + log.Infof("tiered Cache initialized, latest finalized epoch: %v", cache.LatestFinalizedEpoch.Get()) + }() + } + wg.Add(1) + go func() { + defer wg.Done() + // Initialize the persistent redis client + rdc := redis.NewClient(&redis.Options{ + Addr: utils.Config.RedisSessionStoreEndpoint, + ReadTimeout: time.Second * 20, + }) + + if err := rdc.Ping(context.Background()).Err(); err != nil { + log.Fatal(err, "error connecting to persistent redis store", 0) + } + db.PersistentRedisDbClient = rdc + }() } else { log.Warnf("------- EXPORTER RUNNING IN V2 ONLY MODE ------") } @@ -67,7 +106,12 @@ func Run() { wg.Add(1) go func() { defer wg.Done() - db.AlloyWriter, db.AlloyReader = db.MustInitDB(&cfg.AlloyWriter, &cfg.AlloyReader, "pgx", "postgres") + db.ClickHouseWriter, db.ClickHouseReader = db.MustInitDB(&cfg.ClickHouse.WriterDatabase, &cfg.ClickHouse.ReaderDatabase, "clickhouse", "clickhouse") + }() + wg.Add(1) + go func() { + defer wg.Done() + db.ClickHouseNativeWriter = db.MustInitClickhouseNative(&cfg.ClickHouse.WriterDatabase) }() wg.Add(1) @@ -101,57 +145,24 @@ func Run() { } }() - wg.Add(1) - go func() { - defer wg.Done() - bt, err := db.InitBigtable(utils.Config.Bigtable.Project, utils.Config.Bigtable.Instance, fmt.Sprintf("%d", utils.Config.Chain.ClConfig.DepositChainID), utils.Config.RedisCacheEndpoint) - if err != nil { - log.Fatal(err, "error connecting to bigtable", 0) - } - db.BigtableClient = bt - }() - - if utils.Config.TieredCacheProvider == "redis" || len(utils.Config.RedisCacheEndpoint) != 0 { - wg.Add(1) - go func() { - defer wg.Done() - cache.MustInitTieredCache(utils.Config.RedisCacheEndpoint) - log.Infof("tiered Cache initialized, latest finalized epoch: %v", cache.LatestFinalizedEpoch.Get()) - }() - } - - wg.Add(1) - go func() { - defer wg.Done() - // Initialize the persistent redis client - rdc := redis.NewClient(&redis.Options{ - Addr: utils.Config.RedisSessionStoreEndpoint, - ReadTimeout: time.Second * 20, - }) - - if err := rdc.Ping(context.Background()).Err(); err != nil { - log.Fatal(err, "error connecting to persistent redis store", 0) - } - db.PersistentRedisDbClient = rdc - }() - wg.Wait() // enable light-weight db connection monitoring monitoring.Init(false) monitoring.Start() - if utils.Config.TieredCacheProvider != "redis" { - log.Fatal(fmt.Errorf("no cache provider set, please set TierdCacheProvider (example redis)"), "", 0) - } - if !cfg.JustV2 { defer db.ReaderDb.Close() defer db.WriterDb.Close() + defer db.AlloyReader.Close() + defer db.AlloyWriter.Close() + defer db.BigtableClient.Close() } - defer db.AlloyReader.Close() - defer db.AlloyWriter.Close() - defer db.BigtableClient.Close() + defer db.ClickHouseReader.Close() + defer db.ClickHouseWriter.Close() + defer db.ClickHouseNativeWriter.Close() + + wg.Wait() context, err := modules.GetModuleContext() if err != nil { diff --git a/backend/go.mod b/backend/go.mod index fb536e301..a2c48a95e 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -6,7 +6,7 @@ require ( cloud.google.com/go/bigtable v1.21.0 cloud.google.com/go/secretmanager v1.11.5 firebase.google.com/go/v4 v4.14.1 - github.com/ClickHouse/clickhouse-go/v2 v2.17.1 + github.com/ClickHouse/clickhouse-go/v2 v2.30.0 github.com/Gurpartap/storekit-go v0.0.0-20201205024111-36b6cd5c6a21 github.com/alexedwards/scs/redisstore v0.0.0-20240316134038-7e11d57e8885 github.com/alexedwards/scs/v2 v2.8.0 @@ -52,7 +52,7 @@ require ( github.com/juliangruber/go-intersect v1.1.0 github.com/jung-kurt/gofpdf v1.16.2 github.com/k3a/html2text v1.2.1 - github.com/klauspost/compress v1.17.6 + github.com/klauspost/compress v1.17.7 github.com/klauspost/pgzip v1.2.6 github.com/lib/pq v1.10.9 github.com/mailgun/mailgun-go/v4 v4.12.0 @@ -65,6 +65,7 @@ require ( github.com/prysmaticlabs/go-ssz v0.0.0-20210121151755-f6208871c388 github.com/rocket-pool/rocketpool-go v1.8.4-0.20241009143357-7b6894d57365 github.com/rocket-pool/smartnode v1.14.1 + github.com/segmentio/encoding v0.4.0 github.com/sethvargo/go-envconfig v1.1.0 github.com/shopspring/decimal v1.4.0 github.com/sirupsen/logrus v1.9.3 @@ -94,14 +95,14 @@ require ( cloud.google.com/go/iam v1.1.7 // indirect cloud.google.com/go/longrunning v0.5.5 // indirect cloud.google.com/go/storage v1.40.0 // indirect - github.com/ClickHouse/ch-go v0.58.2 // indirect + github.com/ClickHouse/ch-go v0.61.5 // indirect github.com/MicahParks/keyfunc v1.9.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 // indirect github.com/ajg/form v1.5.1 // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/alessio/shellescape v1.4.1 // indirect - github.com/andybalholm/brotli v1.0.6 // indirect + github.com/andybalholm/brotli v1.1.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.0 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.0 // indirect @@ -148,7 +149,7 @@ require ( github.com/glendc/go-external-ip v0.1.0 // indirect github.com/go-chi/chi/v5 v5.0.8 // indirect github.com/go-faster/city v1.0.1 // indirect - github.com/go-faster/errors v0.6.1 // indirect + github.com/go-faster/errors v0.7.1 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect @@ -217,9 +218,9 @@ require ( github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect - github.com/paulmach/orb v0.10.0 // indirect + github.com/paulmach/orb v0.11.1 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect - github.com/pierrec/lz4/v4 v4.1.18 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/polydawn/refmt v0.89.0 // indirect github.com/prometheus/client_model v0.6.0 // indirect @@ -259,9 +260,9 @@ require ( go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect - go.opentelemetry.io/otel v1.24.0 // indirect - go.opentelemetry.io/otel/metric v1.24.0 // indirect - go.opentelemetry.io/otel/trace v1.24.0 // indirect + go.opentelemetry.io/otel v1.26.0 // indirect + go.opentelemetry.io/otel/metric v1.26.0 // indirect + go.opentelemetry.io/otel/trace v1.26.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/mod v0.17.0 // indirect @@ -272,8 +273,8 @@ require ( google.golang.org/appengine v1.6.8 // indirect google.golang.org/appengine/v2 v2.0.2 // indirect google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect google.golang.org/grpc v1.62.1 // indirect gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/backend/go.sum b/backend/go.sum index 087b51058..d21b22d4d 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -26,10 +26,10 @@ github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg6 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.1.0 h1:ksErzDEI1khOiGPgpwuI7x2ebx/uXQNw7xJpn9Eq1+I= github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/ClickHouse/ch-go v0.58.2 h1:jSm2szHbT9MCAB1rJ3WuCJqmGLi5UTjlNu+f530UTS0= -github.com/ClickHouse/ch-go v0.58.2/go.mod h1:Ap/0bEmiLa14gYjCiRkYGbXvbe8vwdrfTYWhsuQ99aw= -github.com/ClickHouse/clickhouse-go/v2 v2.17.1 h1:ZCmAYWpu75IyEi7+Yrs/uaAjiCGY5wfW5kXo64exkX4= -github.com/ClickHouse/clickhouse-go/v2 v2.17.1/go.mod h1:rkGTvFDTLqLIm0ma+13xmcCfr/08Gvs7KmFt1tgiWHQ= +github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4= +github.com/ClickHouse/ch-go v0.61.5/go.mod h1:s1LJW/F/LcFs5HJnuogFMta50kKDO0lf9zzfrbl0RQg= +github.com/ClickHouse/clickhouse-go/v2 v2.30.0 h1:AG4D/hW39qa58+JHQIFOSnxyL46H6h2lrmGGk17dhFo= +github.com/ClickHouse/clickhouse-go/v2 v2.30.0/go.mod h1:i9ZQAojcayW3RsdCb3YR+n+wC2h65eJsZCscZ1Z1wyo= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= @@ -58,8 +58,8 @@ github.com/alexedwards/scs/redisstore v0.0.0-20240316134038-7e11d57e8885/go.mod github.com/alexedwards/scs/v2 v2.8.0 h1:h31yUYoycPuL0zt14c0gd+oqxfRwIj6SOjHdKRZxhEw= github.com/alexedwards/scs/v2 v2.8.0/go.mod h1:ToaROZxyKukJKT/xLcVQAChi5k6+Pn1Gvmdl7h3RRj8= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI= -github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9 h1:goHVqTbFX3AIo0tzGr14pgfAW2ZfPChKO21Z9MGf/gk= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= github.com/attestantio/go-eth2-client v0.19.10 h1:NLs9mcBvZpBTZ3du7Ey2NHQoj8d3UePY7pFBXX6C6qs= @@ -211,10 +211,10 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/docker/cli v24.0.7+incompatible h1:wa/nIwYFW7BVTGa7SWPVyyXU9lgORqUb1xfI36MSkFg= github.com/docker/cli v24.0.7+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM= -github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= -github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/docker v27.3.0+incompatible h1:BNb1QY6o4JdKpqwi9IB+HUYcRRrVN4aGFUTvDmWYK1A= +github.com/docker/docker v27.3.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 h1:C7t6eeMaEQVy6e8CarIhscYQlNmw5e3G36y7l7Y21Ao= @@ -283,8 +283,8 @@ github.com/go-faker/faker/v4 v4.3.0 h1:UXOW7kn/Mwd0u6MR30JjUKVzguT20EB/hBOddAAO+ github.com/go-faker/faker/v4 v4.3.0/go.mod h1:F/bBy8GH9NxOxMInug5Gx4WYeG6fHJZ8Ol/dhcpRub4= github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= -github.com/go-faster/errors v0.6.1 h1:nNIPOBkprlKzkThvS/0YaX8Zs9KewLCOSFQS5BU06FI= -github.com/go-faster/errors v0.6.1/go.mod h1:5MGV2/2T9yvlrbhe9pD9LO5Z/2zCSq2T8j+Jpi2LAyY= +github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= +github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -605,8 +605,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= -github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= +github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= @@ -706,6 +706,8 @@ github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8oh github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -771,8 +773,8 @@ github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI= -github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/opencontainers/runc v1.1.10 h1:EaL5WeO9lv9wmS6SASjszOeQdSctvpbu0DdBQBizE40= github.com/opencontainers/runc v1.1.10/go.mod h1:+/R6+KmDlh+hOO8NkjmgkG9Qzvypzk0yXxAPYYR65+M= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= @@ -781,14 +783,14 @@ github.com/ory/dockertest/v3 v3.10.0 h1:4K3z2VMe8Woe++invjaTB7VRyQXQy5UY+loujO4a github.com/ory/dockertest/v3 v3.10.0/go.mod h1:nr57ZbRWMqfsdGdFNLHz5jjNdDb7VVFnzAeW1n5N1Lg= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/paulmach/orb v0.10.0 h1:guVYVqzxHE/CQ1KpfGO077TR0ATHSNjp4s6XGLn3W9s= -github.com/paulmach/orb v0.10.0/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= +github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= +github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/phpdave11/gofpdi v1.0.7/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= -github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= -github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -857,6 +859,8 @@ github.com/sanity-io/litter v1.5.5/go.mod h1:9gzJgR2i4ZpjZHsKvUXIRQVk7P+yM3e+jAF github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= +github.com/segmentio/encoding v0.4.0 h1:MEBYvRqiUB2nfR2criEXWqwdY6HJOUrCn5hboVOVmy8= +github.com/segmentio/encoding v0.4.0/go.mod h1:/d03Cd8PoaDeceuhUUUQWjU0KhWjrmYrWPgtJHYZSnI= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= @@ -976,6 +980,8 @@ github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofm github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= github.com/ydb-platform/ydb-go-genproto v0.0.0-20240126124512-dbb0e1720dbf h1:ckwNHVo4bv2tqNkgx3W3HANh3ta1j6TR5qw08J1A7Tw= @@ -1003,14 +1009,14 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.4 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= -go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= -go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= -go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= -go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= -go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= -go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= -go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= -go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= +go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= +go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30= +go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= +go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= +go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= +go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= +go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -1208,10 +1214,10 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y= google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s= -google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c h1:kaI7oewGK5YnVwj+Y+EJBO/YN1ht8iTL9XkFHtVZLsc= -google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c/go.mod h1:VQW3tUculP/D4B+xVCo+VgSq8As6wA9ZjHl//pmk+6s= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2 h1:9IZDv+/GcI6u+a4jRFRLxQs0RUCfavGfoOgEW6jpkI0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2/go.mod h1:UCOku4NytXMJuLQE5VuqA5lX3PcHCBo8pxNyvkf4xBs= +google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 h1:RFiFrvy37/mpSpdySBDrUdipW/dHwsRwh3J3+A9VgT4= +google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= diff --git a/backend/pkg/commons/db/clickhouse.go b/backend/pkg/commons/db/clickhouse.go index 0088e9842..77bb1fff3 100644 --- a/backend/pkg/commons/db/clickhouse.go +++ b/backend/pkg/commons/db/clickhouse.go @@ -5,13 +5,15 @@ import ( "crypto/tls" "fmt" "net" - "reflect" - "sync" + "runtime" "time" ch "github.com/ClickHouse/clickhouse-go/v2" "github.com/gobitfly/beaconchain/pkg/commons/log" + "github.com/gobitfly/beaconchain/pkg/commons/metrics" "github.com/gobitfly/beaconchain/pkg/commons/types" + "github.com/gobitfly/beaconchain/pkg/commons/version" + "golang.org/x/sync/errgroup" ) var ClickHouseNativeWriter ch.Conn @@ -56,6 +58,18 @@ func MustInitClickhouseNative(writer *types.DatabaseConfig) ch.Conn { Debugf: func(s string, p ...interface{}) { log.Debugf("CH NATIVE WRITER: "+s, p...) }, + Settings: ch.Settings{ + "deduplicate_blocks_in_dependent_materialized_views": "1", + "update_insert_deduplication_token_in_dependent_materialized_views": "1", + }, + ClientInfo: ch.ClientInfo{ + Products: []struct { + Name string + Version string + }{ + {Name: "beaconchain-explorer", Version: version.Version}, + }, + }, }) if err != nil { log.Fatal(err, "Error connecting to clickhouse native writer", 0) @@ -74,24 +88,42 @@ func ClickHouseTestConnection(db ch.Conn, dataBaseName string) { log.Debugf("connected to clickhouse database %s with version %s", dataBaseName, v) } -func DumpToClickhouse(data interface{}, table string) error { +type UltraFastClickhouseStruct interface { + Get(string) any + Extend(UltraFastClickhouseStruct) error +} + +func UltraFastDumpToClickhouse[T UltraFastClickhouseStruct](data T, target_table string, insert_uuid string) error { start := time.Now() - columns, err := ConvertToColumnar(data) + // add metrics + defer func() { + metrics.TaskDuration.WithLabelValues(fmt.Sprintf("clickhouse_dump_%s_overall", target_table)).Observe(time.Since(start).Seconds()) + }() + now := time.Now() + // get column order & names from clickhouse + var columns []string + err := ClickHouseReader.Select(&columns, "SELECT name FROM system.columns where table=$1 and database=currentDatabase() order by position;", target_table) if err != nil { return err } - log.Debugf("converted to columnar in %s", time.Since(start)) - start = time.Now() - // abort after 3 minutes - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + metrics.TaskDuration.WithLabelValues(fmt.Sprintf("clickhouse_dump_%s_get_columns", target_table)).Observe(time.Since(now).Seconds()) + now = time.Now() + // prepare batch + abortCtx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) defer cancel() - - batch, err := ClickHouseNativeWriter.PrepareBatch(ctx, `INSERT INTO `+table) + ctx := ch.Context(abortCtx, ch.WithSettings(ch.Settings{ + "insert_deduplication_token": insert_uuid, // 重复数据插入时，会根据这个字段进行去重 + "insert_deduplicate": true, + }), ch.WithLogs(func(l *ch.Log) { + log.Debugf("CH NATIVE WRITER: %s", l.Text) + }), + ) + batch, err := ClickHouseNativeWriter.PrepareBatch(ctx, `INSERT INTO `+target_table) if err != nil { return err } - log.Debugf("prepared batch in %s", time.Since(start)) - start = time.Now() + metrics.TaskDuration.WithLabelValues(fmt.Sprintf("clickhouse_dump_%s_prepare_batch", target_table)).Observe(time.Since(now).Seconds()) + now = time.Now() defer func() { if batch.IsSent() { return @@ -101,97 +133,38 @@ func DumpToClickhouse(data interface{}, table string) error { log.Warnf("failed to abort batch: %v", err) } }() - for c := 0; c < len(columns); c++ { - // type assert to correct type - log.Debugf("appending column %d", c) - switch columns[c].(type) { - case []int64: - err = batch.Column(c).Append(columns[c].([]int64)) - case []uint64: - err = batch.Column(c).Append(columns[c].([]uint64)) - case []time.Time: - // appending unix timestamps as int64 to a DateTime column is actually faster than appending time.Time directly - // tho with how many columns we have it doesn't really matter - err = batch.Column(c).Append(columns[c].([]time.Time)) - case []float64: - err = batch.Column(c).Append(columns[c].([]float64)) - case []bool: - err = batch.Column(c).Append(columns[c].([]bool)) - default: - // warning: slow path. works but try to avoid this - cType := reflect.TypeOf(columns[c]) - log.Warnf("fallback: column %d of type %s is not natively supported, falling back to reflection", c, cType) - startSlow := time.Now() - cValue := reflect.ValueOf(columns[c]) - length := cValue.Len() - cSlice := reflect.MakeSlice(reflect.SliceOf(cType.Elem()), length, length) - for i := 0; i < length; i++ { - cSlice.Index(i).Set(cValue.Index(i)) - } - err = batch.Column(c).Append(cSlice.Interface()) - log.Debugf("fallback: appended column %d in %s", c, time.Since(startSlow)) + var g errgroup.Group + g.SetLimit(runtime.NumCPU()) + // iterate columns retrieved from clickhouse + for i, n := range columns { + // Capture the loop variable + col_index := i + col_name := n + if col_name == "_inserted_at" { + continue } - if err != nil { + // Start a new goroutine for each column + g.Go(func() error { + // get it from the struct + column := data.Get(col_name) + if column == nil { + return fmt.Errorf("column %s not found in struct", col_name) + } + // Perform the type assertion and append operation + err = batch.Column(col_index).Append(column) + log.Debugf("appended column %s in %s", col_name, time.Since(now)) return err - } + }) } - log.Debugf("appended all columns to batch in %s", time.Since(start)) - start = time.Now() + if err := g.Wait(); err != nil { + return err + } + metrics.TaskDuration.WithLabelValues(fmt.Sprintf("clickhouse_dump_%s_append_columns", target_table)).Observe(time.Since(now).Seconds()) + now = time.Now() err = batch.Send() if err != nil { return err } - log.Debugf("sent batch in %s", time.Since(start)) + metrics.TaskDuration.WithLabelValues(fmt.Sprintf("clickhouse_dump_%s_send_batch", target_table)).Observe(time.Since(now).Seconds()) return nil } - -// ConvertToColumnar efficiently converts a slice of any struct type to a slice of slices, each representing a column. -func ConvertToColumnar(data interface{}) ([]interface{}, error) { - start := time.Now() - v := reflect.ValueOf(data) - if v.Kind() != reflect.Slice { - return nil, fmt.Errorf("provided data is not a slice") - } - - if v.Len() == 0 { - return nil, fmt.Errorf("slice is empty") - } - - elemType := v.Type().Elem() - if elemType.Kind() != reflect.Struct { - return nil, fmt.Errorf("slice elements are not structs") - } - - numFields := elemType.NumField() - columns := make([]interface{}, numFields) - colValues := make([]reflect.Value, numFields) - - for i := 0; i < numFields; i++ { - fieldType := elemType.Field(i).Type - colSlice := reflect.MakeSlice(reflect.SliceOf(fieldType), v.Len(), v.Len()) - x := reflect.New(colSlice.Type()) - x.Elem().Set(colSlice) - columns[i] = colSlice - colValues[i] = colSlice.Slice(0, v.Len()) - } - - var wg sync.WaitGroup - wg.Add(numFields) - - for j := 0; j < numFields; j++ { - go func(j int) { - defer wg.Done() - for i := 0; i < v.Len(); i++ { - structValue := v.Index(i) - colValues[j].Index(i).Set(structValue.Field(j)) - } - }(j) - } - wg.Wait() - - for i, col := range colValues { - columns[i] = col.Interface() - } - log.Infof("columnarized %d rows with %d columns in %s", v.Len(), numFields, time.Since(start)) - return columns, nil -} diff --git a/backend/pkg/commons/db/db.go b/backend/pkg/commons/db/db.go index 8a16a64dc..09c0add64 100644 --- a/backend/pkg/commons/db/db.go +++ b/backend/pkg/commons/db/db.go @@ -11,6 +11,7 @@ import ( "sort" "strconv" "strings" + "sync" "time" "github.com/ethereum/go-ethereum/common" @@ -63,21 +64,29 @@ const DefaultInfScrollRows = 25 var ErrNoStats = errors.New("no stats available") -func dbTestConnection(dbConn *sqlx.DB, dataBaseName string) { +func dbTestConnection(dbConn *sqlx.DB, databaseBrand string, databaseName string, connectionType string) { // The golang sql driver does not properly implement PingContext // therefore we use a timer to catch db connection timeouts dbConnectionTimeout := time.NewTimer(15 * time.Second) go func() { <-dbConnectionTimeout.C - log.Fatal(fmt.Errorf("timeout while connecting to %s", dataBaseName), "", 0) + log.Fatal(fmt.Errorf("timeout while connecting to %s %s database %s", connectionType, databaseBrand, databaseName), "", 0) }() err := dbConn.Ping() if err != nil { - log.Fatal(fmt.Errorf("unable to ping %s. error: %w", dataBaseName, err), "", 0) + log.Fatal(fmt.Errorf("unable to ping %s %s database %s. error: %w", connectionType, databaseBrand, databaseName, err), "", 0) } + // get the migration version of the database using the goose + // ideally this runs regularly but idk would have to throw into the monitoring process prob? makes the most sense there, tho that isnt exactly connected to prometheus + ver, err := getGooseVersion(dbConn, databaseBrand) + if err != nil { + log.Fatal(fmt.Errorf("unable to get migration version of %s %s database %s. error: %w", connectionType, databaseBrand, databaseName, err), "", 0) + } + metrics.DatabaseVersion.WithLabelValues(databaseBrand, databaseName, fmt.Sprint(ver)).Set(1) + dbConnectionTimeout.Stop() } @@ -130,8 +139,7 @@ func MustInitDB(writer *types.DatabaseConfig, reader *types.DatabaseConfig, driv if err != nil { log.Fatal(err, "error getting Connection Writer database", 0) } - - dbTestConnection(dbConnWriter, fmt.Sprintf("database %v:%v/%v", writer.Host, writer.Port, writer.Name)) + dbTestConnection(dbConnWriter, databaseBrand, writer.Name, "writer") dbConnWriter.SetConnMaxIdleTime(time.Second * 30) dbConnWriter.SetConnMaxLifetime(time.Minute) dbConnWriter.SetMaxOpenConns(writer.MaxOpenConns) @@ -169,7 +177,7 @@ func MustInitDB(writer *types.DatabaseConfig, reader *types.DatabaseConfig, driv log.Fatal(err, "error getting Connection Reader database", 0) } - dbTestConnection(dbConnReader, fmt.Sprintf("database %v:%v/%v", writer.Host, writer.Port, writer.Name)) + dbTestConnection(dbConnReader, databaseBrand, reader.Name, "reader") dbConnReader.SetConnMaxIdleTime(time.Second * 30) dbConnReader.SetConnMaxLifetime(time.Minute) dbConnReader.SetMaxOpenConns(reader.MaxOpenConns) @@ -177,6 +185,28 @@ func MustInitDB(writer *types.DatabaseConfig, reader *types.DatabaseConfig, driv return dbConnWriter, dbConnReader } +// concurrent safe get goose version of a db using a sqlx.DB and a database brand +var GooseVersionMutex = sync.Mutex{} + +func getGooseVersion(db *sqlx.DB, databaseBrand string) (int64, error) { + GooseVersionMutex.Lock() + defer GooseVersionMutex.Unlock() + if databaseBrand == "clickhouse" { + if err := goose.SetDialect("clickhouse"); err != nil { + return 0, err + } + } else { + if err := goose.SetDialect("postgres"); err != nil { + return 0, err + } + } + ver, err := goose.GetDBVersion(db.DB) + if err != nil { + return 0, fmt.Errorf("unable to get migration version of %s database. error: %w", databaseBrand, err) + } + return ver, nil +} + func ApplyEmbeddedDbSchema(version int64, database string) error { var targetDB *sqlx.DB var migrationPath string diff --git a/backend/pkg/commons/db/migrations/clickhouse/20240528095700_init_database.sql b/backend/pkg/commons/db/migrations/clickhouse/20240528095700_init_database.sql index 4f2e040a7..8ceaabd79 100644 --- a/backend/pkg/commons/db/migrations/clickhouse/20240528095700_init_database.sql +++ b/backend/pkg/commons/db/migrations/clickhouse/20240528095700_init_database.sql @@ -1,422 +1,528 @@ -- +goose Up -- +goose StatementBegin -CREATE TABLE validator_dashboard_data_daily +CREATE TABLE IF NOT EXISTS _exporter_metadata ( - `validator_index` UInt64, - `day` Date, - `epoch_start`Int64, - `epoch_end`Int64, - `attestations_source_reward` Nullable(Int64), - `attestations_target_reward` Nullable(Int64), - `attestations_head_reward` Nullable(Int64), - `attestations_inactivity_reward` Nullable(Int64), - `attestations_inclusion_reward` Nullable(Int64), - `attestations_reward` Nullable(Int64), - `attestations_ideal_source_reward` Nullable(Int64), - `attestations_ideal_target_reward` Nullable(Int64), - `attestations_ideal_head_reward` Nullable(Int64), - `attestations_ideal_inactivity_reward` Nullable(Int64), - `attestations_ideal_inclusion_reward` Nullable(Int64), - `attestations_ideal_reward` Nullable(Int64), - `blocks_scheduled` Nullable(Int64), - `blocks_proposed` Nullable(Int64), - `blocks_cl_reward` Nullable(Int64), - `blocks_el_reward` Nullable(Int64), - `sync_scheduled` Nullable(Int64), - `sync_executed` Nullable(Int64), - `sync_rewards` Nullable(Int64), - `slashed` Nullable(Bool), - `balance_start` Nullable(Int64), - `balance_end` Nullable(Int64), - `deposits_count` Nullable(Int64), - `deposits_amount` Nullable(Int64), - `withdrawals_count` Nullable(Int64), - `withdrawals_amount` Nullable(Int64), - `inclusion_delay_sum` Nullable(Int64), - `blocks_expected` Nullable(Float64), - `sync_committees_expected` Nullable(Float64), - `attestations_scheduled` Nullable(Int64), - `attestations_executed` Nullable(Int64), - `attestation_head_executed` Nullable(Int64), - `attestation_source_executed` Nullable(Int64), - `attestation_target_executed` Nullable(Int64), - `optimal_inclusion_delay_sum` Nullable(Int64), - `slashed_by` Nullable(Int64), --- should only ever be set once so any should be fine - `slashed_violation` Nullable(Int64), --- should only ever be set once so any should be fine - `slasher_reward` Nullable(Int64), - `last_executed_duty_epoch` Nullable(Int64), - `blocks_cl_attestations_reward` Nullable(Int64), - `blocks_cl_sync_aggregate_reward` Nullable(Int64), - -- add projection to optimize validator_index queries - Projection validator_index_day_projection - ( - select * order by validator_index, day - ) + `epoch` UInt64 COMMENT 'epoch number', + `insert_batch_id` Nullable(UUID) COMMENT 'id of the batch the epoch is part of during the insert process', + `successful_insert` Nullable(DateTime) COMMENT 'if the batch was successfully inserted. This is set after the exporter has received confirmation from the clickhouse server', + `transfer_batch_id` Nullable(UUID) COMMENT 'id of the batch the epoch is part of during the transfer to the final table', + `successful_transfer` Nullable(DateTime) COMMENT 'if the batch was successfully transferred to the final table. This is set after the exporter has received confirmation from the clickhouse server' ) -ENGINE = MergeTree() -PRIMARY KEY (day, validator_index) -ORDER BY (day, validator_index) -SETTINGS index_granularity = 8192; +ENGINE = ReplacingMergeTree +ORDER BY epoch +SETTINGS index_granularity = 8192, non_replicated_deduplication_window = 2048, replicated_deduplication_window = 2048 +-- +goose StatementEnd +-- +goose StatementBegin +CREATE TABLE IF NOT EXISTS _exporter_tasks ( + `hostname` LowCardinality(String), + `uuid` UUID DEFAULT generateUUIDv4(), + `priority` Int64, + `start_ts` DateTime, + `end_ts` DateTime, + `status` Enum('pending', 'running', 'completed'), +) +ENGINE = ReplacingMergeTree +Order By (hostname, priority, start_ts) -- order by start_ts DESC to get the latest task irs +SETTINGS index_granularity = 8192, non_replicated_deduplication_window = 2048, replicated_deduplication_window = 2048; +-- +goose StatementEnd +-- +goose StatementBegin +SET flatten_nested = 1; -- +goose StatementEnd -- +goose StatementBegin -CREATE TABLE validator_dashboard_data_weekly +--- create sink table that will only be used as a sink for the insert process - materialized views will be to redirect the data to the correct tables +-- uses Null Engine (https://clickhouse.tech/docs/en/engines/table-engines/special/null/) +CREATE TABLE IF NOT EXISTS _insert_sink_validator_dashboard_data_epoch ( - `validator_index` UInt64, - `week` Date, - `epoch_start` SimpleAggregateFunction(min, Int64), - `epoch_end` SimpleAggregateFunction(max, Int64), - `attestations_source_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_target_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_head_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_inactivity_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_inclusion_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_ideal_source_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_ideal_target_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_ideal_head_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_ideal_inactivity_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_ideal_inclusion_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_ideal_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `blocks_scheduled` SimpleAggregateFunction(sum, Nullable(Int64)), - `blocks_proposed` SimpleAggregateFunction(sum, Nullable(Int64)), - `blocks_cl_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `blocks_el_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `sync_scheduled` SimpleAggregateFunction(sum, Nullable(Int64)), - `sync_executed` SimpleAggregateFunction(sum, Nullable(Int64)), - `sync_rewards` SimpleAggregateFunction(sum, Nullable(Int64)), - `slashed` SimpleAggregateFunction(max, Nullable(Bool)), - `balance_start` AggregateFunction(argMin, Nullable(Int64), Int64), - `balance_end` AggregateFunction(argMax, Nullable(Int64), Int64), - `deposits_count` SimpleAggregateFunction(sum, Nullable(Int64)), - `deposits_amount` SimpleAggregateFunction(sum, Nullable(Int64)), - `withdrawals_count` SimpleAggregateFunction(sum, Nullable(Int64)), - `withdrawals_amount` SimpleAggregateFunction(sum, Nullable(Int64)), - `inclusion_delay_sum` SimpleAggregateFunction(sum, Nullable(Int64)), - `blocks_expected` SimpleAggregateFunction(sum, Nullable(Float64)), - `sync_committees_expected` SimpleAggregateFunction(sum, Nullable(Float64)), - `attestations_scheduled` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_executed` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestation_head_executed` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestation_source_executed` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestation_target_executed` SimpleAggregateFunction(sum, Nullable(Int64)), - `optimal_inclusion_delay_sum` SimpleAggregateFunction(sum, Nullable(Int64)), - `slasher_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `slashed_by` SimpleAggregateFunction(any, Nullable(Int64)), --- should only ever be set once so any should be fine - `slashed_violation` SimpleAggregateFunction(any, Nullable(Int64)), --- should only ever be set once so any should be fine - `last_executed_duty_epoch` SimpleAggregateFunction(max, Nullable(Int64)), - `blocks_cl_attestations_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `blocks_cl_sync_aggregate_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - -- add projection to optimize validator_index queries - Projection validator_index_week_projection - ( - select * order by validator_index, week - ) + `validator_index` UInt64 COMMENT 'validator index', + `epoch` Int64 COMMENT 'epoch number', + `epoch_timestamp` DateTime COMMENT 'timestamp of the first slot of the epoch', + `balance_effective_start` Int64 DEFAULT -1 COMMENT 'effective balance at the first slot of the current epoch', + `balance_effective_end` Int64 DEFAULT -1 COMMENT 'effective balance at the last slot of the current epoch', + `balance_start` Int64 COMMENT 'balance at the last slot of the previous epoch', + `balance_end` Int64 COMMENT 'balance at the last slot of the current epoch', + `deposits_count` Int64 COMMENT 'number of deposits', + `deposits_amount` Int64 COMMENT 'total amount of deposits', + `withdrawals_count` Int64 COMMENT 'number of withdrawals', + `withdrawals_amount` Int64 COMMENT 'total amount of withdrawals', + `attestations_scheduled` Int64 COMMENT 'number of attestations scheduled', + `attestations_observed` Int64 COMMENT 'number of attestations executed', + `attestations_head_matched` Int64 COMMENT 'number of attestations matching head', + `attestations_target_matched` Int64 COMMENT 'number of attestations matching target', + `attestations_source_matched` Int64 COMMENT 'number of attestations matching source', + `attestations_head_executed` Int64 COMMENT 'number of attestations executed on head', + `attestations_target_executed` Int64 COMMENT 'number of attestations executed on target', + `attestations_source_executed` Int64 COMMENT 'number of attestations executed on source', + `attestations_head_reward` Int64 COMMENT 'total reward for attestations on head', + `attestations_target_reward` Int64 COMMENT 'total reward for attestations on target', + `attestations_source_reward` Int64 COMMENT 'total reward for attestations on source', + `attestations_inactivity_reward` Int64 COMMENT 'total reward for attestations inactivity', + `attestations_inclusion_reward` Int64 COMMENT 'total reward for attestations inclusion', + `attestations_ideal_head_reward` Int64 COMMENT 'ideal reward for attestations on head' , + `attestations_ideal_target_reward` Int64 COMMENT 'ideal reward for attestations on target', + `attestations_ideal_source_reward` Int64 COMMENT 'ideal reward for attestations on source', + `attestations_ideal_inactivity_reward` Int64 COMMENT 'ideal reward for attestations inactivity', + `attestations_ideal_inclusion_reward` Int64 COMMENT 'ideal reward for attestations inclusion', + `attestations_localized_max_reward` Int64 COMMENT 'slot localized max reward for attestations', + `attestations_hyperlocalized_max_reward` Int64 COMMENT 'committee localized max reward for attestations', + `inclusion_delay_sum` Int64 COMMENT 'sum of inclusion delays', + `optimal_inclusion_delay_sum` Int64 COMMENT 'sum of optimal inclusion delays', + `blocks_status` Nested ( + `slot` Int64, + `proposed` Bool + ) COMMENT 'block status', + `block_rewards` Nested ( + `slot` Int64, + `attestations_reward` Int64, + `sync_aggregate_reward` Int64, + `slasher_reward` Int64 + ) COMMENT 'block rewards', + `blocks_cl_missed_median_reward` Int64 COMMENT 'average reward for missed blocks', + `blocks_slashing_count` Int64 COMMENT 'slashings in block count', + `blocks_expected` Float64 COMMENT 'expected blocks', + `sync_scheduled` Int64 COMMENT 'number of syncs scheduled', -- left because its not supposed to count skipped slots + `sync_status` Nested ( + `slot` Int64, + `executed` Bool + ) COMMENT 'sync status', + `sync_rewards` Nested ( + `slot` Int64, + `reward` Int64 + ) COMMENT 'sync rewards', + `sync_localized_max_reward` Int64 COMMENT 'slot localized max reward for syncs', + `sync_committees_expected` Float64 COMMENT 'expected sync committees', + `slashed` Bool COMMENT 'if the validator was slashed in the epoch ', + `attestation_assignments` Nested ( + `slot` Int64, + `committee` Int64, + `index` Int64 + ) COMMENT 'attestation assignments', + `sync_committee_assignments` Nested ( + `period` Int64, + `index` Int64 + ) COMMENT 'sync committee assignments' ) -ENGINE = AggregatingMergeTree() -PRIMARY KEY (week, validator_index) -ORDER BY (week, validator_index) +ENGINE = Null; +-- +goose StatementEnd +-- +goose StatementBegin +SET flatten_nested = 1; -- reset to default +-- +goose StatementEnd +-- +goose StatementBegin +CREATE TABLE IF NOT EXISTS _unsafe_validator_dashboard_data_epoch +( + `_inserted_at` DateTime COMMENT 'insertion timestamp', + `validator_index` UInt64 COMMENT 'validator index', + `epoch` Int64 COMMENT 'epoch number', + `epoch_timestamp` DateTime COMMENT 'timestamp of the first slot of the epoch', + `balance_effective_start` Int64 COMMENT 'effective balance at the first slot of the current epoch', + `balance_effective_end` Int64 COMMENT 'effective balance at the last slot of the current epoch', + `balance_start` Int64 COMMENT 'balance at the last slot of the previous epoch', + `balance_end` Int64 COMMENT 'balance at the last slot of the current epoch', + `deposits_count` Int64 COMMENT 'number of deposits', + `deposits_amount` Int64 COMMENT 'total amount of deposits', + `withdrawals_count` Int64 COMMENT 'number of withdrawals', + `withdrawals_amount` Int64 COMMENT 'total amount of withdrawals', + `attestations_scheduled` Int64 COMMENT 'number of attestations scheduled', + `attestations_observed` Int64 COMMENT 'number of attestations executed', + `attestations_head_matched` Int64 COMMENT 'number of attestations matching head', + `attestations_target_matched` Int64 COMMENT 'number of attestations matching target', + `attestations_source_matched` Int64 COMMENT 'number of attestations matching source', + `attestations_head_executed` Int64 COMMENT 'number of attestations executed on head', + `attestations_target_executed` Int64 COMMENT 'number of attestations executed on target', + `attestations_source_executed` Int64 COMMENT 'number of attestations executed on source', + `attestations_head_reward_rewards_only` Int64 COMMENT 'total reward for attestations on head, rewards only', + `attestations_head_reward_penalties_only` Int64 COMMENT 'total reward for attestations on head, penalties only', + `attestations_target_reward_rewards_only` Int64 COMMENT 'total reward for attestations on target, rewards only', + `attestations_target_reward_penalties_only` Int64 COMMENT 'total reward for attestations on target, penalties only', + `attestations_source_reward_rewards_only` Int64 COMMENT 'total reward for attestations on source, rewards only', + `attestations_source_reward_penalties_only` Int64 COMMENT 'total reward for attestations on source, penalties only', + `attestations_inactivity_reward_rewards_only` Int64 COMMENT 'total reward for attestations inactivity, rewards only', + `attestations_inactivity_reward_penalties_only` Int64 COMMENT 'total reward for attestations inactivity, penalties only', + `attestations_inclusion_reward_rewards_only` Int64 COMMENT 'total reward for attestations inclusion, rewards only', + `attestations_inclusion_reward_penalties_only` Int64 COMMENT 'total reward for attestations inclusion, penalties', + `attestations_ideal_head_reward` Int64 COMMENT 'ideal reward for attestations on head', + `attestations_ideal_target_reward` Int64 COMMENT 'ideal reward for attestations on target', + `attestations_ideal_source_reward` Int64 COMMENT 'ideal reward for attestations on source', + `attestations_ideal_inactivity_reward` Int64 COMMENT 'ideal reward for attestations inactivity', + `attestations_ideal_inclusion_reward` Int64 COMMENT 'ideal reward for attestations inclusion', + `attestations_localized_max_reward` Int64 COMMENT 'slot localized max reward for attestations', + `attestations_hyperlocalized_max_reward` Int64 COMMENT 'committee localized max reward for attestations', + `inclusion_delay_sum` Int64 COMMENT 'sum of inclusion delays', + `optimal_inclusion_delay_sum` Int64 COMMENT 'sum of optimal inclusion delays', + `blocks_scheduled` Int64 COMMENT 'number of blocks scheduled', + `blocks_proposed` Int64 COMMENT 'number of blocks proposed', + `blocks_cl_reward` Int64 COMMENT 'total consensus layer block reward', + `blocks_cl_attestations_reward` Int64 COMMENT 'attestation consensus layer block reward', + `blocks_cl_sync_aggregate_reward` Int64 COMMENT 'sync aggregate consensus layer block reward', + `blocks_cl_slasher_reward` Int64 COMMENT 'slasher consensus layer block reward', + `blocks_cl_missed_median_reward` Int64 COMMENT 'average reward for missed blocks', + `blocks_slashing_count` Int64 COMMENT 'slashings in block count', + `blocks_expected` Float64 COMMENT 'expected blocks', + `sync_scheduled` Int64 COMMENT 'number of syncs scheduled', + `sync_executed` Int64 COMMENT 'number of syncs executed', + `sync_reward_rewards_only` Int64 COMMENT 'total sync rewards, rewards only', + `sync_reward_penalties_only` Int64 COMMENT 'total sync rewards, penalties only', + `sync_localized_max_reward` Int64 COMMENT 'slot localized max reward for syncs', + `sync_committees_expected` Float64 COMMENT 'expected sync committees', + `slashed` Bool COMMENT 'if the validator was slashed in the epoch' +) +ENGINE = ReplacingMergeTree(_inserted_at) +ORDER BY (toStartOfInterval(epoch_timestamp, INTERVAL 3 HOURS), validator_index, epoch_timestamp, epoch) -- 3 hours is around 30 entries per validator +-- PARTITION BY toStartOfHour(epoch_timestamp) --- this should be fine as the table will be cleaned up by the TTL -- aggressive partitioning not needed anymore due to the order by trick +PARTITION BY toMonday(epoch_timestamp) SETTINGS index_granularity = 8192; -- +goose StatementEnd -- +goose StatementBegin -CREATE TABLE validator_dashboard_data_monthly -( - `validator_index` UInt64, - `month` Date, - `epoch_start` SimpleAggregateFunction(min, Int64), - `epoch_end` SimpleAggregateFunction(max, Int64), - `attestations_source_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_target_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_head_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_inactivity_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_inclusion_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_ideal_source_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_ideal_target_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_ideal_head_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_ideal_inactivity_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_ideal_inclusion_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_ideal_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `blocks_scheduled` SimpleAggregateFunction(sum, Nullable(Int64)), - `blocks_proposed` SimpleAggregateFunction(sum, Nullable(Int64)), - `blocks_cl_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `blocks_el_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `sync_scheduled` SimpleAggregateFunction(sum, Nullable(Int64)), - `sync_executed` SimpleAggregateFunction(sum, Nullable(Int64)), - `sync_rewards` SimpleAggregateFunction(sum, Nullable(Int64)), - `slashed` SimpleAggregateFunction(max, Nullable(Bool)), - `balance_start` AggregateFunction(argMin, Nullable(Int64), Int64), - `balance_end` AggregateFunction(argMax, Nullable(Int64), Int64), - `deposits_count` SimpleAggregateFunction(sum, Nullable(Int64)), - `deposits_amount` SimpleAggregateFunction(sum, Nullable(Int64)), - `withdrawals_count` SimpleAggregateFunction(sum, Nullable(Int64)), - `withdrawals_amount` SimpleAggregateFunction(sum, Nullable(Int64)), - `inclusion_delay_sum` SimpleAggregateFunction(sum, Nullable(Int64)), - `blocks_expected` SimpleAggregateFunction(sum, Nullable(Float64)), - `sync_committees_expected` SimpleAggregateFunction(sum, Nullable(Float64)), - `attestations_scheduled` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_executed` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestation_head_executed` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestation_source_executed` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestation_target_executed` SimpleAggregateFunction(sum, Nullable(Int64)), - `optimal_inclusion_delay_sum` SimpleAggregateFunction(sum, Nullable(Int64)), - `slasher_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `slashed_by` SimpleAggregateFunction(any, Nullable(Int64)), --- should only ever be set once so any should be fine - `slashed_violation` SimpleAggregateFunction(any, Nullable(Int64)), --- should only ever be set once so any should be fine - `last_executed_duty_epoch` SimpleAggregateFunction(max, Nullable(Int64)), - `blocks_cl_attestations_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `blocks_cl_sync_aggregate_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - -- add projection to optimize validator_index queries - Projection validator_index_month_projection - ( - select * order by validator_index, month - ) +-- create safe non replacing version +CREATE TABLE IF NOT EXISTS _final_validator_dashboard_data_epoch as _unsafe_validator_dashboard_data_epoch +ENGINE = MergeTree +ORDER BY (toStartOfInterval(epoch_timestamp, INTERVAL 3 HOURS), validator_index, epoch_timestamp, epoch) +PARTITION BY toMonday(epoch_timestamp) +SETTINGS index_granularity = 8192, non_replicated_deduplication_window = 2048, replicated_deduplication_window = 2048; +-- +goose StatementEnd +-- +goose StatementBegin +-- make _inserted_at column EPHEMERAL +ALTER TABLE _final_validator_dashboard_data_epoch DROP COLUMN IF EXISTS _inserted_at; +-- +goose StatementEnd +-- +goose StatementBegin +-- materialized view to forward data to the unsafe table +CREATE MATERIALIZED VIEW IF NOT EXISTS _mv_unsafe_validator_dashboard_data_epoch TO _unsafe_validator_dashboard_data_epoch +AS SELECT + now() as _inserted_at, + validator_index, + epoch, + epoch_timestamp, + balance_effective_start, + balance_effective_end, + balance_start, + balance_end, + deposits_count, + deposits_amount, + withdrawals_count, + withdrawals_amount, + attestations_scheduled, + attestations_observed, + attestations_head_matched, + attestations_target_matched, + attestations_source_matched, + attestations_head_executed, + attestations_target_executed, + attestations_source_executed, + -- attestations_head_reward + attestations_target_reward + attestations_source_reward + attestations_inactivity_reward + attestations_inclusion_reward as attestations_reward, + -- greatest(attestations_head_reward, 0) + greatest(attestations_target_reward, 0) + greatest(attestations_source_reward, 0) + greatest(attestations_inactivity_reward, 0) + greatest(attestations_inclusion_reward, 0) as attestations_reward_rewards_only, + -- least(attestations_head_reward, 0) + least(attestations_target_reward, 0) + least(attestations_source_reward, 0) + least(attestations_inactivity_reward, 0) + least(attestations_inclusion_reward, 0) as attestations_reward_penalties_only, + greatest(attestations_head_reward, 0) as attestations_head_reward_rewards_only, + least(attestations_head_reward, 0) as attestations_head_reward_penalties_only, + greatest(attestations_target_reward, 0) as attestations_target_reward_rewards_only, + least(attestations_target_reward, 0) as attestations_target_reward_penalties_only, + greatest(attestations_source_reward, 0) as attestations_source_reward_rewards_only, + least(attestations_source_reward, 0) as attestations_source_reward_penalties_only, + greatest(attestations_inactivity_reward, 0) as attestations_inactivity_reward_rewards_only, + least(attestations_inactivity_reward, 0) as attestations_inactivity_reward_penalties_only, + greatest(attestations_inclusion_reward, 0) as attestations_inclusion_reward_rewards_only, + least(attestations_inclusion_reward, 0) as attestations_inclusion_reward_penalties_only, + attestations_ideal_head_reward, + attestations_ideal_target_reward, + attestations_ideal_source_reward, + attestations_ideal_inactivity_reward, + attestations_ideal_inclusion_reward, + attestations_localized_max_reward, + attestations_hyperlocalized_max_reward, + inclusion_delay_sum, + optimal_inclusion_delay_sum, + length(blocks_status.proposed) as blocks_scheduled, + arrayCount(x -> x, blocks_status.proposed) as blocks_proposed, + arraySum(block_rewards.attestations_reward) + arraySum(block_rewards.sync_aggregate_reward) + arraySum(block_rewards.slasher_reward) as blocks_cl_reward, + arraySum(block_rewards.attestations_reward) as blocks_cl_attestations_reward, + arraySum(block_rewards.sync_aggregate_reward) as blocks_cl_sync_aggregate_reward, + arraySum(block_rewards.slasher_reward) as blocks_cl_slasher_reward, + blocks_cl_missed_median_reward, + blocks_slashing_count, + blocks_expected, + sync_scheduled, + arrayCount(x -> x, sync_status.executed) as sync_executed, + arraySum(x -> greatest(0, x), sync_rewards.reward) as sync_reward_rewards_only, + arraySum(x -> least(0, x), sync_rewards.reward) as sync_reward_penalties_only, + sync_localized_max_reward, + sync_committees_expected, + slashed +FROM _insert_sink_validator_dashboard_data_epoch; +-- +goose StatementEnd +-- +goose StatementBegin +-- attesations assignements +CREATE TABLE IF NOT EXISTS validator_attestation_assignments_slot ( + `validator_index` UInt64 COMMENT 'validator index' CODEC(DoubleDelta, ZSTD(8)), + `epoch` Int64 COMMENT 'epoch number' CODEC(Delta, ZSTD(8)), + `epoch_timestamp` DateTime COMMENT 'timestamp of the first slot of the epoch' CODEC(Delta, ZSTD(8)), + `slot` Int64 CODEC(T64, ZSTD(8)), + `committee` Int64 CODEC(T64, ZSTD(8)), + `committee_index` Int64 CODEC(T64, ZSTD(8)), ) -ENGINE = AggregatingMergeTree() -PRIMARY KEY (month, validator_index) -ORDER BY (month, validator_index) +ENGINE = ReplacingMergeTree +ORDER BY (toStartOfInterval(epoch_timestamp, INTERVAL 3 HOURS), validator_index, epoch_timestamp, epoch, slot) +PARTITION BY toMonday(epoch_timestamp) SETTINGS index_granularity = 8192; -- +goose StatementEnd -- +goose StatementBegin -CREATE TABLE validator_dashboard_data_quarterly -( - `validator_index` UInt64, - `quarter` Date, - `epoch_start` SimpleAggregateFunction(min, Int64), - `epoch_end` SimpleAggregateFunction(max, Int64), - `attestations_source_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_target_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_head_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_inactivity_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_inclusion_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_ideal_source_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_ideal_target_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_ideal_head_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_ideal_inactivity_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_ideal_inclusion_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_ideal_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `blocks_scheduled` SimpleAggregateFunction(sum, Nullable(Int64)), - `blocks_proposed` SimpleAggregateFunction(sum, Nullable(Int64)), - `blocks_cl_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `blocks_el_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `sync_scheduled` SimpleAggregateFunction(sum, Nullable(Int64)), - `sync_executed` SimpleAggregateFunction(sum, Nullable(Int64)), - `sync_rewards` SimpleAggregateFunction(sum, Nullable(Int64)), - `slashed` SimpleAggregateFunction(max, Nullable(Bool)), - `balance_start` AggregateFunction(argMin, Nullable(Int64), Int64), - `balance_end` AggregateFunction(argMax, Nullable(Int64), Int64), - `deposits_count` SimpleAggregateFunction(sum, Nullable(Int64)), - `deposits_amount` SimpleAggregateFunction(sum, Nullable(Int64)), - `withdrawals_count` SimpleAggregateFunction(sum, Nullable(Int64)), - `withdrawals_amount` SimpleAggregateFunction(sum, Nullable(Int64)), - `inclusion_delay_sum` SimpleAggregateFunction(sum, Nullable(Int64)), - `blocks_expected` SimpleAggregateFunction(sum, Nullable(Float64)), - `sync_committees_expected` SimpleAggregateFunction(sum, Nullable(Float64)), - `attestations_scheduled` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestations_executed` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestation_head_executed` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestation_source_executed` SimpleAggregateFunction(sum, Nullable(Int64)), - `attestation_target_executed` SimpleAggregateFunction(sum, Nullable(Int64)), - `optimal_inclusion_delay_sum` SimpleAggregateFunction(sum, Nullable(Int64)), - `slasher_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `slashed_by` SimpleAggregateFunction(any, Nullable(Int64)), --- should only ever be set once so any should be fine - `slashed_violation` SimpleAggregateFunction(any, Nullable(Int64)), --- should only ever be set once so any should be fine - `last_executed_duty_epoch` SimpleAggregateFunction(max, Nullable(Int64)), - `blocks_cl_attestations_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - `blocks_cl_sync_aggregate_reward` SimpleAggregateFunction(sum, Nullable(Int64)), - -- add projection to optimize validator_index queries - Projection validator_index_quarter_projection - ( - select * order by validator_index, quarter - ) +-- materialized view to forward data to the unsafe table +CREATE MATERIALIZED VIEW IF NOT EXISTS _mv_validator_attestation_assignments_slot TO validator_attestation_assignments_slot +AS SELECT + validator_index, + epoch, + epoch_timestamp, + x.slot as slot, + x.committee as committee, + x.index as committee_index +FROM _insert_sink_validator_dashboard_data_epoch ARRAY JOIN attestation_assignments as x; +-- +goose StatementEnd +-- +goose StatementBegin +-- proposal assignments +CREATE TABLE IF NOT EXISTS validator_proposal_assignments_slot ( + `validator_index` UInt64 COMMENT 'validator index' CODEC(T64, ZSTD(8)), + `epoch` Int64 COMMENT 'epoch number' CODEC(Delta, ZSTD(8)), + `epoch_timestamp` DateTime COMMENT 'timestamp of the first slot of the epoch' CODEC(Delta, ZSTD(8)), + `slot` Int64 CODEC(Delta, ZSTD(8)) +) +ENGINE = ReplacingMergeTree +PARTITION BY toStartOfQuarter(epoch_timestamp) +ORDER BY (toMonday(epoch_timestamp), validator_index, epoch_timestamp, epoch, slot) +SETTINGS index_granularity = 8192; +-- +goose StatementEnd +-- +goose StatementBegin +-- materialized view to forward data to the unsafe table +CREATE MATERIALIZED VIEW IF NOT EXISTS _mv_validator_proposal_assignments_slot TO validator_proposal_assignments_slot +AS SELECT + validator_index, + epoch, + epoch_timestamp, + slot +FROM _insert_sink_validator_dashboard_data_epoch ARRAY JOIN blocks_status.slot as slot; +-- +goose StatementEnd +-- +goose StatementBegin +-- sync committee assignments +CREATE TABLE IF NOT EXISTS validator_sync_committee_assignments_epoch ( + `validator_index` UInt64 COMMENT 'validator index' CODEC(T64, ZSTD(8)), + `epoch` Int64 COMMENT 'epoch number' CODEC(Delta, ZSTD(8)), + `epoch_timestamp` DateTime COMMENT 'timestamp of the first slot of the epoch' CODEC(Delta, ZSTD(8)), + `period` Int64 CODEC(Delta, ZSTD(8)), + `period_index` Int64 CODEC(Delta, ZSTD(8)) +) +ENGINE = ReplacingMergeTree +PARTITION BY toStartOfMonth(epoch_timestamp) +ORDER BY (validator_index, epoch_timestamp, epoch, period) +SETTINGS index_granularity = 8192; +-- +goose StatementEnd +-- +goose StatementBegin +-- materialized view to forward data to the unsafe table +CREATE MATERIALIZED VIEW IF NOT EXISTS _mv_validator_sync_committee_assignments_epoch TO validator_sync_committee_assignments_epoch +AS SELECT + validator_index, + epoch, + epoch_timestamp, + x.period as period, + x.index as period_index +FROM _insert_sink_validator_dashboard_data_epoch ARRAY JOIN sync_committee_assignments as x; +-- +goose StatementEnd +-- +goose StatementBegin +-- proposal reward table +CREATE TABLE IF NOT EXISTS validator_proposal_rewards_slot ( + `validator_index` UInt64 COMMENT 'validator index' CODEC(T64, ZSTD(8)), + `epoch` Int64 COMMENT 'epoch number' CODEC(Delta, ZSTD(8)), + `epoch_timestamp` DateTime COMMENT 'timestamp of the first slot of the epoch' CODEC(Delta, ZSTD(8)), + `slot` Int64 CODEC(Delta, ZSTD(8)), + `attestations_reward` Int64 COMMENT 'reward for including attestations in the proposal' CODEC(T64, ZSTD(8)), + `sync_aggregate_reward` Int64 COMMENT 'reward for including sync aggregate in the proposal' CODEC(T64, ZSTD(8)), + `slasher_reward` Int64 COMMENT 'reward for including slasher in the proposal' CODEC(T64, ZSTD(8)), ) -ENGINE = AggregatingMergeTree() -PRIMARY KEY (quarter, validator_index) -ORDER BY (quarter, validator_index) +ENGINE = ReplacingMergeTree +ORDER BY (toMonday(epoch_timestamp), validator_index, epoch_timestamp, epoch, slot) +PARTITION BY toStartOfQuarter(epoch_timestamp) SETTINGS index_granularity = 8192; -- +goose StatementEnd -- +goose StatementBegin - -CREATE MATERIALIZED VIEW validator_dashboard_data_weekly_mv TO validator_dashboard_data_weekly +-- materialized view to forward data to the unsafe table +CREATE MATERIALIZED VIEW IF NOT EXISTS _mv_validator_proposal_rewards_slot TO validator_proposal_rewards_slot AS SELECT - validator_index AS validator_index, - toMonday(day) AS week, - minSimpleState(epoch_start) as epoch_start, - maxSimpleState(epoch_end) as epoch_end, - sumSimpleState(attestations_source_reward) as attestations_source_reward, - sumSimpleState(attestations_target_reward) as attestations_target_reward, - sumSimpleState(attestations_head_reward) as attestations_head_reward, - sumSimpleState(attestations_inactivity_reward) as attestations_inactivity_reward, - sumSimpleState(attestations_inclusion_reward) as attestations_inclusion_reward, - sumSimpleState(attestations_reward) as attestations_reward, - sumSimpleState(attestations_ideal_source_reward) as attestations_ideal_source_reward, - sumSimpleState(attestations_ideal_target_reward) as attestations_ideal_target_reward, - sumSimpleState(attestations_ideal_head_reward) as attestations_ideal_head_reward, - sumSimpleState(attestations_ideal_inactivity_reward) as attestations_ideal_inactivity_reward, - sumSimpleState(attestations_ideal_inclusion_reward) as attestations_ideal_inclusion_reward, - sumSimpleState(attestations_ideal_reward) as attestations_ideal_reward, - sumSimpleState(blocks_scheduled) as blocks_scheduled, - sumSimpleState(blocks_proposed) as blocks_proposed, - sumSimpleState(blocks_cl_reward) as blocks_cl_reward, - sumSimpleState(blocks_el_reward) as blocks_el_reward, - sumSimpleState(sync_scheduled) as sync_scheduled, - sumSimpleState(sync_executed) as sync_executed, - sumSimpleState(sync_rewards) as sync_rewards, - maxSimpleState(slashed) as slashed, - argMinState(foo.balance_start, foo.epoch_start) as balance_start, - argMaxState(foo.balance_end, foo.epoch_end) as balance_end, - sumSimpleState(deposits_count) as deposits_count, - sumSimpleState(deposits_amount) as deposits_amount, - sumSimpleState(withdrawals_count) as withdrawals_count, - sumSimpleState(withdrawals_amount) as withdrawals_amount, - sumSimpleState(inclusion_delay_sum) as inclusion_delay_sum, - sumSimpleState(blocks_expected) as blocks_expected, - sumSimpleState(sync_committees_expected) as sync_committees_expected, - sumSimpleState(attestations_scheduled) as attestations_scheduled, - sumSimpleState(attestations_executed) as attestations_executed, - sumSimpleState(attestation_head_executed) as attestation_head_executed, - sumSimpleState(attestation_source_executed) as attestation_source_executed, - sumSimpleState(attestation_target_executed) as attestation_target_executed, - sumSimpleState(optimal_inclusion_delay_sum) as optimal_inclusion_delay_sum, - sumSimpleState(slasher_reward) as slasher_reward, - anySimpleState(slashed_by) as slashed_by, - anySimpleState(slashed_violation) as slashed_violation, - maxSimpleState(last_executed_duty_epoch) as last_executed_duty_epoch, - sumSimpleState(blocks_cl_attestations_reward) as blocks_cl_attestations_reward, - sumSimpleState(blocks_cl_sync_aggregate_reward) as blocks_cl_sync_aggregate_reward -FROM validator_dashboard_data_daily foo -GROUP BY - week, - validator_index --- +goose StatementEnd --- +goose StatementBegin - -CREATE MATERIALIZED VIEW validator_dashboard_data_monthly_mv TO validator_dashboard_data_monthly + validator_index, + epoch, + epoch_timestamp, + x.slot as slot, + x.attestations_reward as attestations_reward, + x.sync_aggregate_reward as sync_aggregate_reward, + x.slasher_reward as slasher_reward +FROM _insert_sink_validator_dashboard_data_epoch ARRAY JOIN block_rewards as x; +-- +goose StatementEnd +-- +goose StatementBegin +CREATE TABLE IF NOT EXISTS validator_sync_committee_rewards_slot ( + `validator_index` UInt64 COMMENT 'validator index' CODEC(T64, ZSTD(8)), + `epoch` Int64 COMMENT 'epoch number' CODEC(Delta, ZSTD(8)), + `epoch_timestamp` DateTime COMMENT 'timestamp of the first slot of the epoch' CODEC(Delta, ZSTD(8)), + `slot` Int64 CODEC(Delta, ZSTD(8)), + `reward` Int64 COMMENT 'reward for the sync committee contribution' CODEC(T64, ZSTD(8)) +) +ENGINE = ReplacingMergeTree +ORDER BY (toStartOfDay(epoch_timestamp), validator_index, epoch_timestamp, epoch, slot) +PARTITION BY toStartOfMonth(epoch_timestamp) +SETTINGS index_granularity = 8192; +-- +goose StatementEnd +-- +goose StatementBegin +CREATE MATERIALIZED VIEW IF NOT EXISTS _mv_validator_sync_committee_rewards_slot TO validator_sync_committee_rewards_slot AS SELECT - validator_index AS validator_index, - toStartOfMonth(week) AS month, - minSimpleState(epoch_start) as epoch_start, - maxSimpleState(epoch_end) as epoch_end, - sumSimpleState(attestations_source_reward) as attestations_source_reward, - sumSimpleState(attestations_target_reward) as attestations_target_reward, - sumSimpleState(attestations_head_reward) as attestations_head_reward, - sumSimpleState(attestations_inactivity_reward) as attestations_inactivity_reward, - sumSimpleState(attestations_inclusion_reward) as attestations_inclusion_reward, - sumSimpleState(attestations_reward) as attestations_reward, - sumSimpleState(attestations_ideal_source_reward) as attestations_ideal_source_reward, - sumSimpleState(attestations_ideal_target_reward) as attestations_ideal_target_reward, - sumSimpleState(attestations_ideal_head_reward) as attestations_ideal_head_reward, - sumSimpleState(attestations_ideal_inactivity_reward) as attestations_ideal_inactivity_reward, - sumSimpleState(attestations_ideal_inclusion_reward) as attestations_ideal_inclusion_reward, - sumSimpleState(attestations_ideal_reward) as attestations_ideal_reward, - sumSimpleState(blocks_scheduled) as blocks_scheduled, - sumSimpleState(blocks_proposed) as blocks_proposed, - sumSimpleState(blocks_cl_reward) as blocks_cl_reward, - sumSimpleState(blocks_el_reward) as blocks_el_reward, - sumSimpleState(sync_scheduled) as sync_scheduled, - sumSimpleState(sync_executed) as sync_executed, - sumSimpleState(sync_rewards) as sync_rewards, - maxSimpleState(slashed) as slashed, - argMinMergeState(balance_start) as balance_start, - argMaxMergeState(balance_end) as balance_end, - sumSimpleState(deposits_count) as deposits_count, - sumSimpleState(deposits_amount) as deposits_amount, - sumSimpleState(withdrawals_count) as withdrawals_count, - sumSimpleState(withdrawals_amount) as withdrawals_amount, - sumSimpleState(inclusion_delay_sum) as inclusion_delay_sum, - sumSimpleState(blocks_expected) as blocks_expected, - sumSimpleState(sync_committees_expected) as sync_committees_expected, - sumSimpleState(attestations_scheduled) as attestations_scheduled, - sumSimpleState(attestations_executed) as attestations_executed, - sumSimpleState(attestation_head_executed) as attestation_head_executed, - sumSimpleState(attestation_source_executed) as attestation_source_executed, - sumSimpleState(attestation_target_executed) as attestation_target_executed, - sumSimpleState(optimal_inclusion_delay_sum) as optimal_inclusion_delay_sum, - sumSimpleState(slasher_reward) as slasher_reward, - anySimpleState(slashed_by) as slashed_by, - anySimpleState(slashed_violation) as slashed_violation, - maxSimpleState(last_executed_duty_epoch) as last_executed_duty_epoch, - sumSimpleState(blocks_cl_attestations_reward) as blocks_cl_attestations_reward, - sumSimpleState(blocks_cl_sync_aggregate_reward) as blocks_cl_sync_aggregate_reward -FROM validator_dashboard_data_weekly -GROUP BY - month, - validator_index --- +goose StatementEnd --- +goose StatementBegin - -CREATE MATERIALIZED VIEW validator_dashboard_data_quarterly_mv TO validator_dashboard_data_quarterly + validator_index, + epoch, + epoch_timestamp, + x.slot as slot, + x.reward as reward +FROM _insert_sink_validator_dashboard_data_epoch ARRAY JOIN sync_rewards as x; +-- +goose StatementEnd +-- +goose StatementBegin +-- sync_executed_map array join to generate table of validator_index, epoch_timestamp, slot +CREATE TABLE IF NOT EXISTS validator_sync_committee_votes_slot ( + `validator_index` UInt64 COMMENT 'validator index' CODEC(DoubleDelta, ZSTD(8)), + `epoch` Int64 COMMENT 'epoch number' CODEC(Delta, ZSTD(8)), + `epoch_timestamp` DateTime COMMENT 'timestamp of the first slot of the epoch' CODEC(Delta, ZSTD(8)), + `slot` Int64 COMMENT 'slot number' CODEC(T64, ZSTD(8)), + `executed` Bool COMMENT 'if the sync was executed' +) +ENGINE = ReplacingMergeTree +ORDER BY (toStartOfDay(epoch_timestamp), validator_index, epoch_timestamp, epoch, slot) +PARTITION BY toStartOfMonth(epoch_timestamp) +SETTINGS index_granularity = 8192; +-- +goose StatementEnd +-- +goose StatementBegin +-- materialized view to forward data to the unsafe table +CREATE MATERIALIZED VIEW IF NOT EXISTS _mv_validator_sync_committee_votes_slot TO validator_sync_committee_votes_slot AS SELECT - validator_index AS validator_index, - toStartOfQuarter(month) AS quarter, - minSimpleState(epoch_start) as epoch_start, - maxSimpleState(epoch_end) as epoch_end, - sumSimpleState(attestations_source_reward) as attestations_source_reward, - sumSimpleState(attestations_target_reward) as attestations_target_reward, - sumSimpleState(attestations_head_reward) as attestations_head_reward, - sumSimpleState(attestations_inactivity_reward) as attestations_inactivity_reward, - sumSimpleState(attestations_inclusion_reward) as attestations_inclusion_reward, - sumSimpleState(attestations_reward) as attestations_reward, - sumSimpleState(attestations_ideal_source_reward) as attestations_ideal_source_reward, - sumSimpleState(attestations_ideal_target_reward) as attestations_ideal_target_reward, - sumSimpleState(attestations_ideal_head_reward) as attestations_ideal_head_reward, - sumSimpleState(attestations_ideal_inactivity_reward) as attestations_ideal_inactivity_reward, - sumSimpleState(attestations_ideal_inclusion_reward) as attestations_ideal_inclusion_reward, - sumSimpleState(attestations_ideal_reward) as attestations_ideal_reward, - sumSimpleState(blocks_scheduled) as blocks_scheduled, - sumSimpleState(blocks_proposed) as blocks_proposed, - sumSimpleState(blocks_cl_reward) as blocks_cl_reward, - sumSimpleState(blocks_el_reward) as blocks_el_reward, - sumSimpleState(sync_scheduled) as sync_scheduled, - sumSimpleState(sync_executed) as sync_executed, - sumSimpleState(sync_rewards) as sync_rewards, - maxSimpleState(slashed) as slashed, - argMinMergeState(balance_start) as balance_start, - argMaxMergeState(balance_end) as balance_end, - sumSimpleState(deposits_count) as deposits_count, - sumSimpleState(deposits_amount) as deposits_amount, - sumSimpleState(withdrawals_count) as withdrawals_count, - sumSimpleState(withdrawals_amount) as withdrawals_amount, - sumSimpleState(inclusion_delay_sum) as inclusion_delay_sum, - sumSimpleState(blocks_expected) as blocks_expected, - sumSimpleState(sync_committees_expected) as sync_committees_expected, - sumSimpleState(attestations_scheduled) as attestations_scheduled, - sumSimpleState(attestations_executed) as attestations_executed, - sumSimpleState(attestation_head_executed) as attestation_head_executed, - sumSimpleState(attestation_source_executed) as attestation_source_executed, - sumSimpleState(attestation_target_executed) as attestation_target_executed, - sumSimpleState(optimal_inclusion_delay_sum) as optimal_inclusion_delay_sum, - sumSimpleState(slasher_reward) as slasher_reward, - anySimpleState(slashed_by) as slashed_by, - anySimpleState(slashed_violation) as slashed_violation, - maxSimpleState(last_executed_duty_epoch) as last_executed_duty_epoch, - sumSimpleState(blocks_cl_attestations_reward) as blocks_cl_attestations_reward, - sumSimpleState(blocks_cl_sync_aggregate_reward) as blocks_cl_sync_aggregate_reward -FROM validator_dashboard_data_monthly -GROUP BY - month, - validator_index --- +goose StatementEnd - + validator_index, + epoch, + epoch_timestamp, + x.slot as slot, + x.executed as executed +FROM _insert_sink_validator_dashboard_data_epoch ARRAY JOIN sync_status as x; +-- +goose StatementEnd +-- +goose StatementBegin +CREATE TABLE IF NOT EXISTS _final_validator_dashboard_data_hourly ( + `validator_index` UInt64 COMMENT 'validator index' CODEC(DoubleDelta, ZSTD(8)), + `t` DateTime COMMENT 'timestamp of the aggregated data' CODEC(Delta, ZSTD(8)), + `epoch_map` AggregateFunction(groupArraySortedIf(2048), Int64, Bool) COMMENT 'consistency check data - only available for validator 0', + `epoch_start` SimpleAggregateFunction(min, Int64) COMMENT 'first epoch included in the aggregation', + `epoch_end` SimpleAggregateFunction(max, Int64) COMMENT 'last epoch included in the aggregation', + `balance_start` AggregateFunction(argMin, Int64, Int64) COMMENT 'balance at the first slot of the first epoch included in the aggregation', + `balance_end` AggregateFunction(argMax, Int64, Int64) COMMENT 'balance at the last slot of the last epoch included in the aggregation', + `balance_min` SimpleAggregateFunction(min, Int64) COMMENT 'minimum balance in the aggregation', + `balance_max` SimpleAggregateFunction(max, Int64) COMMENT 'maximum balance in the aggregation', + `deposits_count` SimpleAggregateFunction(sum, Int64) COMMENT 'number of deposits', + `deposits_amount` SimpleAggregateFunction(sum, Int64) COMMENT 'total amount of deposits', + `withdrawals_count` SimpleAggregateFunction(sum, Int64) COMMENT 'number of withdrawals', + `withdrawals_amount` SimpleAggregateFunction(sum, Int64) COMMENT 'total amount of withdrawals', + `attestations_scheduled` SimpleAggregateFunction(sum, Int64) COMMENT 'number of attestations scheduled', + `attestations_observed` SimpleAggregateFunction(sum, Int64) COMMENT 'number of attestations executed', + `attestations_head_matched` SimpleAggregateFunction(sum, Int64) COMMENT 'number of attestations matching head', + `attestations_target_matched` SimpleAggregateFunction(sum, Int64) COMMENT 'number of attestations matching target', + `attestations_source_matched` SimpleAggregateFunction(sum, Int64) COMMENT 'number of attestations matching source', + `attestations_head_executed` SimpleAggregateFunction(sum, Int64) COMMENT 'number of attestations executed on head', + `attestations_target_executed` SimpleAggregateFunction(sum, Int64) COMMENT 'number of attestations executed on target', + `attestations_source_executed` SimpleAggregateFunction(sum, Int64) COMMENT 'number of attestations executed on source', + `attestations_head_reward_rewards_only` SimpleAggregateFunction(sum, Int64) COMMENT 'total reward for attestations on head, rewards only' CODEC(T64, ZSTD(8)), + `attestations_head_reward_penalties_only` SimpleAggregateFunction(sum, Int64) COMMENT 'total reward for attestations on head, penalties only' CODEC(T64, ZSTD(8)), + `attestations_target_reward_rewards_only` SimpleAggregateFunction(sum, Int64) COMMENT 'total reward for attestations on target, rewards only' CODEC(T64, ZSTD(8)), + `attestations_target_reward_penalties_only` SimpleAggregateFunction(sum, Int64) COMMENT 'total reward for attestations on target, penalties only' CODEC(T64, ZSTD(8)), + `attestations_source_reward_rewards_only` SimpleAggregateFunction(sum, Int64) COMMENT 'total reward for attestations on source, rewards only' CODEC(T64, ZSTD(8)), + `attestations_source_reward_penalties_only` SimpleAggregateFunction(sum, Int64) COMMENT 'total reward for attestations on source, penalties only' CODEC(T64, ZSTD(8)), + `attestations_inactivity_reward_rewards_only` SimpleAggregateFunction(sum, Int64) COMMENT 'total reward for attestations inactivity, rewards only' CODEC(T64, ZSTD(8)), + `attestations_inactivity_reward_penalties_only` SimpleAggregateFunction(sum, Int64) COMMENT 'total reward for attestations inactivity, penalties only' CODEC(T64, ZSTD(8)), + `attestations_inclusion_reward_rewards_only` SimpleAggregateFunction(sum, Int64) COMMENT 'total reward for attestations inclusion, rewards only' CODEC(T64, ZSTD(8)), + `attestations_inclusion_reward_penalties_only` SimpleAggregateFunction(sum, Int64) COMMENT 'total reward for attestations inclusion, penalties only' CODEC(T64, ZSTD(8)), + `attestations_ideal_head_reward` SimpleAggregateFunction(sum, Int64) COMMENT 'ideal reward for attestations on head' CODEC(Delta, ZSTD(8)), + `attestations_ideal_target_reward` SimpleAggregateFunction(sum, Int64) COMMENT 'ideal reward for attestations on target' CODEC(Delta, ZSTD(8)), + `attestations_ideal_source_reward` SimpleAggregateFunction(sum, Int64) COMMENT 'ideal reward for attestations on source' CODEC(Delta, ZSTD(8)), + `attestations_ideal_inactivity_reward` SimpleAggregateFunction(sum, Int64) COMMENT 'ideal reward for attestations inactivity' CODEC(Delta, ZSTD(8)), + `attestations_ideal_inclusion_reward` SimpleAggregateFunction(sum, Int64) COMMENT 'ideal reward for attestations inclusion' CODEC(Delta, ZSTD(8)), + `attestations_localized_max_reward` SimpleAggregateFunction(sum, Int64) COMMENT 'slot localized max reward for attestations' CODEC(Delta, ZSTD(8)), + `attestations_hyperlocalized_max_reward` SimpleAggregateFunction(sum, Int64) COMMENT 'committee localized max reward for attestations' CODEC(Delta, ZSTD(8)), + `inclusion_delay_sum` SimpleAggregateFunction(sum, Int64) COMMENT 'sum of inclusion delays' CODEC(T64, ZSTD(8)), + `optimal_inclusion_delay_sum` SimpleAggregateFunction(sum, Int64) COMMENT 'sum of optimal inclusion delays' CODEC(T64, ZSTD(8)), + `blocks_scheduled` SimpleAggregateFunction(sum, Int64) COMMENT 'number of blocks scheduled', + `blocks_proposed` SimpleAggregateFunction(sum, Int64) COMMENT 'number of blocks proposed', + `blocks_cl_reward` SimpleAggregateFunction(sum, Int64) COMMENT 'total consensus layer block reward', + `blocks_cl_attestations_reward` SimpleAggregateFunction(sum, Int64) COMMENT 'attestation consensus layer block reward', + `blocks_cl_sync_aggregate_reward` SimpleAggregateFunction(sum, Int64) COMMENT 'sync aggregate consensus layer block reward', + `blocks_cl_slasher_reward` SimpleAggregateFunction(sum, Int64) COMMENT 'slasher consensus layer block reward', + `blocks_cl_missed_median_reward` SimpleAggregateFunction(sum, Int64) COMMENT 'average reward for missed blocks' CODEC(T64, ZSTD(8)), + `blocks_slashing_count` SimpleAggregateFunction(sum, Int64) COMMENT 'slashings in block count', + `blocks_expected` SimpleAggregateFunction(sum, Float64) COMMENT 'expected blocks' CODEC(FPC, ZSTD(8)), + `sync_scheduled` SimpleAggregateFunction(sum, Int64) COMMENT 'number of syncs scheduled', + `sync_executed` SimpleAggregateFunction(sum, Int64) COMMENT 'number of syncs executed', + `sync_reward_rewards_only` SimpleAggregateFunction(sum, Int64) COMMENT 'total sync rewards, rewards only' CODEC(T64, ZSTD(8)), + `sync_reward_penalties_only` SimpleAggregateFunction(sum, Int64) COMMENT 'total sync rewards, penalties only' CODEC(T64, ZSTD(8)), + `sync_localized_max_reward` SimpleAggregateFunction(sum, Int64) COMMENT 'slot localized max reward for syncs' CODEC(Delta, ZSTD(8)), + `sync_committees_expected` SimpleAggregateFunction(sum, Float64) COMMENT 'expected sync committees' CODEC(FPC, ZSTD(8)), + `slashed` SimpleAggregateFunction(max, Bool) COMMENT 'if the validator was slashed in the epoch', + `last_executed_duty_epoch` SimpleAggregateFunction(max, Nullable(Int64)), + `last_scheduled_sync_epoch` SimpleAggregateFunction(max, Nullable(Int64)), + `last_scheduled_block_epoch` SimpleAggregateFunction(max, Nullable(Int64)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toStartOfMonth(t) +ORDER BY (toStartOfDay(t), validator_index, t) +SETTINGS index_granularity = 8192, non_replicated_deduplication_window = 2048, replicated_deduplication_window = 2048; +-- +goose StatementEnd +-- +goose StatementBegin +-- daily +CREATE TABLE IF NOT EXISTS _final_validator_dashboard_data_daily as _final_validator_dashboard_data_hourly +ENGINE = AggregatingMergeTree +PARTITION BY toStartOfYear(t) +ORDER BY (toStartOfMonth(t), validator_index, t) +SETTINGS index_granularity = 8192, non_replicated_deduplication_window = 2048, replicated_deduplication_window = 2048; +-- +goose StatementEnd +-- +goose StatementBegin +-- weekly +CREATE TABLE IF NOT EXISTS _final_validator_dashboard_data_weekly as _final_validator_dashboard_data_hourly +ENGINE = AggregatingMergeTree +PARTITION BY toStartOfInterval(t, INTERVAL 3 YEARS) +ORDER BY (toStartOfInterval(t, INTERVAL 6 MONTHS), validator_index, t) +SETTINGS index_granularity = 8192, non_replicated_deduplication_window = 2048, replicated_deduplication_window = 2048; +-- +goose StatementEnd +-- +goose StatementBegin +-- monthly +CREATE TABLE IF NOT EXISTS _final_validator_dashboard_data_monthly as _final_validator_dashboard_data_hourly +ENGINE = AggregatingMergeTree +ORDER BY (toStartOfYear(t), validator_index, t) +SETTINGS index_granularity = 8192, non_replicated_deduplication_window = 2048, replicated_deduplication_window = 2048; +-- +goose StatementEnd -- +goose Down -- +goose StatementBegin -DROP TABLE IF EXISTS validator_dashboard_data_daily; +DROP TABLE IF EXISTS _final_validator_dashboard_data_monthly; +-- +goose StatementEnd +-- +goose StatementBegin +DROP TABLE IF EXISTS _final_validator_dashboard_data_weekly; -- +goose StatementEnd -- +goose StatementBegin -DROP TABLE IF EXISTS validator_dashboard_data_weekly; +DROP TABLE IF EXISTS _final_validator_dashboard_data_daily; -- +goose StatementEnd -- +goose StatementBegin -DROP TABLE IF EXISTS validator_dashboard_data_monthly; +DROP TABLE IF EXISTS _final_validator_dashboard_data_hourly; -- +goose StatementEnd -- +goose StatementBegin -DROP TABLE IF EXISTS validator_dashboard_data_quarterly; +DROP TABLE IF EXISTS validator_proposal_assignments_slot; -- +goose StatementEnd -- +goose StatementBegin -DROP TABLE IF EXISTS validator_dashboard_data_weekly_mv; +DROP TABLE IF EXISTS validator_attestation_assignments_slot; -- +goose StatementEnd -- +goose StatementBegin -DROP TABLE IF EXISTS validator_dashboard_data_monthly_mv; +DROP TABLE IF EXISTS _mv_validator_proposal_assignments_slot; -- +goose StatementEnd -- +goose StatementBegin -DROP TABLE IF EXISTS validator_dashboard_data_quarterly_mv; --- +goose StatementEnd \ No newline at end of file +DROP TABLE IF EXISTS _mv_validator_attestation_assignments_slot; +-- +goose StatementEnd +-- +goose StatementBegin +DROP TABLE IF EXISTS _mv_unsafe_validator_dashboard_data_epoch; +-- +goose StatementEnd +-- +goose StatementBegin +DROP TABLE IF EXISTS _unsafe_validator_dashboard_data_epoch; +-- +goose StatementEnd +-- +goose StatementBegin +DROP TABLE IF EXISTS _insert_sink_validator_dashboard_data_epoch; +-- +goose StatementEnd +-- +goose StatementBegin +DROP TABLE IF EXISTS _final_validator_dashboard_data_epoch; +-- +goose StatementEnd diff --git a/backend/pkg/commons/db/migrations/clickhouse/20240702103057_add_materialized_views.sql b/backend/pkg/commons/db/migrations/clickhouse/20240702103057_add_materialized_views.sql new file mode 100644 index 000000000..85c1e8c8b --- /dev/null +++ b/backend/pkg/commons/db/migrations/clickhouse/20240702103057_add_materialized_views.sql @@ -0,0 +1,286 @@ +-- +goose Up +-- materialized views +-- +goose StatementBegin +CREATE MATERIALIZED VIEW IF NOT EXISTS _mv_final_validator_dashboard_data_hourly TO _final_validator_dashboard_data_hourly +AS SELECT + validator_index AS validator_index, + toStartOfHour(epoch_timestamp) AS t, + groupArraySortedIfState(2048)(-foo.epoch, validator_index = 0) AS epoch_map, + min(foo.epoch) AS epoch_start, + max(foo.epoch) AS epoch_end, + argMinState(foo.balance_start, foo.epoch) AS balance_start, + argMaxState(foo.balance_end, foo.epoch) AS balance_end, + least(min(foo.balance_start), min(foo.balance_end)) AS balance_min, + greatest(max(foo.balance_start), max(foo.balance_end)) AS balance_max, + sum(deposits_count) AS deposits_count, + sum(deposits_amount) AS deposits_amount, + sum(withdrawals_count) AS withdrawals_count, + sum(withdrawals_amount) AS withdrawals_amount, + sum(attestations_scheduled) AS attestations_scheduled, + sum(attestations_observed) AS attestations_observed, + sum(attestations_head_matched) AS attestations_head_matched, + sum(attestations_source_matched) AS attestations_source_matched, + sum(attestations_target_matched) AS attestations_target_matched, + sum(attestations_head_executed) AS attestations_head_executed, + sum(attestations_source_executed) AS attestations_source_executed, + sum(attestations_target_executed) AS attestations_target_executed, + sum(attestations_head_reward_rewards_only) AS attestations_head_reward_rewards_only, + sum(attestations_head_reward_penalties_only) AS attestations_head_reward_penalties_only, + sum(attestations_source_reward_rewards_only) AS attestations_source_reward_rewards_only, + sum(attestations_source_reward_penalties_only) AS attestations_source_reward_penalties_only, + sum(attestations_target_reward_rewards_only) AS attestations_target_reward_rewards_only, + sum(attestations_target_reward_penalties_only) AS attestations_target_reward_penalties_only, + sum(attestations_inclusion_reward_rewards_only) AS attestations_inclusion_reward_rewards_only, + sum(attestations_inclusion_reward_penalties_only) AS attestations_inclusion_reward_penalties_only, + sum(attestations_inactivity_reward_rewards_only) as attestations_inactivity_reward_rewards_only, + sum(attestations_inactivity_reward_penalties_only) as attestations_inactivity_reward_penalties_only, + sum(attestations_ideal_head_reward) AS attestations_ideal_head_reward, + sum(attestations_ideal_source_reward) AS attestations_ideal_source_reward, + sum(attestations_ideal_target_reward) AS attestations_ideal_target_reward, + sum(attestations_ideal_inclusion_reward) AS attestations_ideal_inclusion_reward, + sum(attestations_ideal_inactivity_reward) AS attestations_ideal_inactivity_reward, + sum(attestations_localized_max_reward) AS attestations_localized_max_reward, + sum(attestations_hyperlocalized_max_reward) AS attestations_hyperlocalized_max_reward, + sum(inclusion_delay_sum) AS inclusion_delay_sum, + sum(optimal_inclusion_delay_sum) AS optimal_inclusion_delay_sum, + sum(blocks_scheduled) AS blocks_scheduled, + sum(blocks_proposed) AS blocks_proposed, + sum(blocks_cl_reward) AS blocks_cl_reward, + sum(blocks_cl_attestations_reward) AS blocks_cl_attestations_reward, + sum(blocks_cl_sync_aggregate_reward) AS blocks_cl_sync_aggregate_reward, + sum(blocks_cl_slasher_reward) AS blocks_cl_slasher_reward, + sum(blocks_cl_missed_median_reward) AS blocks_cl_missed_median_reward, + sum(blocks_slashing_count) AS blocks_slashing_count, + sum(blocks_expected) AS blocks_expected, + sum(sync_scheduled) AS sync_scheduled, + sum(sync_executed) AS sync_executed, + sum(sync_reward_rewards_only) AS sync_reward_rewards_only, + sum(sync_reward_penalties_only) AS sync_reward_penalties_only, + sum(sync_localized_max_reward) AS sync_localized_max_reward, + sum(sync_committees_expected) AS sync_committees_expected, + max(slashed) AS slashed, + maxIfOrNull(foo.epoch, (foo.blocks_proposed != 0) OR (foo.sync_executed != 0) OR (foo.attestations_observed != 0)) AS last_executed_duty_epoch, + maxIfOrNull(foo.epoch, foo.sync_scheduled != 0) AS last_scheduled_sync_epoch, + maxIfOrNull(foo.epoch, foo.blocks_proposed != 0) AS last_scheduled_block_epoch +FROM _final_validator_dashboard_data_epoch AS foo +GROUP BY + t, + validator_index +-- +goose StatementEnd +-- +goose StatementBegin +-- hour => day. need to merge already merged states +CREATE MATERIALIZED VIEW IF NOT EXISTS _mv_final_validator_dashboard_data_daily TO _final_validator_dashboard_data_daily +AS SELECT + validator_index AS validator_index, + toStartOfDay(foo.t) AS t, + groupArraySortedIfMergeState(2048)(epoch_map) AS epoch_map, + min(epoch_start) AS epoch_start, + max(epoch_end) AS epoch_end, + argMinStateMerge(balance_start) AS balance_start, + argMaxStateMerge(balance_end) AS balance_end, + min(balance_min) AS balance_min, + max(balance_max) AS balance_max, + sum(deposits_count) AS deposits_count, + sum(deposits_amount) AS deposits_amount, + sum(withdrawals_count) AS withdrawals_count, + sum(withdrawals_amount) AS withdrawals_amount, + sum(attestations_scheduled) AS attestations_scheduled, + sum(attestations_observed) AS attestations_observed, + sum(attestations_head_matched) AS attestations_head_matched, + sum(attestations_source_matched) AS attestations_source_matched, + sum(attestations_target_matched) AS attestations_target_matched, + sum(attestations_head_executed) AS attestations_head_executed, + sum(attestations_source_executed) AS attestations_source_executed, + sum(attestations_target_executed) AS attestations_target_executed, + sum(attestations_head_reward_rewards_only) AS attestations_head_reward_rewards_only, + sum(attestations_head_reward_penalties_only) AS attestations_head_reward_penalties_only, + sum(attestations_source_reward_rewards_only) AS attestations_source_reward_rewards_only, + sum(attestations_source_reward_penalties_only) AS attestations_source_reward_penalties_only, + sum(attestations_target_reward_rewards_only) AS attestations_target_reward_rewards_only, + sum(attestations_target_reward_penalties_only) AS attestations_target_reward_penalties_only, + sum(attestations_inclusion_reward_rewards_only) AS attestations_inclusion_reward_rewards_only, + sum(attestations_inclusion_reward_penalties_only) AS attestations_inclusion_reward_penalties_only, + sum(attestations_inactivity_reward_rewards_only) as attestations_inactivity_reward_rewards_only, + sum(attestations_inactivity_reward_penalties_only) as attestations_inactivity_reward_penalties_only, + sum(attestations_ideal_head_reward) AS attestations_ideal_head_reward, + sum(attestations_ideal_source_reward) AS attestations_ideal_source_reward, + sum(attestations_ideal_target_reward) AS attestations_ideal_target_reward, + sum(attestations_ideal_inclusion_reward) AS attestations_ideal_inclusion_reward, + sum(attestations_ideal_inactivity_reward) AS attestations_ideal_inactivity_reward, + sum(attestations_localized_max_reward) AS attestations_localized_max_reward, + sum(attestations_hyperlocalized_max_reward) AS attestations_hyperlocalized_max_reward, + sum(inclusion_delay_sum) AS inclusion_delay_sum, + sum(optimal_inclusion_delay_sum) AS optimal_inclusion_delay_sum, + sum(blocks_scheduled) AS blocks_scheduled, + sum(blocks_proposed) AS blocks_proposed, + sum(blocks_cl_reward) AS blocks_cl_reward, + sum(blocks_cl_attestations_reward) AS blocks_cl_attestations_reward, + sum(blocks_cl_sync_aggregate_reward) AS blocks_cl_sync_aggregate_reward, + sum(blocks_cl_slasher_reward) AS blocks_cl_slasher_reward, + sum(blocks_cl_missed_median_reward) AS blocks_cl_missed_median_reward, + sum(blocks_slashing_count) AS blocks_slashing_count, + sum(blocks_expected) AS blocks_expected, + sum(sync_scheduled) AS sync_scheduled, + sum(sync_executed) AS sync_executed, + sum(sync_reward_rewards_only) AS sync_reward_rewards_only, + sum(sync_reward_penalties_only) AS sync_reward_penalties_only, + sum(sync_localized_max_reward) AS sync_localized_max_reward, + sum(sync_committees_expected) AS sync_committees_expected, + max(slashed) AS slashed, + max(last_executed_duty_epoch) AS last_executed_duty_epoch, + max(last_scheduled_sync_epoch) AS last_scheduled_sync_epoch, + max(last_scheduled_block_epoch) AS last_scheduled_block_epoch +FROM _final_validator_dashboard_data_hourly AS foo +GROUP BY + t, + validator_index +-- +goose StatementEnd +-- +goose StatementBegin +-- day => week. same as the daily view, but with a different grouping +CREATE MATERIALIZED VIEW IF NOT EXISTS _mv_final_validator_dashboard_data_weekly TO _final_validator_dashboard_data_weekly +AS SELECT + validator_index AS validator_index, + toMonday(foo.t) AS t, + groupArraySortedIfMergeState(2048)(epoch_map) AS epoch_map, + min(epoch_start) AS epoch_start, + max(epoch_end) AS epoch_end, + argMinStateMerge(balance_start) AS balance_start, + argMaxStateMerge(balance_end) AS balance_end, + min(balance_min) AS balance_min, + max(balance_max) AS balance_max, + sum(deposits_count) AS deposits_count, + sum(deposits_amount) AS deposits_amount, + sum(withdrawals_count) AS withdrawals_count, + sum(withdrawals_amount) AS withdrawals_amount, + sum(attestations_scheduled) AS attestations_scheduled, + sum(attestations_observed) AS attestations_observed, + sum(attestations_head_matched) AS attestations_head_matched, + sum(attestations_source_matched) AS attestations_source_matched, + sum(attestations_target_matched) AS attestations_target_matched, + sum(attestations_head_executed) AS attestations_head_executed, + sum(attestations_source_executed) AS attestations_source_executed, + sum(attestations_target_executed) AS attestations_target_executed, + sum(attestations_head_reward_rewards_only) AS attestations_head_reward_rewards_only, + sum(attestations_head_reward_penalties_only) AS attestations_head_reward_penalties_only, + sum(attestations_source_reward_rewards_only) AS attestations_source_reward_rewards_only, + sum(attestations_source_reward_penalties_only) AS attestations_source_reward_penalties_only, + sum(attestations_target_reward_rewards_only) AS attestations_target_reward_rewards_only, + sum(attestations_target_reward_penalties_only) AS attestations_target_reward_penalties_only, + sum(attestations_inclusion_reward_rewards_only) AS attestations_inclusion_reward_rewards_only, + sum(attestations_inclusion_reward_penalties_only) AS attestations_inclusion_reward_penalties_only, + sum(attestations_inactivity_reward_rewards_only) as attestations_inactivity_reward_rewards_only, + sum(attestations_inactivity_reward_penalties_only) as attestations_inactivity_reward_penalties_only, + sum(attestations_ideal_head_reward) AS attestations_ideal_head_reward, + sum(attestations_ideal_source_reward) AS attestations_ideal_source_reward, + sum(attestations_ideal_target_reward) AS attestations_ideal_target_reward, + sum(attestations_ideal_inclusion_reward) AS attestations_ideal_inclusion_reward, + sum(attestations_ideal_inactivity_reward) AS attestations_ideal_inactivity_reward, + sum(attestations_localized_max_reward) AS attestations_localized_max_reward, + sum(attestations_hyperlocalized_max_reward) AS attestations_hyperlocalized_max_reward, + sum(inclusion_delay_sum) AS inclusion_delay_sum, + sum(optimal_inclusion_delay_sum) AS optimal_inclusion_delay_sum, + sum(blocks_scheduled) AS blocks_scheduled, + sum(blocks_proposed) AS blocks_proposed, + sum(blocks_cl_reward) AS blocks_cl_reward, + sum(blocks_cl_attestations_reward) AS blocks_cl_attestations_reward, + sum(blocks_cl_sync_aggregate_reward) AS blocks_cl_sync_aggregate_reward, + sum(blocks_cl_slasher_reward) AS blocks_cl_slasher_reward, + sum(blocks_cl_missed_median_reward) AS blocks_cl_missed_median_reward, + sum(blocks_slashing_count) AS blocks_slashing_count, + sum(blocks_expected) AS blocks_expected, + sum(sync_scheduled) AS sync_scheduled, + sum(sync_executed) AS sync_executed, + sum(sync_reward_rewards_only) AS sync_reward_rewards_only, + sum(sync_reward_penalties_only) AS sync_reward_penalties_only, + sum(sync_localized_max_reward) AS sync_localized_max_reward, + sum(sync_committees_expected) AS sync_committees_expected, + max(slashed) AS slashed, + max(last_executed_duty_epoch) AS last_executed_duty_epoch, + max(last_scheduled_sync_epoch) AS last_scheduled_sync_epoch, + max(last_scheduled_block_epoch) AS last_scheduled_block_epoch +FROM _final_validator_dashboard_data_daily AS foo +GROUP BY + t, + validator_index +-- +goose StatementEnd +-- +goose StatementBegin +-- day => month. same as the daily view, but with a different grouping +CREATE MATERIALIZED VIEW IF NOT EXISTS _mv_final_validator_dashboard_data_monthly TO _final_validator_dashboard_data_monthly +AS SELECT + validator_index AS validator_index, + toStartOfMonth(foo.t) AS t, + groupArraySortedIfMergeState(2048)(epoch_map) AS epoch_map, + min(epoch_start) AS epoch_start, + max(epoch_end) AS epoch_end, + argMinStateMerge(balance_start) AS balance_start, + argMaxStateMerge(balance_end) AS balance_end, + min(balance_min) AS balance_min, + max(balance_max) AS balance_max, + sum(deposits_count) AS deposits_count, + sum(deposits_amount) AS deposits_amount, + sum(withdrawals_count) AS withdrawals_count, + sum(withdrawals_amount) AS withdrawals_amount, + sum(attestations_scheduled) AS attestations_scheduled, + sum(attestations_observed) AS attestations_observed, + sum(attestations_head_matched) AS attestations_head_matched, + sum(attestations_source_matched) AS attestations_source_matched, + sum(attestations_target_matched) AS attestations_target_matched, + sum(attestations_head_executed) AS attestations_head_executed, + sum(attestations_source_executed) AS attestations_source_executed, + sum(attestations_target_executed) AS attestations_target_executed, + sum(attestations_head_reward_rewards_only) AS attestations_head_reward_rewards_only, + sum(attestations_head_reward_penalties_only) AS attestations_head_reward_penalties_only, + sum(attestations_source_reward_rewards_only) AS attestations_source_reward_rewards_only, + sum(attestations_source_reward_penalties_only) AS attestations_source_reward_penalties_only, + sum(attestations_target_reward_rewards_only) AS attestations_target_reward_rewards_only, + sum(attestations_target_reward_penalties_only) AS attestations_target_reward_penalties_only, + sum(attestations_inclusion_reward_rewards_only) AS attestations_inclusion_reward_rewards_only, + sum(attestations_inclusion_reward_penalties_only) AS attestations_inclusion_reward_penalties_only, + sum(attestations_inactivity_reward_rewards_only) as attestations_inactivity_reward_rewards_only, + sum(attestations_inactivity_reward_penalties_only) as attestations_inactivity_reward_penalties_only, + sum(attestations_ideal_head_reward) AS attestations_ideal_head_reward, + sum(attestations_ideal_source_reward) AS attestations_ideal_source_reward, + sum(attestations_ideal_target_reward) AS attestations_ideal_target_reward, + sum(attestations_ideal_inclusion_reward) AS attestations_ideal_inclusion_reward, + sum(attestations_ideal_inactivity_reward) AS attestations_ideal_inactivity_reward, + sum(attestations_localized_max_reward) AS attestations_localized_max_reward, + sum(attestations_hyperlocalized_max_reward) AS attestations_hyperlocalized_max_reward, + sum(inclusion_delay_sum) AS inclusion_delay_sum, + sum(optimal_inclusion_delay_sum) AS optimal_inclusion_delay_sum, + sum(blocks_scheduled) AS blocks_scheduled, + sum(blocks_proposed) AS blocks_proposed, + sum(blocks_cl_reward) AS blocks_cl_reward, + sum(blocks_cl_attestations_reward) AS blocks_cl_attestations_reward, + sum(blocks_cl_sync_aggregate_reward) AS blocks_cl_sync_aggregate_reward, + sum(blocks_cl_slasher_reward) AS blocks_cl_slasher_reward, + sum(blocks_cl_missed_median_reward) AS blocks_cl_missed_median_reward, + sum(blocks_slashing_count) AS blocks_slashing_count, + sum(blocks_expected) AS blocks_expected, + sum(sync_scheduled) AS sync_scheduled, + sum(sync_executed) AS sync_executed, + sum(sync_reward_rewards_only) AS sync_reward_rewards_only, + sum(sync_reward_penalties_only) AS sync_reward_penalties_only, + sum(sync_localized_max_reward) AS sync_localized_max_reward, + sum(sync_committees_expected) AS sync_committees_expected, + max(slashed) AS slashed, + max(last_executed_duty_epoch) AS last_executed_duty_epoch, + max(last_scheduled_sync_epoch) AS last_scheduled_sync_epoch, + max(last_scheduled_block_epoch) AS last_scheduled_block_epoch +FROM _final_validator_dashboard_data_daily AS foo +GROUP BY + t, + validator_index +-- +goose StatementEnd +-- +goose Down +-- +goose StatementBegin +DROP MATERIALIZED VIEW IF EXISTS _final_validator_dashboard_data_hourly_mv +-- +goose StatementEnd +-- +goose StatementBegin +DROP MATERIALIZED VIEW IF EXISTS _final_validator_dashboard_data_daily_mv +-- +goose StatementEnd +-- +goose StatementBegin +DROP MATERIALIZED VIEW IF EXISTS _final_validator_dashboard_data_weekly_mv +-- +goose StatementEnd +-- +goose StatementBegin +DROP MATERIALIZED VIEW IF EXISTS _final_validator_dashboard_data_monthly_mv +-- +goose StatementEnd diff --git a/backend/pkg/commons/db/migrations/clickhouse/20240827153322_views.sql b/backend/pkg/commons/db/migrations/clickhouse/20240827153322_views.sql new file mode 100644 index 000000000..aa307fbd3 --- /dev/null +++ b/backend/pkg/commons/db/migrations/clickhouse/20240827153322_views.sql @@ -0,0 +1,55 @@ +-- +goose Up +-- +goose StatementBegin +create or replace view validator_dashboard_data_epoch as + select * + from _final_validator_dashboard_data_epoch +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_hourly as + select * + from _final_validator_dashboard_data_hourly +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_daily as + select * + from _final_validator_dashboard_data_daily +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_weekly as + select * + from _final_validator_dashboard_data_weekly +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_monthly as + select * + from _final_validator_dashboard_data_monthly +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_epoch_max_ts as + select max(epoch_timestamp) as t + from _final_validator_dashboard_data_epoch +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_hourly_max_ts as + select max(t) as t + from _final_validator_dashboard_data_hourly +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_daily_max_ts as + select max(t) as t + from _final_validator_dashboard_data_daily +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_weekly_max_ts as + select max(t) as t + from _final_validator_dashboard_data_weekly +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_monthly_max_ts as + select max(t) as t + from _final_validator_dashboard_data_monthly +-- +goose StatementEnd +-- +goose Down +-- +goose StatementBegin + +-- +goose StatementEnd diff --git a/backend/pkg/commons/db/migrations/clickhouse/20241019135256_add_rollings.sql b/backend/pkg/commons/db/migrations/clickhouse/20241019135256_add_rollings.sql new file mode 100644 index 000000000..cd632b273 --- /dev/null +++ b/backend/pkg/commons/db/migrations/clickhouse/20241019135256_add_rollings.sql @@ -0,0 +1,78 @@ +-- +goose Up +-- +goose StatementBegin +-- hourly +create table if not exists _final_validator_dashboard_rolling_1h as _final_validator_dashboard_data_hourly ENGINE AggregatingMergeTree ORDER BY (validator_index) +-- +goose StatementEnd +-- +goose StatementBegin +ALTER table _final_validator_dashboard_rolling_1h MODIFY COLUMN t SimpleAggregateFunction(max, DateTime) +-- +goose StatementEnd +-- +goose StatementBegin +create table if not exists _unsafe_validator_dashboard_rolling_1h as _final_validator_dashboard_rolling_1h ENGINE AggregatingMergeTree ORDER BY (validator_index) +-- +goose StatementEnd +-- +goose StatementBegin +create table if not exists _final_validator_dashboard_rolling_24h as _final_validator_dashboard_rolling_1h ENGINE AggregatingMergeTree ORDER BY (validator_index) +-- +goose StatementEnd +-- +goose StatementBegin +create table if not exists _unsafe_validator_dashboard_rolling_24h as _final_validator_dashboard_rolling_1h ENGINE AggregatingMergeTree ORDER BY (validator_index) +-- +goose StatementEnd +-- +goose StatementBegin +create table if not exists _final_validator_dashboard_rolling_7d as _final_validator_dashboard_rolling_1h ENGINE AggregatingMergeTree ORDER BY (validator_index) +-- +goose StatementEnd +-- +goose StatementBegin +create table if not exists _unsafe_validator_dashboard_rolling_7d as _final_validator_dashboard_rolling_1h ENGINE AggregatingMergeTree ORDER BY (validator_index) +-- +goose StatementEnd +-- +goose StatementBegin +create table if not exists _final_validator_dashboard_rolling_30d as _final_validator_dashboard_rolling_1h ENGINE AggregatingMergeTree ORDER BY (validator_index) +-- +goose StatementEnd +-- +goose StatementBegin +create table if not exists _unsafe_validator_dashboard_rolling_30d as _final_validator_dashboard_rolling_1h ENGINE AggregatingMergeTree ORDER BY (validator_index) +-- +goose StatementEnd +-- +goose StatementBegin +create table if not exists _final_validator_dashboard_rolling_90d as _final_validator_dashboard_rolling_1h ENGINE AggregatingMergeTree ORDER BY (validator_index) +-- +goose StatementEnd +-- +goose StatementBegin +create table if not exists _unsafe_validator_dashboard_rolling_90d as _final_validator_dashboard_rolling_1h ENGINE AggregatingMergeTree ORDER BY (validator_index) +-- +goose StatementEnd +-- +goose StatementBegin +create table if not exists _final_validator_dashboard_rolling_total as _final_validator_dashboard_rolling_1h ENGINE AggregatingMergeTree ORDER BY (validator_index) +-- +goose StatementEnd +-- +goose StatementBegin +create table if not exists _unsafe_validator_dashboard_rolling_total as _final_validator_dashboard_rolling_1h ENGINE AggregatingMergeTree ORDER BY (validator_index) +-- +goose StatementEnd +-- +goose Down +-- +goose StatementBegin +drop table if exists _final_validator_dashboard_rolling_1h +-- +goose StatementEnd +-- +goose StatementBegin +drop table if exists _unsafe_validator_dashboard_rolling_1h +-- +goose StatementEnd +-- +goose StatementBegin +drop table if exists _final_validator_dashboard_rolling_24h +-- +goose StatementEnd +-- +goose StatementBegin +drop table if exists _unsafe_validator_dashboard_rolling_24h +-- +goose StatementEnd +-- +goose StatementBegin +drop table if exists _final_validator_dashboard_rolling_7d +-- +goose StatementEnd +-- +goose StatementBegin +drop table if exists _unsafe_validator_dashboard_rolling_7d +-- +goose StatementEnd +-- +goose StatementBegin +drop table if exists _final_validator_dashboard_rolling_30d +-- +goose StatementEnd +-- +goose StatementBegin +drop table if exists _unsafe_validator_dashboard_rolling_30d +-- +goose StatementEnd +-- +goose StatementBegin +drop table if exists _final_validator_dashboard_rolling_90d +-- +goose StatementEnd +-- +goose StatementBegin +drop table if exists _unsafe_validator_dashboard_rolling_90d +-- +goose StatementEnd +-- +goose StatementBegin +drop table if exists _final_validator_dashboard_rolling_total +-- +goose StatementEnd +-- +goose StatementBegin +drop table if exists _unsafe_validator_dashboard_rolling_total +-- +goose StatementEnd diff --git a/backend/pkg/commons/db/migrations/clickhouse/20241203144505_fix_flaky_primary_key_index.sql b/backend/pkg/commons/db/migrations/clickhouse/20241203144505_fix_flaky_primary_key_index.sql new file mode 100644 index 000000000..4005517b1 --- /dev/null +++ b/backend/pkg/commons/db/migrations/clickhouse/20241203144505_fix_flaky_primary_key_index.sql @@ -0,0 +1,104 @@ +-- +goose Up + +-- +goose StatementBegin +ALTER TABLE _final_validator_dashboard_data_epoch ADD INDEX IF NOT EXISTS _index_epoch_timestamp_min_max epoch_timestamp type minmax() granularity 1 SETTINGS alter_sync = 1; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _final_validator_dashboard_data_epoch MATERIALIZE INDEX _index_epoch_timestamp_min_max; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _unsafe_validator_dashboard_data_epoch ADD INDEX IF NOT EXISTS _index_epoch_timestamp_min_max epoch_timestamp type minmax() granularity 1 SETTINGS alter_sync = 1; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _unsafe_validator_dashboard_data_epoch MATERIALIZE INDEX _index_epoch_timestamp_min_max; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _final_validator_dashboard_data_hourly ADD INDEX IF NOT EXISTS _index_t_min_max t type minmax() granularity 1 SETTINGS alter_sync = 1; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _final_validator_dashboard_data_hourly MATERIALIZE INDEX _index_t_min_max; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _final_validator_dashboard_data_daily ADD INDEX IF NOT EXISTS _index_t_min_max t type minmax() granularity 1 SETTINGS alter_sync = 1; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _final_validator_dashboard_data_daily MATERIALIZE INDEX _index_t_min_max; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _final_validator_dashboard_data_weekly ADD INDEX IF NOT EXISTS _index_t_min_max t type minmax() granularity 1 SETTINGS alter_sync = 1; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _final_validator_dashboard_data_weekly MATERIALIZE INDEX _index_t_min_max; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _final_validator_dashboard_data_monthly ADD INDEX IF NOT EXISTS _index_t_min_max t type minmax() granularity 1 SETTINGS alter_sync = 1; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _final_validator_dashboard_data_monthly MATERIALIZE INDEX _index_t_min_max; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE validator_attestation_assignments_slot ADD INDEX IF NOT EXISTS _index_epoch_timestamp_min_max epoch_timestamp type minmax() granularity 1 SETTINGS alter_sync = 1; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE validator_attestation_assignments_slot MATERIALIZE INDEX _index_epoch_timestamp_min_max; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE validator_proposal_assignments_slot ADD INDEX IF NOT EXISTS _index_epoch_timestamp_min_max epoch_timestamp type minmax() granularity 1 SETTINGS alter_sync = 1; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE validator_proposal_assignments_slot MATERIALIZE INDEX _index_epoch_timestamp_min_max; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE validator_proposal_rewards_slot ADD INDEX IF NOT EXISTS _index_epoch_timestamp_min_max epoch_timestamp type minmax() granularity 1 SETTINGS alter_sync = 1; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE validator_proposal_rewards_slot MATERIALIZE INDEX _index_epoch_timestamp_min_max; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE validator_sync_committee_rewards_slot ADD INDEX IF NOT EXISTS _index_epoch_timestamp_min_max epoch_timestamp type minmax() granularity 1 SETTINGS alter_sync = 1; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE validator_sync_committee_rewards_slot MATERIALIZE INDEX _index_epoch_timestamp_min_max; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE validator_sync_committee_votes_slot ADD INDEX IF NOT EXISTS _index_epoch_timestamp_min_max epoch_timestamp type minmax() granularity 1 SETTINGS alter_sync = 1; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE validator_sync_committee_votes_slot MATERIALIZE INDEX _index_epoch_timestamp_min_max; +-- +goose StatementEnd + +-- +goose Down + +-- +goose StatementBegin +ALTER TABLE _final_validator_dashboard_data_epoch DROP INDEX _index_epoch_timestamp_min_max; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _unsafe_validator_dashboard_data_epoch DROP INDEX _index_epoch_timestamp_min_max; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _final_validator_dashboard_data_hourly DROP INDEX _index_t_min_max; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _final_validator_dashboard_data_daily DROP INDEX _index_t_min_max; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _final_validator_dashboard_data_weekly DROP INDEX _index_t_min_max; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _final_validator_dashboard_data_monthly DROP INDEX _index_t_min_max; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _final_validator_attestation_assignments_slot DROP INDEX _index_epoch_timestamp_min_max; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _final_validator_proposal_assignments_slot DROP INDEX _index_epoch_timestamp_min_max; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _final_validator_proposal_rewards_slot DROP INDEX _index_epoch_timestamp_min_max; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _final_validator_sync_committee_rewards_slot DROP INDEX _index_epoch_timestamp_min_max; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _final_validator_sync_committee_votes_slot DROP INDEX _index_epoch_timestamp_min_max; +-- +goose StatementEnd diff --git a/backend/pkg/commons/db/migrations/clickhouse/20241203175153_add_missing_views.sql b/backend/pkg/commons/db/migrations/clickhouse/20241203175153_add_missing_views.sql new file mode 100644 index 000000000..25999905e --- /dev/null +++ b/backend/pkg/commons/db/migrations/clickhouse/20241203175153_add_missing_views.sql @@ -0,0 +1,51 @@ +-- +goose Up +-- +goose StatementBegin +create or replace view validator_dashboard_rolling_1h as + select * + from _final_validator_dashboard_rolling_1h +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_rolling_24h as + select * + from _final_validator_dashboard_rolling_24h +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_rolling_7d as + select * + from _final_validator_dashboard_rolling_7d +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_rolling_30d as + select * + from _final_validator_dashboard_rolling_30d +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_rolling_90d as + select * + from _final_validator_dashboard_rolling_90d +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_rolling_total as + select * + from _final_validator_dashboard_rolling_total +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +drop view if exists validator_dashboard_rolling_1h +-- +goose StatementEnd +-- +goose StatementBegin +drop view if exists validator_dashboard_rolling_24h +-- +goose StatementEnd +-- +goose StatementBegin +drop view if exists validator_dashboard_rolling_7d +-- +goose StatementEnd +-- +goose StatementBegin +drop view if exists validator_dashboard_rolling_30d +-- +goose StatementEnd +-- +goose StatementBegin +drop view if exists validator_dashboard_rolling_90d +-- +goose StatementEnd +-- +goose StatementBegin +drop view if exists validator_dashboard_rolling_total +-- +goose StatementEnd diff --git a/backend/pkg/commons/db/migrations/clickhouse/20241203175334_add_attestation_reward_sum_column.sql b/backend/pkg/commons/db/migrations/clickhouse/20241203175334_add_attestation_reward_sum_column.sql new file mode 100644 index 000000000..e575bcfe0 --- /dev/null +++ b/backend/pkg/commons/db/migrations/clickhouse/20241203175334_add_attestation_reward_sum_column.sql @@ -0,0 +1,337 @@ +-- +goose Up +-- +goose StatementBegin +create or replace view validator_dashboard_rolling_1h as + select *, ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward + from _final_validator_dashboard_rolling_1h +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_rolling_24h as + select *, ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward + from _final_validator_dashboard_rolling_24h +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_rolling_7d as + select *, ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward + from _final_validator_dashboard_rolling_7d + +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_rolling_30d as + select *, ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward + from _final_validator_dashboard_rolling_30d +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_rolling_90d as + select *, ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward + from _final_validator_dashboard_rolling_90d +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_rolling_total as + select *, ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward + from _final_validator_dashboard_rolling_total +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_epoch as + select *, ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward + from _final_validator_dashboard_data_epoch +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_hourly as + select *, ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward + from _final_validator_dashboard_data_hourly +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_daily as + select *, ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward + from _final_validator_dashboard_data_daily +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_weekly as + select *, ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward + from _final_validator_dashboard_data_weekly +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_monthly as + select *, ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward + from _final_validator_dashboard_data_monthly +-- +goose StatementEnd + + +-- +goose Down + + +-- +goose StatementBegin +create or replace view validator_dashboard_rolling_1h as + select * + from _final_validator_dashboard_rolling_1h +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_rolling_24h as + select * + from _final_validator_dashboard_rolling_24h +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_rolling_7d as + select * + from _final_validator_dashboard_rolling_7d +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_rolling_30d as + select * + from _final_validator_dashboard_rolling_30d +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_rolling_90d as + select * + from _final_validator_dashboard_rolling_90d +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_rolling_total as + select * + from _final_validator_dashboard_rolling_total +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_epoch as + select * + from _final_validator_dashboard_data_epoch +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_hourly as + select * + from _final_validator_dashboard_data_hourly +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_daily as + select * + from _final_validator_dashboard_data_daily +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_weekly as + select * + from _final_validator_dashboard_data_weekly +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_monthly as + select * + from _final_validator_dashboard_data_monthly +-- +goose StatementEnd \ No newline at end of file diff --git a/backend/pkg/commons/db/migrations/clickhouse/20241204121509_rename_rolling_views.sql b/backend/pkg/commons/db/migrations/clickhouse/20241204121509_rename_rolling_views.sql new file mode 100644 index 000000000..094fbd6b2 --- /dev/null +++ b/backend/pkg/commons/db/migrations/clickhouse/20241204121509_rename_rolling_views.sql @@ -0,0 +1,43 @@ +-- +goose Up + +-- +goose StatementBegin +-- +goose StatementEnd +-- +goose StatementBegin +RENAME TABLE validator_dashboard_rolling_1h TO validator_dashboard_data_rolling_1h +-- +goose StatementEnd +-- +goose StatementBegin +RENAME TABLE validator_dashboard_rolling_24h TO validator_dashboard_data_rolling_24h +-- +goose StatementEnd +-- +goose StatementBegin +RENAME TABLE validator_dashboard_rolling_7d TO validator_dashboard_data_rolling_7d +-- +goose StatementEnd +-- +goose StatementBegin +RENAME TABLE validator_dashboard_rolling_30d TO validator_dashboard_data_rolling_30d +-- +goose StatementEnd +-- +goose StatementBegin +RENAME TABLE validator_dashboard_rolling_90d TO validator_dashboard_data_rolling_90d +-- +goose StatementEnd +-- +goose StatementBegin +RENAME TABLE validator_dashboard_rolling_total TO validator_dashboard_data_rolling_total +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +RENAME TABLE validator_dashboard_data_rolling_1h TO validator_dashboard_rolling_1h +-- +goose StatementEnd +-- +goose StatementBegin +RENAME TABLE validator_dashboard_data_rolling_24h TO validator_dashboard_rolling_24h +-- +goose StatementEnd +-- +goose StatementBegin +RENAME TABLE validator_dashboard_data_rolling_7d TO validator_dashboard_rolling_7d +-- +goose StatementEnd +-- +goose StatementBegin +RENAME TABLE validator_dashboard_data_rolling_30d TO validator_dashboard_rolling_30d +-- +goose StatementEnd +-- +goose StatementBegin +RENAME TABLE validator_dashboard_data_rolling_90d TO validator_dashboard_rolling_90d +-- +goose StatementEnd +-- +goose StatementBegin +RENAME TABLE validator_dashboard_data_rolling_total TO validator_dashboard_rolling_total +-- +goose StatementEnd + diff --git a/backend/pkg/commons/db/migrations/clickhouse/20241206114911_add_more_shortcuts_to_view.sql b/backend/pkg/commons/db/migrations/clickhouse/20241206114911_add_more_shortcuts_to_view.sql new file mode 100644 index 000000000..95b4a6809 --- /dev/null +++ b/backend/pkg/commons/db/migrations/clickhouse/20241206114911_add_more_shortcuts_to_view.sql @@ -0,0 +1,645 @@ +-- +goose Up +-- +goose StatementBegin +create or replace view validator_dashboard_data_rolling_1h as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward_penalties_only + attestations_reward_rewards_only as attestations_reward, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_rolling_1h +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_rolling_24h as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward_penalties_only + attestations_reward_rewards_only as attestations_reward, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_rolling_24h +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_rolling_7d as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward_penalties_only + attestations_reward_rewards_only as attestations_reward, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_rolling_7d + +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_rolling_30d as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward_penalties_only + attestations_reward_rewards_only as attestations_reward, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_rolling_30d +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_rolling_90d as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward_penalties_only + attestations_reward_rewards_only as attestations_reward, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_rolling_90d +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_rolling_total as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward_penalties_only + attestations_reward_rewards_only as attestations_reward, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_rolling_total +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_epoch as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward_penalties_only + attestations_reward_rewards_only as attestations_reward, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward + from _final_validator_dashboard_data_epoch +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_hourly as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward_penalties_only + attestations_reward_rewards_only as attestations_reward, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_data_hourly +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_daily as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward_penalties_only + attestations_reward_rewards_only as attestations_reward, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_data_daily +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_weekly as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward_penalties_only + attestations_reward_rewards_only as attestations_reward, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_data_weekly +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_monthly as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward_penalties_only + attestations_reward_rewards_only as attestations_reward, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_data_monthly +-- +goose StatementEnd + + +-- +goose Down + + +-- +goose StatementBegin +create or replace view validator_dashboard_data_rolling_1h as + select *, ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward + from _final_validator_dashboard_rolling_1h +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_rolling_24h as + select *, ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward + from _final_validator_dashboard_rolling_24h +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_rolling_7d as + select *, ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward + from _final_validator_dashboard_rolling_7d + +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_rolling_30d as + select *, ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward + from _final_validator_dashboard_rolling_30d +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_rolling_90d as + select *, ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward + from _final_validator_dashboard_rolling_90d +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_rolling_total as + select *, ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward + from _final_validator_dashboard_rolling_total +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_epoch as + select *, ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward + from _final_validator_dashboard_data_epoch +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_hourly as + select *, ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward + from _final_validator_dashboard_data_hourly +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_daily as + select *, ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward + from _final_validator_dashboard_data_daily +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_weekly as + select *, ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward + from _final_validator_dashboard_data_weekly +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_monthly as + select *, ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward + from _final_validator_dashboard_data_monthly +-- +goose StatementEnd \ No newline at end of file diff --git a/backend/pkg/commons/db/migrations/clickhouse/20241210181608_add_ttl_to_unsafe.sql b/backend/pkg/commons/db/migrations/clickhouse/20241210181608_add_ttl_to_unsafe.sql new file mode 100644 index 000000000..01fc87c7b --- /dev/null +++ b/backend/pkg/commons/db/migrations/clickhouse/20241210181608_add_ttl_to_unsafe.sql @@ -0,0 +1,15 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE _unsafe_validator_dashboard_data_epoch MODIFY SETTING ttl_only_drop_parts = 1; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _unsafe_validator_dashboard_data_epoch MODIFY TTL _inserted_at + INTERVAL 1 WEEK DELETE SETTINGS materialize_ttl_after_modify=1; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE _unsafe_validator_dashboard_data_epoch MODIFY SETTING ttl_only_drop_parts = 0; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _unsafe_validator_dashboard_data_epoch REMOVE TTL SETTINGS materialize_ttl_after_modify=1; +-- +goose StatementEnd diff --git a/backend/pkg/commons/db/migrations/clickhouse/20241211154219_add_missing_helper_column_to_view.sql b/backend/pkg/commons/db/migrations/clickhouse/20241211154219_add_missing_helper_column_to_view.sql new file mode 100644 index 000000000..9283a74f5 --- /dev/null +++ b/backend/pkg/commons/db/migrations/clickhouse/20241211154219_add_missing_helper_column_to_view.sql @@ -0,0 +1,46 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE _unsafe_validator_dashboard_data_epoch DROP COLUMN IF EXISTS sync_reward; -- whoopsie +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _final_validator_dashboard_data_epoch DROP COLUMN IF EXISTS sync_reward; -- whoopsie +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_epoch as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward_penalties_only + attestations_reward_rewards_only as attestations_reward, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_data_epoch; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +-- not needed + +-- +goose StatementEnd diff --git a/backend/pkg/commons/db/migrations/clickhouse/20240821140310_status_reports.sql b/backend/pkg/commons/db/migrations/clickhouse/20241212132011_status_reports.sql similarity index 97% rename from backend/pkg/commons/db/migrations/clickhouse/20240821140310_status_reports.sql rename to backend/pkg/commons/db/migrations/clickhouse/20241212132011_status_reports.sql index f623581e6..2c116a744 100644 --- a/backend/pkg/commons/db/migrations/clickhouse/20240821140310_status_reports.sql +++ b/backend/pkg/commons/db/migrations/clickhouse/20241212132011_status_reports.sql @@ -22,4 +22,4 @@ TTL toDateTime(expires_at + toIntervalMonth(1)) -- +goose Down -- +goose StatementBegin DROP TABLE status_reports IF EXISTS --- +goose StatementEnd +-- +goose StatementEnd \ No newline at end of file diff --git a/backend/pkg/commons/db/migrations/clickhouse/20241212144649_rename_max_ts_views.sql b/backend/pkg/commons/db/migrations/clickhouse/20241212144649_rename_max_ts_views.sql new file mode 100644 index 000000000..422ad9820 --- /dev/null +++ b/backend/pkg/commons/db/migrations/clickhouse/20241212144649_rename_max_ts_views.sql @@ -0,0 +1,33 @@ +-- +goose Up +-- +goose StatementBegin +RENAME validator_dashboard_data_epoch_max_ts TO view_validator_dashboard_data_epoch_max_ts +-- +goose StatementEnd +-- +goose StatementBegin +RENAME validator_dashboard_data_hourly_max_ts TO view_validator_dashboard_data_hourly_max_ts +-- +goose StatementEnd +-- +goose StatementBegin +RENAME validator_dashboard_data_daily_max_ts TO view_validator_dashboard_data_daily_max_ts +-- +goose StatementEnd +-- +goose StatementBegin +RENAME validator_dashboard_data_weekly_max_ts TO view_validator_dashboard_data_weekly_max_ts +-- +goose StatementEnd +-- +goose StatementBegin +RENAME validator_dashboard_data_monthly_max_ts TO view_validator_dashboard_data_monthly_max_ts +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +RENAME view_validator_dashboard_data_epoch_max_ts TO validator_dashboard_data_epoch_max_ts +-- +goose StatementEnd +-- +goose StatementBegin +RENAME view_validator_dashboard_data_hourly_max_ts TO validator_dashboard_data_hourly_max_ts +-- +goose StatementEnd +-- +goose StatementBegin +RENAME view_validator_dashboard_data_daily_max_ts TO validator_dashboard_data_daily_max_ts +-- +goose StatementEnd +-- +goose StatementBegin +RENAME view_validator_dashboard_data_weekly_max_ts TO validator_dashboard_data_weekly_max_ts +-- +goose StatementEnd +-- +goose StatementBegin +RENAME view_validator_dashboard_data_monthly_max_ts TO validator_dashboard_data_monthly_max_ts +-- +goose StatementEnd \ No newline at end of file diff --git a/backend/pkg/commons/db/migrations/clickhouse/20241212161354_fix_epoch_max_ts.sql b/backend/pkg/commons/db/migrations/clickhouse/20241212161354_fix_epoch_max_ts.sql new file mode 100644 index 000000000..ba33a8f05 --- /dev/null +++ b/backend/pkg/commons/db/migrations/clickhouse/20241212161354_fix_epoch_max_ts.sql @@ -0,0 +1,11 @@ +-- +goose Up +-- +goose StatementBegin +create or replace view view_validator_dashboard_data_epoch_max_ts as + select max(epoch_timestamp) as t + from _final_validator_dashboard_data_epoch +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin + +-- +goose StatementEnd diff --git a/backend/pkg/commons/db/migrations/clickhouse/20241217145758_fix_missing_final_in_rolling_view.sql b/backend/pkg/commons/db/migrations/clickhouse/20241217145758_fix_missing_final_in_rolling_view.sql new file mode 100644 index 000000000..42ac336ae --- /dev/null +++ b/backend/pkg/commons/db/migrations/clickhouse/20241217145758_fix_missing_final_in_rolling_view.sql @@ -0,0 +1,403 @@ +-- +goose Up + +-- +goose StatementBegin +create or replace view validator_dashboard_data_rolling_1h as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward_penalties_only + attestations_reward_rewards_only as attestations_reward, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_rolling_1h FINAL +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_rolling_24h as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward_penalties_only + attestations_reward_rewards_only as attestations_reward, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_rolling_24h FINAL +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_rolling_7d as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward_penalties_only + attestations_reward_rewards_only as attestations_reward, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_rolling_7d FINAL + +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_rolling_30d as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward_penalties_only + attestations_reward_rewards_only as attestations_reward, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_rolling_30d FINAL +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_rolling_90d as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward_penalties_only + attestations_reward_rewards_only as attestations_reward, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_rolling_90d FINAL +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_rolling_total as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward_penalties_only + attestations_reward_rewards_only as attestations_reward, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_rolling_total FINAL +-- +goose StatementEnd + +-- +goose Down + +-- +goose StatementBegin +create or replace view validator_dashboard_data_rolling_1h as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward_penalties_only + attestations_reward_rewards_only as attestations_reward, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_rolling_1h +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_rolling_24h as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward_penalties_only + attestations_reward_rewards_only as attestations_reward, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_rolling_24h +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_rolling_7d as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward_penalties_only + attestations_reward_rewards_only as attestations_reward, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_rolling_7d + +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_rolling_30d as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward_penalties_only + attestations_reward_rewards_only as attestations_reward, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_rolling_30d +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_rolling_90d as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward_penalties_only + attestations_reward_rewards_only as attestations_reward, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_rolling_90d +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_rolling_total as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward_penalties_only + attestations_reward_rewards_only as attestations_reward, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_rolling_total +-- +goose StatementEnd \ No newline at end of file diff --git a/backend/pkg/commons/db/migrations/clickhouse/20241218095302_update_deprecated_snowflake_function.sql b/backend/pkg/commons/db/migrations/clickhouse/20241218095302_update_deprecated_snowflake_function.sql new file mode 100644 index 000000000..d6ea568e8 --- /dev/null +++ b/backend/pkg/commons/db/migrations/clickhouse/20241218095302_update_deprecated_snowflake_function.sql @@ -0,0 +1,9 @@ +-- +goose Up +-- +goose StatementBegin +alter table status_reports modify column `inserted_at` Datetime MATERIALIZED snowflakeIDToDateTime(insert_id::UInt64, 1288834974657) +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +alter table status_reports modify column `inserted_at` Datetime MATERIALIZED snowflakeToDateTime(insert_id) +-- +goose StatementEnd diff --git a/backend/pkg/commons/db/migrations/clickhouse/20241218131633_fix_all_max_ts.sql b/backend/pkg/commons/db/migrations/clickhouse/20241218131633_fix_all_max_ts.sql new file mode 100644 index 000000000..a1d1c212c --- /dev/null +++ b/backend/pkg/commons/db/migrations/clickhouse/20241218131633_fix_all_max_ts.sql @@ -0,0 +1,26 @@ +-- +goose Up +-- +goose StatementBegin +create or replace view view_validator_dashboard_data_hourly_max_ts as + select max(t) as t + from _final_validator_dashboard_data_hourly +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view view_validator_dashboard_data_daily_max_ts as + select max(t) as t + from _final_validator_dashboard_data_daily +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view view_validator_dashboard_data_weekly_max_ts as + select max(t) as t + from _final_validator_dashboard_data_weekly +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view view_validator_dashboard_data_monthly_max_ts as + select max(t) as t + from _final_validator_dashboard_data_monthly +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin + +-- +goose StatementEnd diff --git a/backend/pkg/commons/db/migrations/clickhouse/20241218143509_materialize_shortcuts_in_views.sql b/backend/pkg/commons/db/migrations/clickhouse/20241218143509_materialize_shortcuts_in_views.sql new file mode 100644 index 000000000..fcd0343ea --- /dev/null +++ b/backend/pkg/commons/db/migrations/clickhouse/20241218143509_materialize_shortcuts_in_views.sql @@ -0,0 +1,220 @@ +-- +goose Up +-- +goose StatementBegin +alter table _final_validator_dashboard_data_epoch +add column if not exists attestations_reward Int64 Materialized attestations_head_reward_penalties_only + attestations_source_reward_penalties_only + attestations_target_reward_penalties_only + attestations_inclusion_reward_penalties_only + attestations_inactivity_reward_penalties_only +attestations_head_reward_rewards_only + attestations_source_reward_rewards_only + attestations_target_reward_rewards_only + attestations_inclusion_reward_rewards_only + attestations_inactivity_reward_rewards_only +settings mutations_sync=1; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_epoch +add column if not exists attestations_ideal_reward Int64 Materialized attestations_ideal_head_reward + attestations_ideal_source_reward + attestations_ideal_target_reward + attestations_ideal_inclusion_reward + attestations_ideal_inactivity_reward +settings mutations_sync=1; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_hourly +add column if not exists attestations_reward Int64 Materialized attestations_head_reward_penalties_only + attestations_source_reward_penalties_only + attestations_target_reward_penalties_only + attestations_inclusion_reward_penalties_only + attestations_inactivity_reward_penalties_only +attestations_head_reward_rewards_only + attestations_source_reward_rewards_only + attestations_target_reward_rewards_only + attestations_inclusion_reward_rewards_only + attestations_inactivity_reward_rewards_only +settings mutations_sync=1; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_hourly +add column if not exists attestations_ideal_reward Int64 Materialized attestations_ideal_head_reward + attestations_ideal_source_reward + attestations_ideal_target_reward + attestations_ideal_inclusion_reward + attestations_ideal_inactivity_reward +settings mutations_sync=1; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_daily +add column if not exists attestations_reward Int64 Materialized attestations_head_reward_penalties_only + attestations_source_reward_penalties_only + attestations_target_reward_penalties_only + attestations_inclusion_reward_penalties_only + attestations_inactivity_reward_penalties_only +attestations_head_reward_rewards_only + attestations_source_reward_rewards_only + attestations_target_reward_rewards_only + attestations_inclusion_reward_rewards_only + attestations_inactivity_reward_rewards_only +settings mutations_sync=1; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_daily +add column if not exists attestations_ideal_reward Int64 Materialized attestations_ideal_head_reward + attestations_ideal_source_reward + attestations_ideal_target_reward + attestations_ideal_inclusion_reward + attestations_ideal_inactivity_reward +settings mutations_sync=1; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_weekly +add column if not exists attestations_reward Int64 Materialized attestations_head_reward_penalties_only + attestations_source_reward_penalties_only + attestations_target_reward_penalties_only + attestations_inclusion_reward_penalties_only + attestations_inactivity_reward_penalties_only +attestations_head_reward_rewards_only + attestations_source_reward_rewards_only + attestations_target_reward_rewards_only + attestations_inclusion_reward_rewards_only + attestations_inactivity_reward_rewards_only +settings mutations_sync=1; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_weekly +add column if not exists attestations_ideal_reward Int64 Materialized attestations_ideal_head_reward + attestations_ideal_source_reward + attestations_ideal_target_reward + attestations_ideal_inclusion_reward + attestations_ideal_inactivity_reward +settings mutations_sync=1; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_monthly +add column if not exists attestations_reward Int64 Materialized attestations_head_reward_penalties_only + attestations_source_reward_penalties_only + attestations_target_reward_penalties_only + attestations_inclusion_reward_penalties_only + attestations_inactivity_reward_penalties_only +attestations_head_reward_rewards_only + attestations_source_reward_rewards_only + attestations_target_reward_rewards_only + attestations_inclusion_reward_rewards_only + attestations_inactivity_reward_rewards_only +settings mutations_sync=1; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_monthly +add column if not exists attestations_ideal_reward Int64 Materialized attestations_ideal_head_reward + attestations_ideal_source_reward + attestations_ideal_target_reward + attestations_ideal_inclusion_reward + attestations_ideal_inactivity_reward +settings mutations_sync=1; +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_epoch as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward, + attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_data_epoch; +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_hourly as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward, + attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_data_hourly +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_daily as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward, + attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_data_daily +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_weekly as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward, + attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_data_weekly +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_monthly as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward, + attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_data_monthly +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_epoch materialize column attestations_reward; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_epoch materialize column attestations_ideal_reward; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_hourly materialize column attestations_reward; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_hourly materialize column attestations_ideal_reward; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_daily materialize column attestations_reward; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_daily materialize column attestations_ideal_reward; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_weekly materialize column attestations_reward; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_weekly materialize column attestations_ideal_reward; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_monthly materialize column attestations_reward; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_monthly materialize column attestations_ideal_reward; +-- +goose StatementEnd +-- +goose Down +-- +goose StatementBegin +Select('im not writing a down migration for this, just dont migrate down'); +-- +goose StatementEnd diff --git a/backend/pkg/commons/db/migrations/clickhouse/20250102123338_temporarily_disable_optimization.sql b/backend/pkg/commons/db/migrations/clickhouse/20250102123338_temporarily_disable_optimization.sql new file mode 100644 index 000000000..1a0670df8 --- /dev/null +++ b/backend/pkg/commons/db/migrations/clickhouse/20250102123338_temporarily_disable_optimization.sql @@ -0,0 +1,170 @@ +-- +goose Up +-- +goose StatementBegin +create or replace view validator_dashboard_data_epoch as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward_penalties_only + attestations_reward_rewards_only as attestations_reward, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward + from _final_validator_dashboard_data_epoch +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_hourly as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward_penalties_only + attestations_reward_rewards_only as attestations_reward, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_data_hourly +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_daily as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward_penalties_only + attestations_reward_rewards_only as attestations_reward, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_data_daily +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_weekly as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward_penalties_only + attestations_reward_rewards_only as attestations_reward, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_data_weekly +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_monthly as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward_penalties_only + attestations_reward_rewards_only as attestations_reward, + ( + attestations_ideal_head_reward + + attestations_ideal_source_reward + + attestations_ideal_target_reward + + attestations_ideal_inclusion_reward + + attestations_ideal_inactivity_reward + ) as attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_data_monthly +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin + +-- +goose StatementEnd diff --git a/backend/pkg/commons/db/migrations/clickhouse/20250102125004_fix_materialized_shortcuts.sql b/backend/pkg/commons/db/migrations/clickhouse/20250102125004_fix_materialized_shortcuts.sql new file mode 100644 index 000000000..924b88b7b --- /dev/null +++ b/backend/pkg/commons/db/migrations/clickhouse/20250102125004_fix_materialized_shortcuts.sql @@ -0,0 +1,70 @@ +-- +goose Up +-- +goose StatementBegin +alter table _final_validator_dashboard_data_hourly +add column if not exists attestations_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_head_reward_penalties_only + attestations_source_reward_penalties_only + attestations_target_reward_penalties_only + attestations_inclusion_reward_penalties_only + attestations_inactivity_reward_penalties_only +attestations_head_reward_rewards_only + attestations_source_reward_rewards_only + attestations_target_reward_rewards_only + attestations_inclusion_reward_rewards_only + attestations_inactivity_reward_rewards_only +settings mutations_sync=1; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_hourly +add column if not exists attestations_ideal_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_ideal_head_reward + attestations_ideal_source_reward + attestations_ideal_target_reward + attestations_ideal_inclusion_reward + attestations_ideal_inactivity_reward +settings mutations_sync=1; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_daily +add column if not exists attestations_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_head_reward_penalties_only + attestations_source_reward_penalties_only + attestations_target_reward_penalties_only + attestations_inclusion_reward_penalties_only + attestations_inactivity_reward_penalties_only +attestations_head_reward_rewards_only + attestations_source_reward_rewards_only + attestations_target_reward_rewards_only + attestations_inclusion_reward_rewards_only + attestations_inactivity_reward_rewards_only +settings mutations_sync=1; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_daily +add column if not exists attestations_ideal_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_ideal_head_reward + attestations_ideal_source_reward + attestations_ideal_target_reward + attestations_ideal_inclusion_reward + attestations_ideal_inactivity_reward +settings mutations_sync=1; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_weekly +add column if not exists attestations_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_head_reward_penalties_only + attestations_source_reward_penalties_only + attestations_target_reward_penalties_only + attestations_inclusion_reward_penalties_only + attestations_inactivity_reward_penalties_only +attestations_head_reward_rewards_only + attestations_source_reward_rewards_only + attestations_target_reward_rewards_only + attestations_inclusion_reward_rewards_only + attestations_inactivity_reward_rewards_only +settings mutations_sync=1; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_weekly +add column if not exists attestations_ideal_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_ideal_head_reward + attestations_ideal_source_reward + attestations_ideal_target_reward + attestations_ideal_inclusion_reward + attestations_ideal_inactivity_reward +settings mutations_sync=1; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_monthly +add column if not exists attestations_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_head_reward_penalties_only + attestations_source_reward_penalties_only + attestations_target_reward_penalties_only + attestations_inclusion_reward_penalties_only + attestations_inactivity_reward_penalties_only +attestations_head_reward_rewards_only + attestations_source_reward_rewards_only + attestations_target_reward_rewards_only + attestations_inclusion_reward_rewards_only + attestations_inactivity_reward_rewards_only +settings mutations_sync=1; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_monthly +add column if not exists attestations_ideal_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_ideal_head_reward + attestations_ideal_source_reward + attestations_ideal_target_reward + attestations_ideal_inclusion_reward + attestations_ideal_inactivity_reward +settings mutations_sync=1; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_hourly materialize column attestations_reward; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_hourly materialize column attestations_ideal_reward; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_daily materialize column attestations_reward; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_daily materialize column attestations_ideal_reward; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_weekly materialize column attestations_reward; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_weekly materialize column attestations_ideal_reward; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_monthly materialize column attestations_reward; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_monthly materialize column attestations_ideal_reward; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin + +-- +goose StatementEnd diff --git a/backend/pkg/commons/db/migrations/clickhouse/20250102131247_enable_optimization_again.sql b/backend/pkg/commons/db/migrations/clickhouse/20250102131247_enable_optimization_again.sql new file mode 100644 index 000000000..6bc113220 --- /dev/null +++ b/backend/pkg/commons/db/migrations/clickhouse/20250102131247_enable_optimization_again.sql @@ -0,0 +1,141 @@ +-- +goose Up +-- +goose StatementBegin +create or replace view validator_dashboard_data_epoch as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward, + attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_data_epoch; +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_hourly as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward, + attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_data_hourly +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_daily as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward, + attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_data_daily +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_weekly as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward, + attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_data_weekly +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_monthly as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward, + attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_data_monthly +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin + +-- +goose StatementEnd diff --git a/backend/pkg/commons/db/migrations/clickhouse/20250115123428_fix_missing_final_in_non_rolling_views.sql b/backend/pkg/commons/db/migrations/clickhouse/20250115123428_fix_missing_final_in_non_rolling_views.sql new file mode 100644 index 000000000..0a44c8c7d --- /dev/null +++ b/backend/pkg/commons/db/migrations/clickhouse/20250115123428_fix_missing_final_in_non_rolling_views.sql @@ -0,0 +1,141 @@ +-- +goose Up +-- +goose StatementBegin +create or replace view validator_dashboard_data_epoch as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward, + attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_data_epoch; +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_hourly as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward, + attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_data_hourly FINAL +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_daily as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward, + attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_data_daily FINAL +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_weekly as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward, + attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_data_weekly FINAL +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_monthly as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward, + attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_data_monthly FINAL +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin + +-- +goose StatementEnd diff --git a/backend/pkg/commons/db/migrations/clickhouse/20250115130854_fix_materialized_shortcuts_again.sql b/backend/pkg/commons/db/migrations/clickhouse/20250115130854_fix_materialized_shortcuts_again.sql new file mode 100644 index 000000000..9aaf0d29a --- /dev/null +++ b/backend/pkg/commons/db/migrations/clickhouse/20250115130854_fix_materialized_shortcuts_again.sql @@ -0,0 +1,70 @@ +-- +goose Up +-- +goose StatementBegin +alter table _final_validator_dashboard_data_hourly +modify column attestations_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_head_reward_penalties_only + attestations_source_reward_penalties_only + attestations_target_reward_penalties_only + attestations_inclusion_reward_penalties_only + attestations_inactivity_reward_penalties_only +attestations_head_reward_rewards_only + attestations_source_reward_rewards_only + attestations_target_reward_rewards_only + attestations_inclusion_reward_rewards_only + attestations_inactivity_reward_rewards_only +settings mutations_sync=1; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_hourly +modify column attestations_ideal_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_ideal_head_reward + attestations_ideal_source_reward + attestations_ideal_target_reward + attestations_ideal_inclusion_reward + attestations_ideal_inactivity_reward +settings mutations_sync=1; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_daily +modify column attestations_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_head_reward_penalties_only + attestations_source_reward_penalties_only + attestations_target_reward_penalties_only + attestations_inclusion_reward_penalties_only + attestations_inactivity_reward_penalties_only +attestations_head_reward_rewards_only + attestations_source_reward_rewards_only + attestations_target_reward_rewards_only + attestations_inclusion_reward_rewards_only + attestations_inactivity_reward_rewards_only +settings mutations_sync=1; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_daily +modify column attestations_ideal_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_ideal_head_reward + attestations_ideal_source_reward + attestations_ideal_target_reward + attestations_ideal_inclusion_reward + attestations_ideal_inactivity_reward +settings mutations_sync=1; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_weekly +modify column attestations_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_head_reward_penalties_only + attestations_source_reward_penalties_only + attestations_target_reward_penalties_only + attestations_inclusion_reward_penalties_only + attestations_inactivity_reward_penalties_only +attestations_head_reward_rewards_only + attestations_source_reward_rewards_only + attestations_target_reward_rewards_only + attestations_inclusion_reward_rewards_only + attestations_inactivity_reward_rewards_only +settings mutations_sync=1; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_weekly +modify column attestations_ideal_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_ideal_head_reward + attestations_ideal_source_reward + attestations_ideal_target_reward + attestations_ideal_inclusion_reward + attestations_ideal_inactivity_reward +settings mutations_sync=1; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_monthly +modify column attestations_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_head_reward_penalties_only + attestations_source_reward_penalties_only + attestations_target_reward_penalties_only + attestations_inclusion_reward_penalties_only + attestations_inactivity_reward_penalties_only +attestations_head_reward_rewards_only + attestations_source_reward_rewards_only + attestations_target_reward_rewards_only + attestations_inclusion_reward_rewards_only + attestations_inactivity_reward_rewards_only +settings mutations_sync=1; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_monthly +modify column attestations_ideal_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_ideal_head_reward + attestations_ideal_source_reward + attestations_ideal_target_reward + attestations_ideal_inclusion_reward + attestations_ideal_inactivity_reward +settings mutations_sync=1; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_hourly materialize column attestations_reward; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_hourly materialize column attestations_ideal_reward; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_daily materialize column attestations_reward; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_daily materialize column attestations_ideal_reward; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_weekly materialize column attestations_reward; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_weekly materialize column attestations_ideal_reward; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_monthly materialize column attestations_reward; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_data_monthly materialize column attestations_ideal_reward; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin + +-- +goose StatementEnd diff --git a/backend/pkg/commons/db/migrations/clickhouse/20250117140906_rollings_materialized_shortcuts.sql b/backend/pkg/commons/db/migrations/clickhouse/20250117140906_rollings_materialized_shortcuts.sql new file mode 100644 index 000000000..2e311d652 --- /dev/null +++ b/backend/pkg/commons/db/migrations/clickhouse/20250117140906_rollings_materialized_shortcuts.sql @@ -0,0 +1,293 @@ +-- +goose Up +-- +goose StatementBegin +alter table _unsafe_validator_dashboard_rolling_1h +add column attestations_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_head_reward_penalties_only + attestations_source_reward_penalties_only + attestations_target_reward_penalties_only + attestations_inclusion_reward_penalties_only + attestations_inactivity_reward_penalties_only +attestations_head_reward_rewards_only + attestations_source_reward_rewards_only + attestations_target_reward_rewards_only + attestations_inclusion_reward_rewards_only + attestations_inactivity_reward_rewards_only +settings mutations_sync=2; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_rolling_1h +add column attestations_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_head_reward_penalties_only + attestations_source_reward_penalties_only + attestations_target_reward_penalties_only + attestations_inclusion_reward_penalties_only + attestations_inactivity_reward_penalties_only +attestations_head_reward_rewards_only + attestations_source_reward_rewards_only + attestations_target_reward_rewards_only + attestations_inclusion_reward_rewards_only + attestations_inactivity_reward_rewards_only +settings mutations_sync=2; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _unsafe_validator_dashboard_rolling_1h +add column attestations_ideal_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_ideal_head_reward + attestations_ideal_source_reward + attestations_ideal_target_reward + attestations_ideal_inclusion_reward + attestations_ideal_inactivity_reward +settings mutations_sync=2; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_rolling_1h +add column attestations_ideal_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_ideal_head_reward + attestations_ideal_source_reward + attestations_ideal_target_reward + attestations_ideal_inclusion_reward + attestations_ideal_inactivity_reward +settings mutations_sync=2; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _unsafe_validator_dashboard_rolling_24h +add column attestations_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_head_reward_penalties_only + attestations_source_reward_penalties_only + attestations_target_reward_penalties_only + attestations_inclusion_reward_penalties_only + attestations_inactivity_reward_penalties_only +attestations_head_reward_rewards_only + attestations_source_reward_rewards_only + attestations_target_reward_rewards_only + attestations_inclusion_reward_rewards_only + attestations_inactivity_reward_rewards_only +settings mutations_sync=2; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_rolling_24h +add column attestations_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_head_reward_penalties_only + attestations_source_reward_penalties_only + attestations_target_reward_penalties_only + attestations_inclusion_reward_penalties_only + attestations_inactivity_reward_penalties_only +attestations_head_reward_rewards_only + attestations_source_reward_rewards_only + attestations_target_reward_rewards_only + attestations_inclusion_reward_rewards_only + attestations_inactivity_reward_rewards_only +settings mutations_sync=2; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _unsafe_validator_dashboard_rolling_24h +add column attestations_ideal_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_ideal_head_reward + attestations_ideal_source_reward + attestations_ideal_target_reward + attestations_ideal_inclusion_reward + attestations_ideal_inactivity_reward +settings mutations_sync=2; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_rolling_24h +add column attestations_ideal_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_ideal_head_reward + attestations_ideal_source_reward + attestations_ideal_target_reward + attestations_ideal_inclusion_reward + attestations_ideal_inactivity_reward +settings mutations_sync=2; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _unsafe_validator_dashboard_rolling_7d +add column attestations_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_head_reward_penalties_only + attestations_source_reward_penalties_only + attestations_target_reward_penalties_only + attestations_inclusion_reward_penalties_only + attestations_inactivity_reward_penalties_only +attestations_head_reward_rewards_only + attestations_source_reward_rewards_only + attestations_target_reward_rewards_only + attestations_inclusion_reward_rewards_only + attestations_inactivity_reward_rewards_only +settings mutations_sync=2; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_rolling_7d +add column attestations_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_head_reward_penalties_only + attestations_source_reward_penalties_only + attestations_target_reward_penalties_only + attestations_inclusion_reward_penalties_only + attestations_inactivity_reward_penalties_only +attestations_head_reward_rewards_only + attestations_source_reward_rewards_only + attestations_target_reward_rewards_only + attestations_inclusion_reward_rewards_only + attestations_inactivity_reward_rewards_only +settings mutations_sync=2; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _unsafe_validator_dashboard_rolling_7d +add column attestations_ideal_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_ideal_head_reward + attestations_ideal_source_reward + attestations_ideal_target_reward + attestations_ideal_inclusion_reward + attestations_ideal_inactivity_reward +settings mutations_sync=2; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_rolling_7d +add column attestations_ideal_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_ideal_head_reward + attestations_ideal_source_reward + attestations_ideal_target_reward + attestations_ideal_inclusion_reward + attestations_ideal_inactivity_reward +settings mutations_sync=2; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _unsafe_validator_dashboard_rolling_30d +add column attestations_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_head_reward_penalties_only + attestations_source_reward_penalties_only + attestations_target_reward_penalties_only + attestations_inclusion_reward_penalties_only + attestations_inactivity_reward_penalties_only +attestations_head_reward_rewards_only + attestations_source_reward_rewards_only + attestations_target_reward_rewards_only + attestations_inclusion_reward_rewards_only + attestations_inactivity_reward_rewards_only +settings mutations_sync=2; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_rolling_30d +add column attestations_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_head_reward_penalties_only + attestations_source_reward_penalties_only + attestations_target_reward_penalties_only + attestations_inclusion_reward_penalties_only + attestations_inactivity_reward_penalties_only +attestations_head_reward_rewards_only + attestations_source_reward_rewards_only + attestations_target_reward_rewards_only + attestations_inclusion_reward_rewards_only + attestations_inactivity_reward_rewards_only +settings mutations_sync=2; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _unsafe_validator_dashboard_rolling_30d +add column attestations_ideal_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_ideal_head_reward + attestations_ideal_source_reward + attestations_ideal_target_reward + attestations_ideal_inclusion_reward + attestations_ideal_inactivity_reward +settings mutations_sync=2; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_rolling_30d +add column attestations_ideal_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_ideal_head_reward + attestations_ideal_source_reward + attestations_ideal_target_reward + attestations_ideal_inclusion_reward + attestations_ideal_inactivity_reward +settings mutations_sync=2; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _unsafe_validator_dashboard_rolling_90d +add column attestations_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_head_reward_penalties_only + attestations_source_reward_penalties_only + attestations_target_reward_penalties_only + attestations_inclusion_reward_penalties_only + attestations_inactivity_reward_penalties_only +attestations_head_reward_rewards_only + attestations_source_reward_rewards_only + attestations_target_reward_rewards_only + attestations_inclusion_reward_rewards_only + attestations_inactivity_reward_rewards_only +settings mutations_sync=2; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_rolling_90d +add column attestations_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_head_reward_penalties_only + attestations_source_reward_penalties_only + attestations_target_reward_penalties_only + attestations_inclusion_reward_penalties_only + attestations_inactivity_reward_penalties_only +attestations_head_reward_rewards_only + attestations_source_reward_rewards_only + attestations_target_reward_rewards_only + attestations_inclusion_reward_rewards_only + attestations_inactivity_reward_rewards_only +settings mutations_sync=2; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _unsafe_validator_dashboard_rolling_90d +add column attestations_ideal_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_ideal_head_reward + attestations_ideal_source_reward + attestations_ideal_target_reward + attestations_ideal_inclusion_reward + attestations_ideal_inactivity_reward +settings mutations_sync=2; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_rolling_90d +add column attestations_ideal_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_ideal_head_reward + attestations_ideal_source_reward + attestations_ideal_target_reward + attestations_ideal_inclusion_reward + attestations_ideal_inactivity_reward +settings mutations_sync=2; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _unsafe_validator_dashboard_rolling_total +add column attestations_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_head_reward_penalties_only + attestations_source_reward_penalties_only + attestations_target_reward_penalties_only + attestations_inclusion_reward_penalties_only + attestations_inactivity_reward_penalties_only +attestations_head_reward_rewards_only + attestations_source_reward_rewards_only + attestations_target_reward_rewards_only + attestations_inclusion_reward_rewards_only + attestations_inactivity_reward_rewards_only +settings mutations_sync=2; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_rolling_total +add column attestations_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_head_reward_penalties_only + attestations_source_reward_penalties_only + attestations_target_reward_penalties_only + attestations_inclusion_reward_penalties_only + attestations_inactivity_reward_penalties_only +attestations_head_reward_rewards_only + attestations_source_reward_rewards_only + attestations_target_reward_rewards_only + attestations_inclusion_reward_rewards_only + attestations_inactivity_reward_rewards_only +settings mutations_sync=2; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _unsafe_validator_dashboard_rolling_total +add column attestations_ideal_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_ideal_head_reward + attestations_ideal_source_reward + attestations_ideal_target_reward + attestations_ideal_inclusion_reward + attestations_ideal_inactivity_reward +settings mutations_sync=2; +-- +goose StatementEnd +-- +goose StatementBegin +alter table _final_validator_dashboard_rolling_total +add column attestations_ideal_reward SimpleAggregateFunction(sum, Int64) Materialized attestations_ideal_head_reward + attestations_ideal_source_reward + attestations_ideal_target_reward + attestations_ideal_inclusion_reward + attestations_ideal_inactivity_reward +settings mutations_sync=2; +-- +goose StatementEnd +--- we dont bother materializing as it will be materialized on the next run anyways +--- we do have to update the views tho +-- +goose StatementBegin +create or replace view validator_dashboard_data_rolling_1h as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward, + attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_rolling_1h FINAL +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_rolling_24h as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward, + attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_rolling_24h FINAL +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_rolling_7d as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward, + attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_rolling_7d FINAL + +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_rolling_30d as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward, + attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_rolling_30d FINAL +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_rolling_90d as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward, + attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_rolling_90d FINAL +-- +goose StatementEnd +-- +goose StatementBegin +create or replace view validator_dashboard_data_rolling_total as + select *, + attestations_head_reward_penalties_only + attestations_head_reward_rewards_only as attestations_head_reward, + attestations_source_reward_penalties_only + attestations_source_reward_rewards_only as attestations_source_reward, + attestations_target_reward_penalties_only + attestations_target_reward_rewards_only as attestations_target_reward, + attestations_inclusion_reward_penalties_only + attestations_inclusion_reward_rewards_only as attestations_inclusion_reward, + attestations_inactivity_reward_penalties_only + attestations_inactivity_reward_rewards_only as attestations_inactivity_reward, + ( + attestations_head_reward_rewards_only + + attestations_source_reward_rewards_only + + attestations_target_reward_rewards_only + + attestations_inclusion_reward_rewards_only + + attestations_inactivity_reward_rewards_only + ) as attestations_reward_rewards_only, + ( + attestations_head_reward_penalties_only + + attestations_source_reward_penalties_only + + attestations_target_reward_penalties_only + + attestations_inclusion_reward_penalties_only + + attestations_inactivity_reward_penalties_only + ) as attestations_reward_penalties_only, + attestations_reward, + attestations_ideal_reward, + sync_reward_rewards_only + sync_reward_penalties_only as sync_reward + from _final_validator_dashboard_rolling_total FINAL +-- +goose StatementEnd + + + +-- +goose Down +-- +goose StatementBegin + +-- +goose StatementEnd diff --git a/backend/pkg/commons/db/migrations/clickhouse/20250117152735_rollings_no_compression.sql b/backend/pkg/commons/db/migrations/clickhouse/20250117152735_rollings_no_compression.sql new file mode 100644 index 000000000..e3920925c --- /dev/null +++ b/backend/pkg/commons/db/migrations/clickhouse/20250117152735_rollings_no_compression.sql @@ -0,0 +1,773 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE _unsafe_validator_dashboard_rolling_1h + MODIFY COLUMN `validator_index` CODEC(NONE), + MODIFY COLUMN `t` CODEC(NONE), + MODIFY COLUMN `epoch_map` CODEC(NONE), + MODIFY COLUMN `epoch_start` CODEC(NONE), + MODIFY COLUMN `epoch_end` CODEC(NONE), + MODIFY COLUMN `balance_start` CODEC(NONE), + MODIFY COLUMN `balance_end` CODEC(NONE), + MODIFY COLUMN `balance_min` CODEC(NONE), + MODIFY COLUMN `balance_max` CODEC(NONE), + MODIFY COLUMN `deposits_count` CODEC(NONE), + MODIFY COLUMN `deposits_amount` CODEC(NONE), + MODIFY COLUMN `withdrawals_count` CODEC(NONE), + MODIFY COLUMN `withdrawals_amount` CODEC(NONE), + MODIFY COLUMN `attestations_scheduled` CODEC(NONE), + MODIFY COLUMN `attestations_observed` CODEC(NONE), + MODIFY COLUMN `attestations_head_matched` CODEC(NONE), + MODIFY COLUMN `attestations_target_matched` CODEC(NONE), + MODIFY COLUMN `attestations_source_matched` CODEC(NONE), + MODIFY COLUMN `attestations_head_executed` CODEC(NONE), + MODIFY COLUMN `attestations_target_executed` CODEC(NONE), + MODIFY COLUMN `attestations_source_executed` CODEC(NONE), + MODIFY COLUMN `attestations_head_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_head_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_target_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_target_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_source_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_source_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_inactivity_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_inactivity_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_inclusion_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_inclusion_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_head_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_target_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_source_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_inactivity_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_inclusion_reward` CODEC(NONE), + MODIFY COLUMN `attestations_localized_max_reward` CODEC(NONE), + MODIFY COLUMN `attestations_hyperlocalized_max_reward` CODEC(NONE), + MODIFY COLUMN `inclusion_delay_sum` CODEC(NONE), + MODIFY COLUMN `optimal_inclusion_delay_sum` CODEC(NONE), + MODIFY COLUMN `blocks_scheduled` CODEC(NONE), + MODIFY COLUMN `blocks_proposed` CODEC(NONE), + MODIFY COLUMN `blocks_cl_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_attestations_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_sync_aggregate_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_slasher_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_missed_median_reward` CODEC(NONE), + MODIFY COLUMN `blocks_slashing_count` CODEC(NONE), + MODIFY COLUMN `blocks_expected` CODEC(NONE), + MODIFY COLUMN `sync_scheduled` CODEC(NONE), + MODIFY COLUMN `sync_executed` CODEC(NONE), + MODIFY COLUMN `sync_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `sync_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `sync_localized_max_reward` CODEC(NONE), + MODIFY COLUMN `sync_committees_expected` CODEC(NONE), + MODIFY COLUMN `slashed` CODEC(NONE), + MODIFY COLUMN `last_executed_duty_epoch` CODEC(NONE), + MODIFY COLUMN `last_scheduled_sync_epoch` CODEC(NONE), + MODIFY COLUMN `last_scheduled_block_epoch` CODEC(NONE), + MODIFY COLUMN `attestations_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_reward` CODEC(NONE); +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _final_validator_dashboard_rolling_1h + MODIFY COLUMN `validator_index` CODEC(NONE), + MODIFY COLUMN `t` CODEC(NONE), + MODIFY COLUMN `epoch_map` CODEC(NONE), + MODIFY COLUMN `epoch_start` CODEC(NONE), + MODIFY COLUMN `epoch_end` CODEC(NONE), + MODIFY COLUMN `balance_start` CODEC(NONE), + MODIFY COLUMN `balance_end` CODEC(NONE), + MODIFY COLUMN `balance_min` CODEC(NONE), + MODIFY COLUMN `balance_max` CODEC(NONE), + MODIFY COLUMN `deposits_count` CODEC(NONE), + MODIFY COLUMN `deposits_amount` CODEC(NONE), + MODIFY COLUMN `withdrawals_count` CODEC(NONE), + MODIFY COLUMN `withdrawals_amount` CODEC(NONE), + MODIFY COLUMN `attestations_scheduled` CODEC(NONE), + MODIFY COLUMN `attestations_observed` CODEC(NONE), + MODIFY COLUMN `attestations_head_matched` CODEC(NONE), + MODIFY COLUMN `attestations_target_matched` CODEC(NONE), + MODIFY COLUMN `attestations_source_matched` CODEC(NONE), + MODIFY COLUMN `attestations_head_executed` CODEC(NONE), + MODIFY COLUMN `attestations_target_executed` CODEC(NONE), + MODIFY COLUMN `attestations_source_executed` CODEC(NONE), + MODIFY COLUMN `attestations_head_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_head_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_target_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_target_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_source_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_source_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_inactivity_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_inactivity_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_inclusion_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_inclusion_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_head_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_target_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_source_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_inactivity_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_inclusion_reward` CODEC(NONE), + MODIFY COLUMN `attestations_localized_max_reward` CODEC(NONE), + MODIFY COLUMN `attestations_hyperlocalized_max_reward` CODEC(NONE), + MODIFY COLUMN `inclusion_delay_sum` CODEC(NONE), + MODIFY COLUMN `optimal_inclusion_delay_sum` CODEC(NONE), + MODIFY COLUMN `blocks_scheduled` CODEC(NONE), + MODIFY COLUMN `blocks_proposed` CODEC(NONE), + MODIFY COLUMN `blocks_cl_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_attestations_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_sync_aggregate_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_slasher_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_missed_median_reward` CODEC(NONE), + MODIFY COLUMN `blocks_slashing_count` CODEC(NONE), + MODIFY COLUMN `blocks_expected` CODEC(NONE), + MODIFY COLUMN `sync_scheduled` CODEC(NONE), + MODIFY COLUMN `sync_executed` CODEC(NONE), + MODIFY COLUMN `sync_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `sync_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `sync_localized_max_reward` CODEC(NONE), + MODIFY COLUMN `sync_committees_expected` CODEC(NONE), + MODIFY COLUMN `slashed` CODEC(NONE), + MODIFY COLUMN `last_executed_duty_epoch` CODEC(NONE), + MODIFY COLUMN `last_scheduled_sync_epoch` CODEC(NONE), + MODIFY COLUMN `last_scheduled_block_epoch` CODEC(NONE), + MODIFY COLUMN `attestations_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_reward` CODEC(NONE); +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _unsafe_validator_dashboard_rolling_24h + MODIFY COLUMN `validator_index` CODEC(NONE), + MODIFY COLUMN `t` CODEC(NONE), + MODIFY COLUMN `epoch_map` CODEC(NONE), + MODIFY COLUMN `epoch_start` CODEC(NONE), + MODIFY COLUMN `epoch_end` CODEC(NONE), + MODIFY COLUMN `balance_start` CODEC(NONE), + MODIFY COLUMN `balance_end` CODEC(NONE), + MODIFY COLUMN `balance_min` CODEC(NONE), + MODIFY COLUMN `balance_max` CODEC(NONE), + MODIFY COLUMN `deposits_count` CODEC(NONE), + MODIFY COLUMN `deposits_amount` CODEC(NONE), + MODIFY COLUMN `withdrawals_count` CODEC(NONE), + MODIFY COLUMN `withdrawals_amount` CODEC(NONE), + MODIFY COLUMN `attestations_scheduled` CODEC(NONE), + MODIFY COLUMN `attestations_observed` CODEC(NONE), + MODIFY COLUMN `attestations_head_matched` CODEC(NONE), + MODIFY COLUMN `attestations_target_matched` CODEC(NONE), + MODIFY COLUMN `attestations_source_matched` CODEC(NONE), + MODIFY COLUMN `attestations_head_executed` CODEC(NONE), + MODIFY COLUMN `attestations_target_executed` CODEC(NONE), + MODIFY COLUMN `attestations_source_executed` CODEC(NONE), + MODIFY COLUMN `attestations_head_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_head_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_target_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_target_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_source_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_source_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_inactivity_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_inactivity_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_inclusion_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_inclusion_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_head_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_target_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_source_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_inactivity_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_inclusion_reward` CODEC(NONE), + MODIFY COLUMN `attestations_localized_max_reward` CODEC(NONE), + MODIFY COLUMN `attestations_hyperlocalized_max_reward` CODEC(NONE), + MODIFY COLUMN `inclusion_delay_sum` CODEC(NONE), + MODIFY COLUMN `optimal_inclusion_delay_sum` CODEC(NONE), + MODIFY COLUMN `blocks_scheduled` CODEC(NONE), + MODIFY COLUMN `blocks_proposed` CODEC(NONE), + MODIFY COLUMN `blocks_cl_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_attestations_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_sync_aggregate_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_slasher_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_missed_median_reward` CODEC(NONE), + MODIFY COLUMN `blocks_slashing_count` CODEC(NONE), + MODIFY COLUMN `blocks_expected` CODEC(NONE), + MODIFY COLUMN `sync_scheduled` CODEC(NONE), + MODIFY COLUMN `sync_executed` CODEC(NONE), + MODIFY COLUMN `sync_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `sync_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `sync_localized_max_reward` CODEC(NONE), + MODIFY COLUMN `sync_committees_expected` CODEC(NONE), + MODIFY COLUMN `slashed` CODEC(NONE), + MODIFY COLUMN `last_executed_duty_epoch` CODEC(NONE), + MODIFY COLUMN `last_scheduled_sync_epoch` CODEC(NONE), + MODIFY COLUMN `last_scheduled_block_epoch` CODEC(NONE), + MODIFY COLUMN `attestations_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_reward` CODEC(NONE); +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _final_validator_dashboard_rolling_24h + MODIFY COLUMN `validator_index` CODEC(NONE), + MODIFY COLUMN `t` CODEC(NONE), + MODIFY COLUMN `epoch_map` CODEC(NONE), + MODIFY COLUMN `epoch_start` CODEC(NONE), + MODIFY COLUMN `epoch_end` CODEC(NONE), + MODIFY COLUMN `balance_start` CODEC(NONE), + MODIFY COLUMN `balance_end` CODEC(NONE), + MODIFY COLUMN `balance_min` CODEC(NONE), + MODIFY COLUMN `balance_max` CODEC(NONE), + MODIFY COLUMN `deposits_count` CODEC(NONE), + MODIFY COLUMN `deposits_amount` CODEC(NONE), + MODIFY COLUMN `withdrawals_count` CODEC(NONE), + MODIFY COLUMN `withdrawals_amount` CODEC(NONE), + MODIFY COLUMN `attestations_scheduled` CODEC(NONE), + MODIFY COLUMN `attestations_observed` CODEC(NONE), + MODIFY COLUMN `attestations_head_matched` CODEC(NONE), + MODIFY COLUMN `attestations_target_matched` CODEC(NONE), + MODIFY COLUMN `attestations_source_matched` CODEC(NONE), + MODIFY COLUMN `attestations_head_executed` CODEC(NONE), + MODIFY COLUMN `attestations_target_executed` CODEC(NONE), + MODIFY COLUMN `attestations_source_executed` CODEC(NONE), + MODIFY COLUMN `attestations_head_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_head_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_target_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_target_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_source_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_source_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_inactivity_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_inactivity_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_inclusion_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_inclusion_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_head_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_target_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_source_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_inactivity_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_inclusion_reward` CODEC(NONE), + MODIFY COLUMN `attestations_localized_max_reward` CODEC(NONE), + MODIFY COLUMN `attestations_hyperlocalized_max_reward` CODEC(NONE), + MODIFY COLUMN `inclusion_delay_sum` CODEC(NONE), + MODIFY COLUMN `optimal_inclusion_delay_sum` CODEC(NONE), + MODIFY COLUMN `blocks_scheduled` CODEC(NONE), + MODIFY COLUMN `blocks_proposed` CODEC(NONE), + MODIFY COLUMN `blocks_cl_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_attestations_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_sync_aggregate_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_slasher_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_missed_median_reward` CODEC(NONE), + MODIFY COLUMN `blocks_slashing_count` CODEC(NONE), + MODIFY COLUMN `blocks_expected` CODEC(NONE), + MODIFY COLUMN `sync_scheduled` CODEC(NONE), + MODIFY COLUMN `sync_executed` CODEC(NONE), + MODIFY COLUMN `sync_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `sync_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `sync_localized_max_reward` CODEC(NONE), + MODIFY COLUMN `sync_committees_expected` CODEC(NONE), + MODIFY COLUMN `slashed` CODEC(NONE), + MODIFY COLUMN `last_executed_duty_epoch` CODEC(NONE), + MODIFY COLUMN `last_scheduled_sync_epoch` CODEC(NONE), + MODIFY COLUMN `last_scheduled_block_epoch` CODEC(NONE), + MODIFY COLUMN `attestations_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_reward` CODEC(NONE); +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _unsafe_validator_dashboard_rolling_7d + MODIFY COLUMN `validator_index` CODEC(NONE), + MODIFY COLUMN `t` CODEC(NONE), + MODIFY COLUMN `epoch_map` CODEC(NONE), + MODIFY COLUMN `epoch_start` CODEC(NONE), + MODIFY COLUMN `epoch_end` CODEC(NONE), + MODIFY COLUMN `balance_start` CODEC(NONE), + MODIFY COLUMN `balance_end` CODEC(NONE), + MODIFY COLUMN `balance_min` CODEC(NONE), + MODIFY COLUMN `balance_max` CODEC(NONE), + MODIFY COLUMN `deposits_count` CODEC(NONE), + MODIFY COLUMN `deposits_amount` CODEC(NONE), + MODIFY COLUMN `withdrawals_count` CODEC(NONE), + MODIFY COLUMN `withdrawals_amount` CODEC(NONE), + MODIFY COLUMN `attestations_scheduled` CODEC(NONE), + MODIFY COLUMN `attestations_observed` CODEC(NONE), + MODIFY COLUMN `attestations_head_matched` CODEC(NONE), + MODIFY COLUMN `attestations_target_matched` CODEC(NONE), + MODIFY COLUMN `attestations_source_matched` CODEC(NONE), + MODIFY COLUMN `attestations_head_executed` CODEC(NONE), + MODIFY COLUMN `attestations_target_executed` CODEC(NONE), + MODIFY COLUMN `attestations_source_executed` CODEC(NONE), + MODIFY COLUMN `attestations_head_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_head_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_target_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_target_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_source_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_source_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_inactivity_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_inactivity_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_inclusion_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_inclusion_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_head_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_target_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_source_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_inactivity_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_inclusion_reward` CODEC(NONE), + MODIFY COLUMN `attestations_localized_max_reward` CODEC(NONE), + MODIFY COLUMN `attestations_hyperlocalized_max_reward` CODEC(NONE), + MODIFY COLUMN `inclusion_delay_sum` CODEC(NONE), + MODIFY COLUMN `optimal_inclusion_delay_sum` CODEC(NONE), + MODIFY COLUMN `blocks_scheduled` CODEC(NONE), + MODIFY COLUMN `blocks_proposed` CODEC(NONE), + MODIFY COLUMN `blocks_cl_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_attestations_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_sync_aggregate_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_slasher_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_missed_median_reward` CODEC(NONE), + MODIFY COLUMN `blocks_slashing_count` CODEC(NONE), + MODIFY COLUMN `blocks_expected` CODEC(NONE), + MODIFY COLUMN `sync_scheduled` CODEC(NONE), + MODIFY COLUMN `sync_executed` CODEC(NONE), + MODIFY COLUMN `sync_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `sync_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `sync_localized_max_reward` CODEC(NONE), + MODIFY COLUMN `sync_committees_expected` CODEC(NONE), + MODIFY COLUMN `slashed` CODEC(NONE), + MODIFY COLUMN `last_executed_duty_epoch` CODEC(NONE), + MODIFY COLUMN `last_scheduled_sync_epoch` CODEC(NONE), + MODIFY COLUMN `last_scheduled_block_epoch` CODEC(NONE), + MODIFY COLUMN `attestations_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_reward` CODEC(NONE); +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _final_validator_dashboard_rolling_7d + MODIFY COLUMN `validator_index` CODEC(NONE), + MODIFY COLUMN `t` CODEC(NONE), + MODIFY COLUMN `epoch_map` CODEC(NONE), + MODIFY COLUMN `epoch_start` CODEC(NONE), + MODIFY COLUMN `epoch_end` CODEC(NONE), + MODIFY COLUMN `balance_start` CODEC(NONE), + MODIFY COLUMN `balance_end` CODEC(NONE), + MODIFY COLUMN `balance_min` CODEC(NONE), + MODIFY COLUMN `balance_max` CODEC(NONE), + MODIFY COLUMN `deposits_count` CODEC(NONE), + MODIFY COLUMN `deposits_amount` CODEC(NONE), + MODIFY COLUMN `withdrawals_count` CODEC(NONE), + MODIFY COLUMN `withdrawals_amount` CODEC(NONE), + MODIFY COLUMN `attestations_scheduled` CODEC(NONE), + MODIFY COLUMN `attestations_observed` CODEC(NONE), + MODIFY COLUMN `attestations_head_matched` CODEC(NONE), + MODIFY COLUMN `attestations_target_matched` CODEC(NONE), + MODIFY COLUMN `attestations_source_matched` CODEC(NONE), + MODIFY COLUMN `attestations_head_executed` CODEC(NONE), + MODIFY COLUMN `attestations_target_executed` CODEC(NONE), + MODIFY COLUMN `attestations_source_executed` CODEC(NONE), + MODIFY COLUMN `attestations_head_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_head_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_target_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_target_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_source_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_source_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_inactivity_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_inactivity_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_inclusion_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_inclusion_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_head_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_target_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_source_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_inactivity_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_inclusion_reward` CODEC(NONE), + MODIFY COLUMN `attestations_localized_max_reward` CODEC(NONE), + MODIFY COLUMN `attestations_hyperlocalized_max_reward` CODEC(NONE), + MODIFY COLUMN `inclusion_delay_sum` CODEC(NONE), + MODIFY COLUMN `optimal_inclusion_delay_sum` CODEC(NONE), + MODIFY COLUMN `blocks_scheduled` CODEC(NONE), + MODIFY COLUMN `blocks_proposed` CODEC(NONE), + MODIFY COLUMN `blocks_cl_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_attestations_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_sync_aggregate_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_slasher_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_missed_median_reward` CODEC(NONE), + MODIFY COLUMN `blocks_slashing_count` CODEC(NONE), + MODIFY COLUMN `blocks_expected` CODEC(NONE), + MODIFY COLUMN `sync_scheduled` CODEC(NONE), + MODIFY COLUMN `sync_executed` CODEC(NONE), + MODIFY COLUMN `sync_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `sync_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `sync_localized_max_reward` CODEC(NONE), + MODIFY COLUMN `sync_committees_expected` CODEC(NONE), + MODIFY COLUMN `slashed` CODEC(NONE), + MODIFY COLUMN `last_executed_duty_epoch` CODEC(NONE), + MODIFY COLUMN `last_scheduled_sync_epoch` CODEC(NONE), + MODIFY COLUMN `last_scheduled_block_epoch` CODEC(NONE), + MODIFY COLUMN `attestations_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_reward` CODEC(NONE); +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _unsafe_validator_dashboard_rolling_30d + MODIFY COLUMN `validator_index` CODEC(NONE), + MODIFY COLUMN `t` CODEC(NONE), + MODIFY COLUMN `epoch_map` CODEC(NONE), + MODIFY COLUMN `epoch_start` CODEC(NONE), + MODIFY COLUMN `epoch_end` CODEC(NONE), + MODIFY COLUMN `balance_start` CODEC(NONE), + MODIFY COLUMN `balance_end` CODEC(NONE), + MODIFY COLUMN `balance_min` CODEC(NONE), + MODIFY COLUMN `balance_max` CODEC(NONE), + MODIFY COLUMN `deposits_count` CODEC(NONE), + MODIFY COLUMN `deposits_amount` CODEC(NONE), + MODIFY COLUMN `withdrawals_count` CODEC(NONE), + MODIFY COLUMN `withdrawals_amount` CODEC(NONE), + MODIFY COLUMN `attestations_scheduled` CODEC(NONE), + MODIFY COLUMN `attestations_observed` CODEC(NONE), + MODIFY COLUMN `attestations_head_matched` CODEC(NONE), + MODIFY COLUMN `attestations_target_matched` CODEC(NONE), + MODIFY COLUMN `attestations_source_matched` CODEC(NONE), + MODIFY COLUMN `attestations_head_executed` CODEC(NONE), + MODIFY COLUMN `attestations_target_executed` CODEC(NONE), + MODIFY COLUMN `attestations_source_executed` CODEC(NONE), + MODIFY COLUMN `attestations_head_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_head_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_target_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_target_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_source_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_source_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_inactivity_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_inactivity_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_inclusion_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_inclusion_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_head_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_target_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_source_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_inactivity_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_inclusion_reward` CODEC(NONE), + MODIFY COLUMN `attestations_localized_max_reward` CODEC(NONE), + MODIFY COLUMN `attestations_hyperlocalized_max_reward` CODEC(NONE), + MODIFY COLUMN `inclusion_delay_sum` CODEC(NONE), + MODIFY COLUMN `optimal_inclusion_delay_sum` CODEC(NONE), + MODIFY COLUMN `blocks_scheduled` CODEC(NONE), + MODIFY COLUMN `blocks_proposed` CODEC(NONE), + MODIFY COLUMN `blocks_cl_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_attestations_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_sync_aggregate_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_slasher_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_missed_median_reward` CODEC(NONE), + MODIFY COLUMN `blocks_slashing_count` CODEC(NONE), + MODIFY COLUMN `blocks_expected` CODEC(NONE), + MODIFY COLUMN `sync_scheduled` CODEC(NONE), + MODIFY COLUMN `sync_executed` CODEC(NONE), + MODIFY COLUMN `sync_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `sync_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `sync_localized_max_reward` CODEC(NONE), + MODIFY COLUMN `sync_committees_expected` CODEC(NONE), + MODIFY COLUMN `slashed` CODEC(NONE), + MODIFY COLUMN `last_executed_duty_epoch` CODEC(NONE), + MODIFY COLUMN `last_scheduled_sync_epoch` CODEC(NONE), + MODIFY COLUMN `last_scheduled_block_epoch` CODEC(NONE), + MODIFY COLUMN `attestations_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_reward` CODEC(NONE); +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _final_validator_dashboard_rolling_30d + MODIFY COLUMN `validator_index` CODEC(NONE), + MODIFY COLUMN `t` CODEC(NONE), + MODIFY COLUMN `epoch_map` CODEC(NONE), + MODIFY COLUMN `epoch_start` CODEC(NONE), + MODIFY COLUMN `epoch_end` CODEC(NONE), + MODIFY COLUMN `balance_start` CODEC(NONE), + MODIFY COLUMN `balance_end` CODEC(NONE), + MODIFY COLUMN `balance_min` CODEC(NONE), + MODIFY COLUMN `balance_max` CODEC(NONE), + MODIFY COLUMN `deposits_count` CODEC(NONE), + MODIFY COLUMN `deposits_amount` CODEC(NONE), + MODIFY COLUMN `withdrawals_count` CODEC(NONE), + MODIFY COLUMN `withdrawals_amount` CODEC(NONE), + MODIFY COLUMN `attestations_scheduled` CODEC(NONE), + MODIFY COLUMN `attestations_observed` CODEC(NONE), + MODIFY COLUMN `attestations_head_matched` CODEC(NONE), + MODIFY COLUMN `attestations_target_matched` CODEC(NONE), + MODIFY COLUMN `attestations_source_matched` CODEC(NONE), + MODIFY COLUMN `attestations_head_executed` CODEC(NONE), + MODIFY COLUMN `attestations_target_executed` CODEC(NONE), + MODIFY COLUMN `attestations_source_executed` CODEC(NONE), + MODIFY COLUMN `attestations_head_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_head_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_target_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_target_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_source_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_source_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_inactivity_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_inactivity_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_inclusion_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_inclusion_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_head_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_target_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_source_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_inactivity_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_inclusion_reward` CODEC(NONE), + MODIFY COLUMN `attestations_localized_max_reward` CODEC(NONE), + MODIFY COLUMN `attestations_hyperlocalized_max_reward` CODEC(NONE), + MODIFY COLUMN `inclusion_delay_sum` CODEC(NONE), + MODIFY COLUMN `optimal_inclusion_delay_sum` CODEC(NONE), + MODIFY COLUMN `blocks_scheduled` CODEC(NONE), + MODIFY COLUMN `blocks_proposed` CODEC(NONE), + MODIFY COLUMN `blocks_cl_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_attestations_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_sync_aggregate_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_slasher_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_missed_median_reward` CODEC(NONE), + MODIFY COLUMN `blocks_slashing_count` CODEC(NONE), + MODIFY COLUMN `blocks_expected` CODEC(NONE), + MODIFY COLUMN `sync_scheduled` CODEC(NONE), + MODIFY COLUMN `sync_executed` CODEC(NONE), + MODIFY COLUMN `sync_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `sync_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `sync_localized_max_reward` CODEC(NONE), + MODIFY COLUMN `sync_committees_expected` CODEC(NONE), + MODIFY COLUMN `slashed` CODEC(NONE), + MODIFY COLUMN `last_executed_duty_epoch` CODEC(NONE), + MODIFY COLUMN `last_scheduled_sync_epoch` CODEC(NONE), + MODIFY COLUMN `last_scheduled_block_epoch` CODEC(NONE), + MODIFY COLUMN `attestations_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_reward` CODEC(NONE); +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _unsafe_validator_dashboard_rolling_90d + MODIFY COLUMN `validator_index` CODEC(NONE), + MODIFY COLUMN `t` CODEC(NONE), + MODIFY COLUMN `epoch_map` CODEC(NONE), + MODIFY COLUMN `epoch_start` CODEC(NONE), + MODIFY COLUMN `epoch_end` CODEC(NONE), + MODIFY COLUMN `balance_start` CODEC(NONE), + MODIFY COLUMN `balance_end` CODEC(NONE), + MODIFY COLUMN `balance_min` CODEC(NONE), + MODIFY COLUMN `balance_max` CODEC(NONE), + MODIFY COLUMN `deposits_count` CODEC(NONE), + MODIFY COLUMN `deposits_amount` CODEC(NONE), + MODIFY COLUMN `withdrawals_count` CODEC(NONE), + MODIFY COLUMN `withdrawals_amount` CODEC(NONE), + MODIFY COLUMN `attestations_scheduled` CODEC(NONE), + MODIFY COLUMN `attestations_observed` CODEC(NONE), + MODIFY COLUMN `attestations_head_matched` CODEC(NONE), + MODIFY COLUMN `attestations_target_matched` CODEC(NONE), + MODIFY COLUMN `attestations_source_matched` CODEC(NONE), + MODIFY COLUMN `attestations_head_executed` CODEC(NONE), + MODIFY COLUMN `attestations_target_executed` CODEC(NONE), + MODIFY COLUMN `attestations_source_executed` CODEC(NONE), + MODIFY COLUMN `attestations_head_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_head_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_target_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_target_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_source_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_source_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_inactivity_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_inactivity_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_inclusion_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_inclusion_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_head_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_target_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_source_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_inactivity_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_inclusion_reward` CODEC(NONE), + MODIFY COLUMN `attestations_localized_max_reward` CODEC(NONE), + MODIFY COLUMN `attestations_hyperlocalized_max_reward` CODEC(NONE), + MODIFY COLUMN `inclusion_delay_sum` CODEC(NONE), + MODIFY COLUMN `optimal_inclusion_delay_sum` CODEC(NONE), + MODIFY COLUMN `blocks_scheduled` CODEC(NONE), + MODIFY COLUMN `blocks_proposed` CODEC(NONE), + MODIFY COLUMN `blocks_cl_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_attestations_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_sync_aggregate_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_slasher_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_missed_median_reward` CODEC(NONE), + MODIFY COLUMN `blocks_slashing_count` CODEC(NONE), + MODIFY COLUMN `blocks_expected` CODEC(NONE), + MODIFY COLUMN `sync_scheduled` CODEC(NONE), + MODIFY COLUMN `sync_executed` CODEC(NONE), + MODIFY COLUMN `sync_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `sync_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `sync_localized_max_reward` CODEC(NONE), + MODIFY COLUMN `sync_committees_expected` CODEC(NONE), + MODIFY COLUMN `slashed` CODEC(NONE), + MODIFY COLUMN `last_executed_duty_epoch` CODEC(NONE), + MODIFY COLUMN `last_scheduled_sync_epoch` CODEC(NONE), + MODIFY COLUMN `last_scheduled_block_epoch` CODEC(NONE), + MODIFY COLUMN `attestations_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_reward` CODEC(NONE); +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _final_validator_dashboard_rolling_90d + MODIFY COLUMN `validator_index` CODEC(NONE), + MODIFY COLUMN `t` CODEC(NONE), + MODIFY COLUMN `epoch_map` CODEC(NONE), + MODIFY COLUMN `epoch_start` CODEC(NONE), + MODIFY COLUMN `epoch_end` CODEC(NONE), + MODIFY COLUMN `balance_start` CODEC(NONE), + MODIFY COLUMN `balance_end` CODEC(NONE), + MODIFY COLUMN `balance_min` CODEC(NONE), + MODIFY COLUMN `balance_max` CODEC(NONE), + MODIFY COLUMN `deposits_count` CODEC(NONE), + MODIFY COLUMN `deposits_amount` CODEC(NONE), + MODIFY COLUMN `withdrawals_count` CODEC(NONE), + MODIFY COLUMN `withdrawals_amount` CODEC(NONE), + MODIFY COLUMN `attestations_scheduled` CODEC(NONE), + MODIFY COLUMN `attestations_observed` CODEC(NONE), + MODIFY COLUMN `attestations_head_matched` CODEC(NONE), + MODIFY COLUMN `attestations_target_matched` CODEC(NONE), + MODIFY COLUMN `attestations_source_matched` CODEC(NONE), + MODIFY COLUMN `attestations_head_executed` CODEC(NONE), + MODIFY COLUMN `attestations_target_executed` CODEC(NONE), + MODIFY COLUMN `attestations_source_executed` CODEC(NONE), + MODIFY COLUMN `attestations_head_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_head_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_target_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_target_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_source_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_source_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_inactivity_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_inactivity_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_inclusion_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_inclusion_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_head_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_target_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_source_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_inactivity_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_inclusion_reward` CODEC(NONE), + MODIFY COLUMN `attestations_localized_max_reward` CODEC(NONE), + MODIFY COLUMN `attestations_hyperlocalized_max_reward` CODEC(NONE), + MODIFY COLUMN `inclusion_delay_sum` CODEC(NONE), + MODIFY COLUMN `optimal_inclusion_delay_sum` CODEC(NONE), + MODIFY COLUMN `blocks_scheduled` CODEC(NONE), + MODIFY COLUMN `blocks_proposed` CODEC(NONE), + MODIFY COLUMN `blocks_cl_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_attestations_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_sync_aggregate_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_slasher_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_missed_median_reward` CODEC(NONE), + MODIFY COLUMN `blocks_slashing_count` CODEC(NONE), + MODIFY COLUMN `blocks_expected` CODEC(NONE), + MODIFY COLUMN `sync_scheduled` CODEC(NONE), + MODIFY COLUMN `sync_executed` CODEC(NONE), + MODIFY COLUMN `sync_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `sync_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `sync_localized_max_reward` CODEC(NONE), + MODIFY COLUMN `sync_committees_expected` CODEC(NONE), + MODIFY COLUMN `slashed` CODEC(NONE), + MODIFY COLUMN `last_executed_duty_epoch` CODEC(NONE), + MODIFY COLUMN `last_scheduled_sync_epoch` CODEC(NONE), + MODIFY COLUMN `last_scheduled_block_epoch` CODEC(NONE), + MODIFY COLUMN `attestations_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_reward` CODEC(NONE); +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _unsafe_validator_dashboard_rolling_total + MODIFY COLUMN `validator_index` CODEC(NONE), + MODIFY COLUMN `t` CODEC(NONE), + MODIFY COLUMN `epoch_map` CODEC(NONE), + MODIFY COLUMN `epoch_start` CODEC(NONE), + MODIFY COLUMN `epoch_end` CODEC(NONE), + MODIFY COLUMN `balance_start` CODEC(NONE), + MODIFY COLUMN `balance_end` CODEC(NONE), + MODIFY COLUMN `balance_min` CODEC(NONE), + MODIFY COLUMN `balance_max` CODEC(NONE), + MODIFY COLUMN `deposits_count` CODEC(NONE), + MODIFY COLUMN `deposits_amount` CODEC(NONE), + MODIFY COLUMN `withdrawals_count` CODEC(NONE), + MODIFY COLUMN `withdrawals_amount` CODEC(NONE), + MODIFY COLUMN `attestations_scheduled` CODEC(NONE), + MODIFY COLUMN `attestations_observed` CODEC(NONE), + MODIFY COLUMN `attestations_head_matched` CODEC(NONE), + MODIFY COLUMN `attestations_target_matched` CODEC(NONE), + MODIFY COLUMN `attestations_source_matched` CODEC(NONE), + MODIFY COLUMN `attestations_head_executed` CODEC(NONE), + MODIFY COLUMN `attestations_target_executed` CODEC(NONE), + MODIFY COLUMN `attestations_source_executed` CODEC(NONE), + MODIFY COLUMN `attestations_head_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_head_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_target_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_target_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_source_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_source_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_inactivity_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_inactivity_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_inclusion_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_inclusion_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_head_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_target_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_source_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_inactivity_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_inclusion_reward` CODEC(NONE), + MODIFY COLUMN `attestations_localized_max_reward` CODEC(NONE), + MODIFY COLUMN `attestations_hyperlocalized_max_reward` CODEC(NONE), + MODIFY COLUMN `inclusion_delay_sum` CODEC(NONE), + MODIFY COLUMN `optimal_inclusion_delay_sum` CODEC(NONE), + MODIFY COLUMN `blocks_scheduled` CODEC(NONE), + MODIFY COLUMN `blocks_proposed` CODEC(NONE), + MODIFY COLUMN `blocks_cl_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_attestations_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_sync_aggregate_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_slasher_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_missed_median_reward` CODEC(NONE), + MODIFY COLUMN `blocks_slashing_count` CODEC(NONE), + MODIFY COLUMN `blocks_expected` CODEC(NONE), + MODIFY COLUMN `sync_scheduled` CODEC(NONE), + MODIFY COLUMN `sync_executed` CODEC(NONE), + MODIFY COLUMN `sync_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `sync_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `sync_localized_max_reward` CODEC(NONE), + MODIFY COLUMN `sync_committees_expected` CODEC(NONE), + MODIFY COLUMN `slashed` CODEC(NONE), + MODIFY COLUMN `last_executed_duty_epoch` CODEC(NONE), + MODIFY COLUMN `last_scheduled_sync_epoch` CODEC(NONE), + MODIFY COLUMN `last_scheduled_block_epoch` CODEC(NONE), + MODIFY COLUMN `attestations_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_reward` CODEC(NONE); +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE _final_validator_dashboard_rolling_total + MODIFY COLUMN `validator_index` CODEC(NONE), + MODIFY COLUMN `t` CODEC(NONE), + MODIFY COLUMN `epoch_map` CODEC(NONE), + MODIFY COLUMN `epoch_start` CODEC(NONE), + MODIFY COLUMN `epoch_end` CODEC(NONE), + MODIFY COLUMN `balance_start` CODEC(NONE), + MODIFY COLUMN `balance_end` CODEC(NONE), + MODIFY COLUMN `balance_min` CODEC(NONE), + MODIFY COLUMN `balance_max` CODEC(NONE), + MODIFY COLUMN `deposits_count` CODEC(NONE), + MODIFY COLUMN `deposits_amount` CODEC(NONE), + MODIFY COLUMN `withdrawals_count` CODEC(NONE), + MODIFY COLUMN `withdrawals_amount` CODEC(NONE), + MODIFY COLUMN `attestations_scheduled` CODEC(NONE), + MODIFY COLUMN `attestations_observed` CODEC(NONE), + MODIFY COLUMN `attestations_head_matched` CODEC(NONE), + MODIFY COLUMN `attestations_target_matched` CODEC(NONE), + MODIFY COLUMN `attestations_source_matched` CODEC(NONE), + MODIFY COLUMN `attestations_head_executed` CODEC(NONE), + MODIFY COLUMN `attestations_target_executed` CODEC(NONE), + MODIFY COLUMN `attestations_source_executed` CODEC(NONE), + MODIFY COLUMN `attestations_head_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_head_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_target_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_target_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_source_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_source_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_inactivity_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_inactivity_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_inclusion_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `attestations_inclusion_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_head_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_target_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_source_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_inactivity_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_inclusion_reward` CODEC(NONE), + MODIFY COLUMN `attestations_localized_max_reward` CODEC(NONE), + MODIFY COLUMN `attestations_hyperlocalized_max_reward` CODEC(NONE), + MODIFY COLUMN `inclusion_delay_sum` CODEC(NONE), + MODIFY COLUMN `optimal_inclusion_delay_sum` CODEC(NONE), + MODIFY COLUMN `blocks_scheduled` CODEC(NONE), + MODIFY COLUMN `blocks_proposed` CODEC(NONE), + MODIFY COLUMN `blocks_cl_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_attestations_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_sync_aggregate_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_slasher_reward` CODEC(NONE), + MODIFY COLUMN `blocks_cl_missed_median_reward` CODEC(NONE), + MODIFY COLUMN `blocks_slashing_count` CODEC(NONE), + MODIFY COLUMN `blocks_expected` CODEC(NONE), + MODIFY COLUMN `sync_scheduled` CODEC(NONE), + MODIFY COLUMN `sync_executed` CODEC(NONE), + MODIFY COLUMN `sync_reward_rewards_only` CODEC(NONE), + MODIFY COLUMN `sync_reward_penalties_only` CODEC(NONE), + MODIFY COLUMN `sync_localized_max_reward` CODEC(NONE), + MODIFY COLUMN `sync_committees_expected` CODEC(NONE), + MODIFY COLUMN `slashed` CODEC(NONE), + MODIFY COLUMN `last_executed_duty_epoch` CODEC(NONE), + MODIFY COLUMN `last_scheduled_sync_epoch` CODEC(NONE), + MODIFY COLUMN `last_scheduled_block_epoch` CODEC(NONE), + MODIFY COLUMN `attestations_reward` CODEC(NONE), + MODIFY COLUMN `attestations_ideal_reward` CODEC(NONE); +-- +goose StatementEnd +-- +goose Down +-- +goose StatementBegin + +-- +goose StatementEnd diff --git a/backend/pkg/commons/metrics/metrics.go b/backend/pkg/commons/metrics/metrics.go index 2470124bb..7cd5e729c 100644 --- a/backend/pkg/commons/metrics/metrics.go +++ b/backend/pkg/commons/metrics/metrics.go @@ -29,6 +29,10 @@ var ( Name: "deployment_type", Help: "Gauge with deployment-type in label", }, []string{"deployment_type"}) + DatabaseVersion = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "db_version", + Help: "Gauge with database and version in labels", + }, []string{"brand", "name", "version"}) HttpRequestsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "http_requests_total", Help: "Total number of requests by path, method and status_code.", diff --git a/backend/pkg/commons/types/chain.go b/backend/pkg/commons/types/chain.go index 36321267e..2a38204b2 100644 --- a/backend/pkg/commons/types/chain.go +++ b/backend/pkg/commons/types/chain.go @@ -2,8 +2,8 @@ package types type ForkVersion struct { Epoch uint64 - CurrentVersion []byte - PreviousVersion []byte + CurrentVersion string + PreviousVersion string } // https://github.com/ethereum/consensus-specs/blob/dev/configs/mainnet.yaml @@ -24,8 +24,8 @@ type ClChainConfig struct { AltairForkEpoch uint64 `yaml:"ALTAIR_FORK_EPOCH"` BellatrixForkVersion string `yaml:"BELLATRIX_FORK_VERSION"` BellatrixForkEpoch uint64 `yaml:"BELLATRIX_FORK_EPOCH"` - CappellaForkVersion string `yaml:"CAPELLA_FORK_VERSION"` - CappellaForkEpoch uint64 `yaml:"CAPELLA_FORK_EPOCH"` + CapellaForkVersion string `yaml:"CAPELLA_FORK_VERSION"` + CapellaForkEpoch uint64 `yaml:"CAPELLA_FORK_EPOCH"` DenebForkVersion string `yaml:"DENEB_FORK_VERSION"` DenebForkEpoch uint64 `yaml:"DENEB_FORK_EPOCH"` Eip6110ForkVersion string `yaml:"EIP6110_FORK_VERSION"` diff --git a/backend/pkg/commons/types/config.go b/backend/pkg/commons/types/config.go index 3629ba370..c9236f62b 100644 --- a/backend/pkg/commons/types/config.go +++ b/backend/pkg/commons/types/config.go @@ -220,6 +220,15 @@ type Config struct { MevBoostRelayExporter struct { Enabled bool `yaml:"enabled" env:"ENABLED"` } `yaml:"mevBoostRelayExporter" env:", prefix=MEVBOOSTRELAY_EXPORTER_"` + DashboardExporter struct { + RollingsInParallel int64 `yaml:"rollingsAtOnce" envconfig:"DASHBOARD_EXPORTER_ROLLINGS_AT_ONCE"` // how many rollings to do at once + RollingPartsInParallel int64 `yaml:"rollingsInParallel" envconfig:"DASHBOARD_EXPORTER_ROLLINGS_IN_PARALLEL"` // how man parts of a single rolling to do at once + TransferInParallel int64 `yaml:"transferInParallel" envconfig:"DASHBOARD_EXPORTER_TRANSFER_IN_PARALLEL"` // how many transfers to do at once + TransferAtOnce int64 `yaml:"transferAtOnce" envconfig:"DASHBOARD_EXPORTER_TRANSFER_AT_ONCE"` // how much data to transfer in a single transfer + FetchAtOnceLimit int64 `yaml:"fetchAtOnceLimit" envconfig:"DASHBOARD_EXPORTER_FETCH_AT_ONCE_LIMIT"` // how much data to fetch in a single fetch + InsertAtOnceLimit int64 `yaml:"insertAtOnceLimit" envconfig:"DASHBOARD_EXPORTER_INSERT_AT_ONCE_LIMIT"` // how much data to insert in a single insert + InsertInParallel int64 `yaml:"insertInParallel" envconfig:"DASHBOARD_EXPORTER_INSERT_IN_PARALLEL"` // how many inserts to do at once + } `yaml:"dashboardExporter" env:", prefix=DASHBOARD_EXPORTER_"` Pprof struct { Enabled bool `yaml:"enabled" env:"ENABLED"` Port string `yaml:"port" env:"PORT"` diff --git a/backend/pkg/commons/utils/config.go b/backend/pkg/commons/utils/config.go index 134f926fc..51fd05513 100644 --- a/backend/pkg/commons/utils/config.go +++ b/backend/pkg/commons/utils/config.go @@ -240,6 +240,30 @@ func ReadConfig(cfg *types.Config, path string) error { } } + // dashboard exporter default limits + + if cfg.DashboardExporter.RollingsInParallel == 0 { + cfg.DashboardExporter.RollingsInParallel = 3 + } + if cfg.DashboardExporter.RollingPartsInParallel == 0 { + cfg.DashboardExporter.RollingPartsInParallel = 3 + } + if cfg.DashboardExporter.TransferInParallel == 0 { + cfg.DashboardExporter.TransferInParallel = 3 + } + if cfg.DashboardExporter.TransferAtOnce == 0 { + cfg.DashboardExporter.TransferAtOnce = 2 + } + if cfg.DashboardExporter.FetchAtOnceLimit == 0 { + cfg.DashboardExporter.FetchAtOnceLimit = 2 + } + if cfg.DashboardExporter.InsertAtOnceLimit == 0 { + cfg.DashboardExporter.InsertAtOnceLimit = 2 + } + if cfg.DashboardExporter.InsertInParallel == 0 { + cfg.DashboardExporter.InsertInParallel = 2 + } + // we check for machine chain id just for safety if cfg.Chain.Id != 0 && cfg.Chain.Id != cfg.Chain.ClConfig.DepositChainID { log.Fatal(fmt.Errorf("cfg.Chain.Id != cfg.Chain.ClConfig.DepositChainID: %v != %v", cfg.Chain.Id, cfg.Chain.ClConfig.DepositChainID), "", 0) @@ -393,8 +417,8 @@ func setCLConfig(cfg *types.Config) error { AltairForkEpoch: *jr.Data.AltairForkEpoch, BellatrixForkVersion: jr.Data.BellatrixForkVersion, BellatrixForkEpoch: *jr.Data.BellatrixForkEpoch, - CappellaForkVersion: jr.Data.CapellaForkVersion, - CappellaForkEpoch: *jr.Data.CapellaForkEpoch, + CapellaForkVersion: jr.Data.CapellaForkVersion, + CapellaForkEpoch: *jr.Data.CapellaForkEpoch, DenebForkVersion: jr.Data.DenebForkVersion, DenebForkEpoch: *jr.Data.DenebForkEpoch, SecondsPerSlot: uint64(jr.Data.SecondsPerSlot), diff --git a/backend/pkg/commons/utils/eth.go b/backend/pkg/commons/utils/eth.go index 73b9b8bb2..0268cd3b4 100644 --- a/backend/pkg/commons/utils/eth.go +++ b/backend/pkg/commons/utils/eth.go @@ -228,11 +228,11 @@ func SyncPeriodOfEpoch(epoch uint64) uint64 { if epoch < Config.Chain.ClConfig.AltairForkEpoch { return 0 } - return epoch / Config.Chain.ClConfig.EpochsPerSyncCommitteePeriod + return (epoch - Config.Chain.ClConfig.AltairForkEpoch) / Config.Chain.ClConfig.EpochsPerSyncCommitteePeriod } func FirstEpochOfSyncPeriod(syncPeriod uint64) uint64 { - return syncPeriod * Config.Chain.ClConfig.EpochsPerSyncCommitteePeriod + return Config.Chain.ClConfig.AltairForkEpoch + (syncPeriod * Config.Chain.ClConfig.EpochsPerSyncCommitteePeriod) } func SlotsPerSyncCommittee() uint64 { diff --git a/backend/pkg/commons/utils/utils.go b/backend/pkg/commons/utils/utils.go index 20c45e829..8301863bf 100644 --- a/backend/pkg/commons/utils/utils.go +++ b/backend/pkg/commons/utils/utils.go @@ -97,31 +97,31 @@ func SliceContains(list []string, target string) bool { // ForkVersionAtEpoch returns the forkversion active a specific epoch func ForkVersionAtEpoch(epoch uint64) *types.ForkVersion { - if epoch >= Config.Chain.ClConfig.CappellaForkEpoch { + if epoch >= Config.Chain.ClConfig.CapellaForkEpoch { return &types.ForkVersion{ - Epoch: Config.Chain.ClConfig.CappellaForkEpoch, - CurrentVersion: MustParseHex(Config.Chain.ClConfig.CappellaForkVersion), - PreviousVersion: MustParseHex(Config.Chain.ClConfig.BellatrixForkVersion), + Epoch: Config.Chain.ClConfig.CapellaForkEpoch, + CurrentVersion: Config.Chain.ClConfig.CapellaForkVersion, + PreviousVersion: Config.Chain.ClConfig.BellatrixForkVersion, } } if epoch >= Config.Chain.ClConfig.BellatrixForkEpoch { return &types.ForkVersion{ Epoch: Config.Chain.ClConfig.BellatrixForkEpoch, - CurrentVersion: MustParseHex(Config.Chain.ClConfig.BellatrixForkVersion), - PreviousVersion: MustParseHex(Config.Chain.ClConfig.AltairForkVersion), + CurrentVersion: Config.Chain.ClConfig.BellatrixForkVersion, + PreviousVersion: Config.Chain.ClConfig.AltairForkVersion, } } if epoch >= Config.Chain.ClConfig.AltairForkEpoch { return &types.ForkVersion{ Epoch: Config.Chain.ClConfig.AltairForkEpoch, - CurrentVersion: MustParseHex(Config.Chain.ClConfig.AltairForkVersion), - PreviousVersion: MustParseHex(Config.Chain.ClConfig.GenesisForkVersion), + CurrentVersion: Config.Chain.ClConfig.AltairForkVersion, + PreviousVersion: Config.Chain.ClConfig.GenesisForkVersion, } } return &types.ForkVersion{ Epoch: 0, - CurrentVersion: MustParseHex(Config.Chain.ClConfig.GenesisForkVersion), - PreviousVersion: MustParseHex(Config.Chain.ClConfig.GenesisForkVersion), + CurrentVersion: Config.Chain.ClConfig.GenesisForkVersion, + PreviousVersion: Config.Chain.ClConfig.GenesisForkVersion, } } diff --git a/backend/pkg/consapi/client_node.go b/backend/pkg/consapi/client_node.go index a0c136d01..7910425bc 100644 --- a/backend/pkg/consapi/client_node.go +++ b/backend/pkg/consapi/client_node.go @@ -3,6 +3,7 @@ package consapi import ( "fmt" "net/http" + "net/url" "strings" "time" @@ -10,6 +11,7 @@ import ( "github.com/gobitfly/beaconchain/pkg/consapi/network" "github.com/gobitfly/beaconchain/pkg/consapi/types" "github.com/gobitfly/beaconchain/pkg/consapi/utils" + "github.com/klauspost/compress/gzhttp" ) func NewClient(endpoint string) Client { @@ -18,8 +20,15 @@ func NewClient(endpoint string) Client { func NewClientWithConfig(endpoint string, httpClient *http.Client) Client { if httpClient == nil { + tr := &http.Transport{} + tr.ResponseHeaderTimeout = 60 * time.Second + tr.TLSHandshakeTimeout = 30 * time.Second + tr.DisableCompression = false // we want compression if we can get it. json is very compressible + gztr := gzhttp.Transport(tr, gzhttp.TransportEnableZstd(false)) + httpClient = &http.Client{ - Timeout: 500 * time.Second, + Transport: gztr, + Timeout: 120 * time.Second, } } @@ -93,7 +102,6 @@ func (r *NodeClient) GetValidators(state any, ids []string, status []types.Valid statusStr := strings.Join(utils.ConvertToStringSlice(status), ",") requestURL += fmt.Sprintf("status=%s", statusStr) } - return network.Get[types.StandardValidatorsResponse](r.httpClient, requestURL) } @@ -154,7 +162,26 @@ func (r *NodeClient) GetEvents(topics []types.EventTopic) chan *types.EventRespo req.Header.Set("accept-encoding", "identity") go func() { - stream, err := eventsource.SubscribeWithRequest("", req) + // create a client with compression disabled + // compression can cause delayed events due to chunked encoding + client := &http.Client{ + Transport: &http.Transport{ + DisableCompression: true, + }, + } + url, err := url.Parse(requestURL) + if err != nil { + panic(err) + } + request := &http.Request{ + Method: http.MethodGet, + URL: url, + Header: http.Header{ + "Accept": []string{"text/event-stream"}, + }, + } + stream, err := eventsource.SubscribeWith(requestURL, client, request) + //stream.Logger = log.New(os.Stdout, "eventsource: ", log.LstdFlags) if err != nil { responseCh <- &types.EventResponse{Error: err} diff --git a/backend/pkg/consapi/network/network.go b/backend/pkg/consapi/network/network.go index b6e34fc0a..a102c3288 100644 --- a/backend/pkg/consapi/network/network.go +++ b/backend/pkg/consapi/network/network.go @@ -8,30 +8,66 @@ import ( "net/http" "time" + "github.com/gobitfly/beaconchain/pkg/commons/log" "github.com/gobitfly/beaconchain/pkg/consapi/utils" ) // Helper for get and unmarshal func Get[T any](r *http.Client, url string) (*T, error) { - result, err := HTTPReq("GET", url, r) - if err != nil || result == nil { - var target T - return &target, err - } - return utils.Unmarshal[T](result, err) + return retry[T]("GET", r, url) } // Helper for post and unmarshal func Post[T any](r *http.Client, url string) (*T, error) { - result, err := HTTPReq("POST", url, r) - if err != nil || result == nil { - var target T - return &target, err + return retry[T]("POST", r, url) +} + +func retry[T any](method string, r *http.Client, url string) (*T, error) { + const maxRetries = 16 // Maximum number of retries + var backoffTime = 1 * time.Second // Initial backoff time + + var resp *http.Response + var err error + var e *T + tmr := time.AfterFunc(60*time.Second, func() { + log.WarnWithFields(log.Fields{"url": url}, fmt.Sprintf("%s request taking more than 60 seconds", method)) + }) + defer tmr.Stop() + + for attempt := 0; attempt < maxRetries; attempt++ { + start := time.Now() + resp, err = HTTPReq(method, url, r) + + if resp != nil { + e, err = utils.Unmarshal[T](resp.Body, err) + } + if time.Since(start) > 30*time.Second { + log.Debugf("%s %s took %s", method, url, time.Since(start)) + } + if err == nil { + defer resp.Body.Close() + break + } + // retry as long as it isn't a 404 http error or there has been no error + if resp != nil && resp.StatusCode == http.StatusNotFound { + break + } + httpErr := SpecificError(err) + if httpErr != nil && httpErr.StatusCode == http.StatusNotFound { + break + } + + log.Warnf("Attempt %d for %s %s failed: %v. Retrying in %v...", attempt+1, method, url, err, backoffTime) + time.Sleep(backoffTime) } - return utils.Unmarshal[T](result, err) + if err != nil { + err = fmt.Errorf("after %d attempts, last error: %w", maxRetries, err) + } + + return e, err } -func HTTPReq(method string, requestURL string, httpClient *http.Client) (io.ReadCloser, error) { +func HTTPReq(method string, requestURL string, httpClient *http.Client) (*http.Response, error) { data := []byte{} if method == "POST" { data = []byte("[]") @@ -42,7 +78,7 @@ func HTTPReq(method string, requestURL string, httpClient *http.Client) (io.Read } if httpClient == nil { - httpClient = &http.Client{Timeout: 20 * time.Second} + return nil, errors.New("httpClient is nil") } r.Header.Add("Content-Type", "application/json") @@ -61,7 +97,7 @@ func HTTPReq(method string, requestURL string, httpClient *http.Client) (io.Read } } - return res.Body, nil + return res, nil } type RPCErrorMessage struct { diff --git a/backend/pkg/consapi/types/slot.go b/backend/pkg/consapi/types/slot.go index cfc72acfa..ce6ab5003 100644 --- a/backend/pkg/consapi/types/slot.go +++ b/backend/pkg/consapi/types/slot.go @@ -42,6 +42,23 @@ type AnySignedBlock struct { Signature hexutil.Bytes `json:"signature"` } +type LightAnySignedBlock struct { + BlockRoot hexutil.Bytes + ParentRoot hexutil.Bytes + Slot uint64 + ProposerIndex uint64 + SlashedIndices []uint64 + Attestations []Attestation + Deposits []Deposit + Withdrawals []LightWithdrawal + SyncAggregate *SyncAggregate +} + +type LightWithdrawal struct { + ValidatorIndex uint64 + Amount uint64 +} + type ProposerSlashing struct { SignedHeader1 struct { Message struct { diff --git a/backend/pkg/consapi/types/validator.go b/backend/pkg/consapi/types/validator.go index 4a2cd0674..398e2ceea 100644 --- a/backend/pkg/consapi/types/validator.go +++ b/backend/pkg/consapi/types/validator.go @@ -38,6 +38,20 @@ type StandardValidatorsResponse struct { Data []StandardValidator `json:"data"` } +type LightStandardValidatorsResponse struct { + Epoch uint64 + Data []LightStandardValidator +} + +type UltraLightStandardValidatorsResponse struct { + Data []UltraLightStandardValidator +} +type UltraLightStandardValidator struct { + Index uint64 + EffectiveBalance uint64 + Status ValidatorStatus +} + // eth/v1/beacon/states/{state_id}/validators/{validator_id} type StandardSingleValidatorsResponse struct { ExecutionOptimistic bool `json:"execution_optimistic"` @@ -61,6 +75,15 @@ type StandardValidator struct { } `json:"validator"` } +type LightStandardValidator struct { + Index uint64 + Balance uint64 + Status ValidatorStatus + Pubkey hexutil.Bytes + EffectiveBalance uint64 + Slashed bool +} + // /eth/v1/validator/duties/proposer/{epoch} type StandardProposerAssignmentsResponse struct { DependentRoot hexutil.Bytes `json:"dependent_root"` diff --git a/backend/pkg/consapi/utils/marshal.go b/backend/pkg/consapi/utils/marshal.go index 318687a2f..e3d55e108 100644 --- a/backend/pkg/consapi/utils/marshal.go +++ b/backend/pkg/consapi/utils/marshal.go @@ -1,10 +1,10 @@ package utils import ( - "encoding/json" "io" "github.com/pkg/errors" + "github.com/segmentio/encoding/json" ) func Unmarshal[T any](source io.ReadCloser, err error) (*T, error) { diff --git a/backend/pkg/exporter/db/db.go b/backend/pkg/exporter/db/db.go index 033d652ea..61b8eeab9 100644 --- a/backend/pkg/exporter/db/db.go +++ b/backend/pkg/exporter/db/db.go @@ -2,8 +2,8 @@ package db import ( "bytes" + "context" "database/sql" - "encoding/json" "fmt" "regexp" @@ -12,6 +12,8 @@ import ( "strings" "time" + ch "github.com/ClickHouse/clickhouse-go/v2" + "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/gobitfly/beaconchain/pkg/commons/db" "github.com/gobitfly/beaconchain/pkg/commons/log" "github.com/gobitfly/beaconchain/pkg/commons/metrics" @@ -19,12 +21,10 @@ import ( "github.com/gobitfly/beaconchain/pkg/commons/types" "github.com/gobitfly/beaconchain/pkg/commons/utils" constypes "github.com/gobitfly/beaconchain/pkg/consapi/types" + "github.com/google/uuid" "github.com/jmoiron/sqlx" "github.com/lib/pq" - "github.com/pkg/errors" "github.com/shopspring/decimal" - - "github.com/attestantio/go-eth2-client/spec/phase0" ) func SaveBlock(block *types.Block, forceSlotUpdate bool, tx *sqlx.Tx) error { @@ -898,168 +898,561 @@ func SaveEpoch(epoch uint64, validators []*types.Validator, client rpc.Client, t return nil } -func GetLatestDashboardEpoch() (uint64, error) { - var lastEpoch uint64 - err := db.AlloyWriter.Get(&lastEpoch, fmt.Sprintf("SELECT COALESCE(max(epoch), 0) FROM %s", EpochWriterTableName)) - return lastEpoch, err +type EpochMetadata struct { + Epoch uint64 `ch:"epoch" db:"epoch"` + InsertBatchID *uuid.UUID `ch:"insert_batch_id" db:"insert_batch_id"` + SuccessfulInsert *time.Time `ch:"successful_insert" db:"successful_insert"` + TransferBatchId *uuid.UUID `ch:"transfer_batch_id" db:"transfer_batch_id"` + SuccessfulTransfer *time.Time `ch:"successful_transfer" db:"successful_transfer"` } -func GetOldestDashboardEpoch() (uint64, error) { - var epoch uint64 - err := db.AlloyWriter.Get(&epoch, fmt.Sprintf("SELECT COALESCE(min(epoch), 0) FROM %s", EpochWriterTableName)) - return epoch, err +// +// |-GetIncompleteTransferEpochs +// | - TransferEpochs +// | - PushEpochMetadata (successful_transfer) +// |-GetPendingTransferEpochs +// | - allocate transfer batch ids +// | - PushEpochMetadata (transfer_batch_id) +// | - TransferEpochs +// | - PushEpochMetadata (successful_transfer) + +func TransferEpochs(epochs []EpochMetadata) error { + start := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_transfer_batch").Observe(time.Since(start).Seconds()) + }() + // sanity check, verify that the transfer batch id is set and identical for all epochs + transferBatchID := epochs[0].TransferBatchId + for _, e := range epochs { + if e.TransferBatchId == nil || *e.TransferBatchId != *transferBatchID { + return fmt.Errorf("transfer batch id is not set or not identical for all epochs") + } + } + // sort the epochs + sort.Slice(epochs, func(i, j int) bool { + return epochs[i].Epoch < epochs[j].Epoch + }) + // transfer the epochs + abortCtx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + ctx := ch.Context(abortCtx, ch.WithSettings(ch.Settings{ + "insert_deduplication_token": transferBatchID.String(), + "insert_deduplicate": true, + "select_sequential_consistency": 1, + "use_skip_indexes_if_final": 1, // this is only safe because our index is over a column from the primary key + })) + now := time.Now() + // sanity check, check that there are more than a thousand entries for each epoch + const minEpochEntries = 1000 + for _, e := range epochs { + var count int + err := db.ClickHouseReader.Get(&count, fmt.Sprintf(` + SELECT count() as count + FROM %s + FINAL + WHERE epoch_timestamp = $1 + SETTINGS select_sequential_consistency = 1, use_skip_indexes_if_final = 1 + `, UnsafeEpochsTableName), utils.EpochToTime(e.Epoch)) + if err != nil { + return fmt.Errorf("error fetching epoch count: %w", err) + } + if count < minEpochEntries { + return fmt.Errorf("epoch %v has less than 1000 entries in the unsafe table", e.Epoch) + } + } + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_transfer_sanity_check").Observe(time.Since(now).Seconds()) + now = time.Now() + var epoch_timestamp []time.Time + for _, e := range epochs { + epoch_timestamp = append(epoch_timestamp, utils.EpochToTime(e.Epoch)) + } + err := db.ClickHouseNativeWriter.Exec(ctx, + fmt.Sprintf(` + insert into %s + select + * EXCEPT _inserted_at + from + %s FINAL + where + epoch_timestamp in $1 + `, FinalEpochsTableName, UnsafeEpochsTableName), + epoch_timestamp, + ) + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_transfer_insert").Observe(time.Since(now).Seconds()) + if err != nil { + return fmt.Errorf("error transferring epochs: %w", err) + } + return nil } -func GetMinOldHourlyEpoch() (uint64, error) { - var epoch uint64 - err := db.AlloyWriter.Get(&epoch, fmt.Sprintf("SELECT min(epoch_start) as epoch_start FROM %s", HourWriterTableName)) - return epoch, err +func GetIncompleteInsertEpochs() ([]EpochMetadata, error) { // no limit because it should never grow too large + var epochs []EpochMetadata + err := db.ClickHouseReader.Select(&epochs, + fmt.Sprintf(` + SELECT * + FROM %s + FINAL + WHERE + (successful_insert IS NULL OR successful_insert < now() - interval 5 day) AND + (insert_batch_id IS NOT NULL) AND + (successful_transfer IS NULL) + ORDER BY epoch ASC + SETTINGS select_sequential_consistency = 1 + `, ExporterMetadataTableName)) + if err != nil { + return nil, fmt.Errorf("error fetching incomplete insert epochs: %w", err) + } + return epochs, nil } -type EpochBounds struct { - EpochStart uint64 `db:"epoch_start"` - EpochEnd uint64 `db:"epoch_end"` +func GetLatestFinishedEpoch() (int64, error) { + var epoch int64 + err := db.ClickHouseReader.Get(&epoch, fmt.Sprintf(` + SELECT ifNull(max(toNullable(epoch::Int64)), -1) as epoch + FROM %s + FINAL + WHERE successful_transfer IS NOT NULL + SETTINGS select_sequential_consistency = 1 + `, ExporterMetadataTableName)) + if err != nil { + return 0, fmt.Errorf("error fetching latest finished epoch: %w", err) + } + return epoch, nil } -type DayBounds struct { - Day time.Time `db:"day"` - EpochStart uint64 `db:"epoch_start"` - EpochEnd uint64 `db:"epoch_end"` +func GetOldestUnfinishedTransferEpoch() (int64, error) { + var epoch int64 + err := db.ClickHouseReader.Get(&epoch, fmt.Sprintf(` + SELECT ifNull(min(toNullable(epoch::Int64)), -1) as epoch + FROM %s + FINAL + WHERE successful_transfer IS NULL + SETTINGS select_sequential_consistency = 1 + `, ExporterMetadataTableName)) + if err != nil { + return 0, fmt.Errorf("error fetching oldest unfinished transfer epoch: %w", err) + } + return epoch, nil } -func GetLastExportedTotalEpoch() (*EpochBounds, error) { - var epoch EpochBounds - err := db.AlloyWriter.Get(&epoch, fmt.Sprintf("SELECT COALESCE(max(epoch_start),0) as epoch_start, COALESCE(max(epoch_end),0) as epoch_end FROM %s", RollingTotalWriterTableName)) - return &epoch, err +func GetLatestUnsafeEpoch() (int64, error) { + var epoch int64 + err := db.ClickHouseReader.Get(&epoch, fmt.Sprintf(` + SELECT ifNull(max(toNullable(epoch::Int64)), -1) as epoch + FROM %s + FINAL + WHERE successful_insert IS NOT NULL + SETTINGS select_sequential_consistency = 1 + `, ExporterMetadataTableName)) + if err != nil { + return 0, fmt.Errorf("error fetching latest unsafe epoch: %w", err) + } + return epoch, nil } -func GetLastExportedHour() (*EpochBounds, error) { - var epoch EpochBounds - err := db.AlloyWriter.Get(&epoch, fmt.Sprintf("SELECT COALESCE(max(epoch_start),0) as epoch_start, COALESCE(max(epoch_end),0) as epoch_end FROM %s", HourWriterTableName)) - return &epoch, err -} +// enum for rollings (hourly, daily, weekly, monthly, total) +type Rollings string -func GetLastExportedDay() (*DayBounds, error) { - var epoch DayBounds - err := db.AlloyWriter.Get(&epoch, fmt.Sprintf("SELECT day, epoch_start, epoch_end FROM %[1]s WHERE day = (select max(day) from %[1]s) limit 1;", DayWriterTableName)) - return &epoch, err -} +const ( + Rolling1h Rollings = `validator_dashboard_rolling_1h` + Rolling24h Rollings = `validator_dashboard_rolling_24h` + Rolling7d Rollings = `validator_dashboard_rolling_7d` + Rolling30d Rollings = `validator_dashboard_rolling_30d` + Rolling90d Rollings = `validator_dashboard_rolling_90d` + RollingTotal Rollings = `validator_dashboard_rolling_total` +) -func HasDashboardDataForEpoch(targetEpoch uint64) (bool, error) { - var epoch uint64 - err := db.AlloyWriter.Get(&epoch, fmt.Sprintf("SELECT epoch FROM %s WHERE epoch = $1 LIMIT 1", EpochWriterTableName), targetEpoch) - if err != nil { - if err == sql.ErrNoRows { - return false, nil - } - return false, err +func (r *Rollings) GetDuration() time.Duration { + switch *r { + case Rolling1h: + return time.Hour + case Rolling24h: + return 24 * time.Hour + case Rolling7d: + return 7 * 24 * time.Hour + case Rolling30d: + return 30 * 24 * time.Hour + case Rolling90d: + return 90 * 24 * time.Hour + case RollingTotal: + return 25 * 365 * 24 * time.Hour // 25 years } - return true, nil + return 0 } -// returns epochs between start and end that are missing in the database, start is inclusive end is exclusive -func GetMissingEpochsBetween(start, end int64) ([]uint64, error) { - if start < 0 { - start = 0 - } - if end <= start { - return nil, nil +func NukeUnsafeRollingTable(rolling Rollings) error { + _, err := db.ClickHouseWriter.Exec(fmt.Sprintf(` + TRUNCATE TABLE _unsafe_%s + `, rolling)) + if err != nil { + return fmt.Errorf("error truncating table %s: %w", rolling, err) } + return nil +} - if end-start > 100 { - // for large ranges we use a different approach to avoid making tons of selects - // this performs better for large ranges but is slow for short ranges - var epochs []uint64 - err := db.AlloyWriter.Select(&epochs, fmt.Sprintf(` - WITH - epoch_range AS ( - SELECT generate_series($1::bigint, $2::bigint) AS epoch - ), - distinct_present_epochs AS ( - SELECT DISTINCT epoch - FROM %s - WHERE epoch >= $1 AND epoch <= $2 - ) - SELECT epoch_range.epoch - FROM epoch_range - LEFT JOIN distinct_present_epochs ON epoch_range.epoch = distinct_present_epochs.epoch - WHERE distinct_present_epochs.epoch IS NULL - ORDER BY epoch_range.epoch - `, EpochWriterTableName), start, end-1) - return epochs, err +func GetRollingLastEpoch(rolling Rollings) (int64, error) { + // following doesnt handle epoch 0 correctly. fixing is left as an exercise for the reader + var epoch int64 + // -1 if empty table + err := db.ClickHouseReader.Get(&epoch, fmt.Sprintf(` + SELECT ifNull(max(toNullable(epoch_end::Int64)), -1) as epoch + FROM _final_%s + FINAL + SETTINGS select_sequential_consistency = 1 + `, rolling)) + if err != nil { + return 0, fmt.Errorf("error fetching latest finished epoch for rolling %s: %w", rolling, err) } + return epoch, nil +} - query := `SELECT TO_JSON(ARRAY_AGG(epoch)) AS result_array FROM (` - - for epoch := start; epoch < end; epoch++ { - if epoch != start { - query += " UNION " - } - query += fmt.Sprintf(`SELECT %[1]d AS epoch WHERE NOT EXISTS (SELECT 1 FROM %[2]s WHERE epoch = %[1]d LIMIT 1)`, epoch, EpochWriterTableName) - } +type RollingSources string - query += `) AS result_array;` +const ( + RollingSourceEpochly RollingSources = `_final_validator_dashboard_data_epoch` + RollingSourceHourly RollingSources = `_final_validator_dashboard_data_hourly` + RollingSourceDaily RollingSources = `_final_validator_dashboard_data_daily` + RollingSourceMonthly RollingSources = `_final_validator_dashboard_data_monthly` +) - var jsonArray sql.NullString +type MinMax struct { + Min *time.Time + Max *time.Time +} - err := db.AlloyReader.Get(&jsonArray, query) +func GetMinMaxForRollingSource(table RollingSources, start time.Time, end *time.Time) (*MinMax, error) { + var result MinMax + column := "t" + if table == RollingSourceEpochly { // we were so close to greatness + column = "epoch_timestamp" + } + keys := []string{column + " >= ?"} + values := []interface{}{start} + if end != nil { + keys = append(keys, column+" < ?") + values = append(values, *end) + } + err := db.ClickHouseReader.Get(&result, fmt.Sprintf(` + SELECT min(toNullable(%[1]s)) as min, max(toNullable(%[1]s)) as max + FROM %[2]s + WHERE %[3]s + SETTINGS select_sequential_consistency = 1 + `, column, table, strings.Join(keys, " and ")), values...) if err != nil { - return nil, errors.Wrap(err, "failed to query") + return nil, fmt.Errorf("error fetching min max for rolling source %s: %w", table, err) } - - if !jsonArray.Valid { + if result.Min == nil || result.Max == nil { return nil, nil } + return &result, nil +} - missingEpochs := make([]uint64, 0) - err = json.Unmarshal([]byte(jsonArray.String), &missingEpochs) +func TransferRollingSourceToRolling(rolling Rollings, source RollingSources, minMax MinMax) error { + // transfer the epochs + abortCtx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + ctx := ch.Context(abortCtx, ch.WithSettings(ch.Settings{ + "select_sequential_consistency": 1, + "use_skip_indexes_if_final": 1, // this is only safe because our index is over a column from the primary key + "max_threads": 2, + })) + column := "t" + selector := ` + validator_index AS validator_index, + any(foo.t) AS t, + + groupArraySortedIfMergeState(2048)(epoch_map) AS epoch_map, + min(epoch_start) AS epoch_start, + max(epoch_end) AS epoch_end, + + argMinStateMerge(balance_start) AS balance_start, + argMaxStateMerge(balance_end) AS balance_end, + min(balance_min) AS balance_min, + max(balance_max) AS balance_max, + + sum(deposits_count) AS deposits_count, + sum(deposits_amount) AS deposits_amount, + sum(withdrawals_count) AS withdrawals_count, + sum(withdrawals_amount) AS withdrawals_amount, + + sum(attestations_scheduled) AS attestations_scheduled, + sum(attestations_observed) AS attestations_observed, + sum(attestations_head_matched) AS attestations_head_matched, + sum(attestations_target_matched) AS attestations_target_matched, + sum(attestations_source_matched) AS attestations_source_matched, + + sum(attestations_head_executed) AS attestations_head_executed, + sum(attestations_target_executed) AS attestations_target_executed, + sum(attestations_source_executed) AS attestations_source_executed, + + sum(attestations_head_reward_rewards_only) AS attestations_head_reward_rewards_only, + sum(attestations_head_reward_penalties_only) AS attestations_head_reward_penalties_only, + + sum(attestations_target_reward_rewards_only) AS attestations_target_reward_rewards_only, + sum(attestations_target_reward_penalties_only) AS attestations_target_reward_penalties_only, + + sum(attestations_source_reward_rewards_only) AS attestations_source_reward_rewards_only, + sum(attestations_source_reward_penalties_only) AS attestations_source_reward_penalties_only, + + sum(attestations_inactivity_reward_rewards_only) AS attestations_inactivity_reward_rewards_only, + sum(attestations_inactivity_reward_penalties_only) AS attestations_inactivity_reward_penalties_only, + + sum(attestations_inclusion_reward_rewards_only) AS attestations_inclusion_reward_rewards_only, + sum(attestations_inclusion_reward_penalties_only) AS attestations_inclusion_reward_penalties_only, + + sum(attestations_ideal_head_reward) AS attestations_ideal_head_reward, + sum(attestations_ideal_target_reward) AS attestations_ideal_target_reward, + sum(attestations_ideal_source_reward) AS attestations_ideal_source_reward, + + sum(attestations_ideal_inactivity_reward) AS attestations_ideal_inactivity_reward, + sum(attestations_ideal_inclusion_reward) AS attestations_ideal_inclusion_reward, + + sum(attestations_localized_max_reward) AS attestations_localized_max_reward, + sum(attestations_hyperlocalized_max_reward) AS attestations_hyperlocalized_max_reward, + + sum(inclusion_delay_sum) AS inclusion_delay_sum, + sum(optimal_inclusion_delay_sum) AS optimal_inclusion_delay_sum, + + sum(blocks_scheduled) AS blocks_scheduled, + sum(blocks_proposed) AS blocks_proposed, + sum(blocks_cl_reward) AS blocks_cl_reward, + sum(blocks_cl_attestations_reward) AS blocks_cl_attestations_reward, + sum(blocks_cl_sync_aggregate_reward) AS blocks_cl_sync_aggregate_reward, + sum(blocks_cl_slasher_reward) AS blocks_cl_slasher_reward, + sum(blocks_cl_missed_median_reward) AS blocks_cl_missed_median_reward, + sum(blocks_slashing_count) AS blocks_slashing_count, + sum(blocks_expected) AS blocks_expected, + + sum(sync_scheduled) AS sync_scheduled, + sum(sync_executed) AS sync_executed, + sum(sync_reward_rewards_only) AS sync_reward_rewards_only, + sum(sync_reward_penalties_only) AS sync_reward_penalties_only, + sum(sync_localized_max_reward) AS sync_localized_max_reward, + sum(sync_committees_expected) AS sync_committees_expected, + max(slashed) AS slashed, + max(last_executed_duty_epoch) AS last_executed_duty_epoch, + max(last_scheduled_sync_epoch) AS last_scheduled_sync_epoch, + max(last_scheduled_block_epoch) AS last_scheduled_block_epoch + ` + if source == RollingSourceEpochly { + column = "epoch_timestamp" + // this is gonna be ugly. but cant avoid sadly without code generation + selector = ` + validator_index AS validator_index, + any(epoch_timestamp) AS t, + + groupArraySortedIfState(2048)(-foo.epoch, validator_index = 0) AS epoch_map, + min(foo.epoch) AS epoch_start, + max(foo.epoch) AS epoch_end, + + argMinState(foo.balance_start, foo.epoch) AS balance_start, + argMaxState(foo.balance_end, foo.epoch) AS balance_end, + least(min(foo.balance_start), min(foo.balance_end)) AS balance_min, + greatest(max(foo.balance_start), max(foo.balance_end)) AS balance_max, + + sum(deposits_count) AS deposits_count, + sum(deposits_amount) AS deposits_amount, + sum(withdrawals_count) AS withdrawals_count, + sum(withdrawals_amount) AS withdrawals_amount, + + sum(attestations_scheduled) AS attestations_scheduled, + sum(attestations_observed) AS attestations_observed, + sum(attestations_head_matched) AS attestations_head_matched, + sum(attestations_target_matched) AS attestations_target_matched, + sum(attestations_source_matched) AS attestations_source_matched, + + sum(attestations_head_executed) AS attestations_head_executed, + sum(attestations_target_executed) AS attestations_target_executed, + sum(attestations_source_executed) AS attestations_source_executed, + + sum(attestations_head_reward_rewards_only) AS attestations_head_reward_rewards_only, + sum(attestations_head_reward_penalties_only) AS attestations_head_reward_penalties_only, + + sum(attestations_target_reward_rewards_only) AS attestations_target_reward_rewards_only, + sum(attestations_target_reward_penalties_only) AS attestations_target_reward_penalties_only, + + sum(attestations_source_reward_rewards_only) AS attestations_source_reward_rewards_only, + sum(attestations_source_reward_penalties_only) AS attestations_source_reward_penalties_only, + + sum(attestations_inactivity_reward_rewards_only) AS attestations_inactivity_reward_rewards_only, + sum(attestations_inactivity_reward_penalties_only) AS attestations_inactivity_reward_penalties_only, + + sum(attestations_inclusion_reward_rewards_only) AS attestations_inclusion_reward_rewards_only, + sum(attestations_inclusion_reward_penalties_only) AS attestations_inclusion_reward_penalties_only, + + sum(attestations_ideal_head_reward) AS attestations_ideal_head_reward, + sum(attestations_ideal_target_reward) AS attestations_ideal_target_reward, + sum(attestations_ideal_source_reward) AS attestations_ideal_source_reward, + + sum(attestations_ideal_inactivity_reward) AS attestations_ideal_inactivity_reward, + sum(attestations_ideal_inclusion_reward) AS attestations_ideal_inclusion_reward, + + sum(attestations_localized_max_reward) AS attestations_localized_max_reward, + sum(attestations_hyperlocalized_max_reward) AS attestations_hyperlocalized_max_reward, + + sum(inclusion_delay_sum) AS inclusion_delay_sum, + sum(optimal_inclusion_delay_sum) AS optimal_inclusion_delay_sum, + + sum(blocks_scheduled) AS blocks_scheduled, + sum(blocks_proposed) AS blocks_proposed, + sum(blocks_cl_reward) AS blocks_cl_reward, + sum(blocks_cl_attestations_reward) AS blocks_cl_attestations_reward, + sum(blocks_cl_sync_aggregate_reward) AS blocks_cl_sync_aggregate_reward, + sum(blocks_cl_slasher_reward) AS blocks_cl_slasher_reward, + sum(blocks_cl_missed_median_reward) AS blocks_cl_missed_median_reward, + + sum(blocks_slashing_count) AS blocks_slashing_count, + sum(blocks_expected) AS blocks_expected, + sum(sync_scheduled) AS sync_scheduled, + sum(sync_executed) AS sync_executed, + sum(sync_reward_rewards_only) AS sync_reward_rewards_only, + sum(sync_reward_penalties_only) AS sync_reward_penalties_only, + sum(sync_localized_max_reward) AS sync_localized_max_reward, + sum(sync_committees_expected) AS sync_committees_expected, + max(slashed) AS slashed, + maxIfOrNull(foo.epoch, (foo.blocks_proposed != 0) OR (foo.sync_executed != 0) OR (foo.attestations_observed != 0)) AS last_executed_duty_epoch, + maxIfOrNull(foo.epoch, foo.sync_scheduled != 0) AS last_scheduled_sync_epoch, + maxIfOrNull(foo.epoch, foo.blocks_proposed != 0) AS last_scheduled_block_epoch + ` + } + err := db.ClickHouseNativeWriter.Exec(ctx, + fmt.Sprintf(` + insert into _unsafe_%[1]s + select + %[2]s + from + %[3]s foo -- we dont use final because the target table will do the merge anyways and the filter statement isnt affected by it + where + foo.%[4]s >= ? and foo.%[4]s <= ? + group by + validator_index + `, rolling, selector, source, column), *minMax.Min, *minMax.Max) if err != nil { - return nil, errors.Wrap(err, "failed to unmarshal") + return fmt.Errorf("error transferring epochs: %w", err) } + return nil +} - // sort asc - sort.Slice(missingEpochs, func(i, j int) bool { - return missingEpochs[i] < missingEpochs[j] - }) +func SwapRollingTables(rolling Rollings) error { + // swaps _unsafe_rolling with _final_rolling + _, err := db.ClickHouseWriter.Exec(fmt.Sprintf(` + EXCHANGE TABLES _unsafe_%[1]s AND _final_%[1]s + `, rolling)) + if err != nil { + return fmt.Errorf("error swapping tables %s: %w", rolling, err) + } + return nil +} - return missingEpochs, nil +func OptimizeUnsafeRollingTable(rolling Rollings) error { + _, err := db.ClickHouseWriter.Exec(fmt.Sprintf(` + OPTIMIZE TABLE _unsafe_%s FINAL + `, rolling)) + if err != nil { + return fmt.Errorf("error optimizing table %s: %w", rolling, err) + } + return nil } -func GetPartitionNamesOfTable(tableName string) ([]string, error) { - var partitions []string - err := db.AlloyWriter.Select(&partitions, fmt.Sprintf(` - SELECT inhrelid::regclass AS partition_name - FROM pg_inherits - WHERE inhparent = 'public.%s'::regclass order by 1;`, tableName), - ) - return partitions, err +func GetPendingInsertEpochs(maxEpoch int64, limit int64) ([]EpochMetadata, error) { // done + var epochs []EpochMetadata + // max epoch with assigned insert batch id + maxAssignedEpoch := int64(0) + err := db.ClickHouseReader.Get(&maxAssignedEpoch, fmt.Sprintf(` + SELECT ifNull(max(toNullable(epoch::Int64)), -1) as max_epoch + FROM %s + FINAL + WHERE (insert_batch_id IS NOT NULL) + SETTINGS select_sequential_consistency = 1 + `, ExporterMetadataTableName)) + if err != nil { + return nil, fmt.Errorf("error fetching max assigned epoch: %w", err) + } + // cap the max epoch to the limit + if maxAssignedEpoch > maxEpoch { + return nil, fmt.Errorf("max assigned epoch %v is greater than the max epoch %v", maxAssignedEpoch, maxEpoch) + } + if maxEpoch > maxAssignedEpoch+limit { + maxEpoch = maxAssignedEpoch + limit + } + for i := maxAssignedEpoch + 1; i <= maxEpoch; i++ { + epochs = append(epochs, EpochMetadata{Epoch: uint64(i)}) + } + return epochs, nil } -func AddToColumnEngine(table, columns string) error { - _, err := db.AlloyWriter.Exec(fmt.Sprintf(` - SELECT google_columnar_engine_add( - relation => '%s', - columns => '%s' - ); - `, table, columns)) - return err +func GetIncompleteTransferEpochs() ([]EpochMetadata, error) { // no limit because it should never grow too large + var epochs []EpochMetadata + err := db.ClickHouseReader.Select(&epochs, + fmt.Sprintf(` + SELECT * + FROM %[1]s + FINAL + WHERE + -- data has been inserted to the unsafe table + (successful_insert IS NOT NULL) AND + -- insert to unsafe table is not older than 5 days within any transfer batch + (transfer_batch_id NOT IN (select transfer_batch_id from %[1]s WHERE successful_insert < now() - interval 5 day)) AND + -- data has not been transferred to the final table + (successful_transfer IS NULL) AND + -- data has been assigned a transfer batch id + (transfer_batch_id IS NOT NULL) + ORDER BY epoch ASC + SETTINGS select_sequential_consistency = 1 + `, ExporterMetadataTableName)) + if err != nil { + return nil, fmt.Errorf("error fetching incomplete transfer epochs: %w", err) + } + return epochs, nil } -func AddToColumnEngineAllColumns(table string) error { - _, err := db.AlloyWriter.Exec(fmt.Sprintf(` - SELECT google_columnar_engine_add( - relation => '%s' - ); - `, table)) - return err +func GetPendingTransferEpochs(limit int64) ([]EpochMetadata, error) { + var epochs []EpochMetadata + err := db.ClickHouseReader.Select(&epochs, + fmt.Sprintf(` + SELECT * + FROM %[1]s + FINAL + WHERE + -- data has been inserted to the unsafe table + (successful_insert IS NOT NULL) AND + -- insert to unsafe table is not older than 5 days + (successful_insert >= now() - interval 5 day) AND + -- data has not been assigned a transfer batch id + (transfer_batch_id IS NULL) AND + -- data has not been transferred to the final table + (successful_transfer IS NULL) + ORDER BY epoch ASC + SETTINGS select_sequential_consistency = 1 + `, ExporterMetadataTableName)) + if err != nil { + return nil, fmt.Errorf("error fetching pending transfer epochs: %w", err) + } + return epochs, nil } -const EpochWriterTableName = "validator_dashboard_data_epoch" -const DayWriterTableName = "validator_dashboard_data_daily" -const HourWriterTableName = "validator_dashboard_data_hourly" +func PushEpochMetadata(metdata []EpochMetadata) error { + if len(metdata) == 0 { + return nil + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + batch, err := db.ClickHouseNativeWriter.PrepareBatch(ctx, `INSERT INTO `+ExporterMetadataTableName) + if err != nil { + return fmt.Errorf("error preparing batch: %w", err) + } + for _, m := range metdata { + if err := batch.AppendStruct(&m); err != nil { + return fmt.Errorf("error appending struct to batch: %w", err) + } + } + if err := batch.Send(); err != nil { + return fmt.Errorf("error sending batch: %w", err) + } + return nil +} -const RollingTotalWriterTableName = "validator_dashboard_data_rolling_total" -const RollingDailyWriterTable = "validator_dashboard_data_rolling_daily" -const RollingWeeklyWriterTable = "validator_dashboard_data_rolling_weekly" -const RollingMonthlyWriterTable = "validator_dashboard_data_rolling_monthly" -const RollingNinetyDaysWriterTable = "validator_dashboard_data_rolling_90d" +const ExporterMetadataTableName = "_exporter_metadata" // look i hate metadata tables as much as the next guy but this is a necessary evil +const EpochWriterSink = "_insert_sink_validator_dashboard_data_epoch" +const UnsafeEpochsTableName = "_unsafe_validator_dashboard_data_epoch" +const FinalEpochsTableName = "_final_validator_dashboard_data_epoch" diff --git a/backend/pkg/exporter/modules/base.go b/backend/pkg/exporter/modules/base.go index c23c0ca60..d2e63932a 100644 --- a/backend/pkg/exporter/modules/base.go +++ b/backend/pkg/exporter/modules/base.go @@ -72,7 +72,6 @@ func StartAll(context ModuleContext, modules []ModuleInterface, justV2 bool) { func startSubscriptionModules(context *ModuleContext, modules []ModuleInterface) { goPool := &errgroup.Group{} - log.Infof("initialising exporter modules") // Initialize modules @@ -97,6 +96,7 @@ func startSubscriptionModules(context *ModuleContext, modules []ModuleInterface) types.EventFinalizedCheckpoint, types.EventChainReorg, }) + log.Infof("subscribed to node events") for event := range events { if event.Error != nil { @@ -163,6 +163,8 @@ func notifyAllModules(goPool *errgroup.Group, modules []ModuleInterface, f func( } } func GetModuleContext() (ModuleContext, error) { + var moduleContext ModuleContext + cl := consapi.NewClient("http://" + utils.Config.Indexer.Node.Host + ":" + utils.Config.Indexer.Node.Port) spec, err := cl.GetSpec() @@ -183,11 +185,8 @@ func GetModuleContext() (ModuleContext, error) { if err != nil { log.Fatal(err, "error creating lighthouse client", 0) } - - moduleContext := ModuleContext{ - CL: cl, - ConsClient: clClient, - } + moduleContext.CL = cl + moduleContext.ConsClient = clClient return moduleContext, nil } @@ -217,6 +216,14 @@ func (m ModuleLog) Debugf(format string, args ...interface{}) { log.DebugWithFields(log.Fields{"module": m.module.GetName()}, fmt.Sprintf(format, args...)) } +func (m ModuleLog) Trace(message string) { + log.TraceWithFields(log.Fields{"module": m.module.GetName()}, message) +} + +func (m ModuleLog) Tracef(format string, args ...interface{}) { + log.TraceWithFields(log.Fields{"module": m.module.GetName()}, fmt.Sprintf(format, args...)) +} + func (m ModuleLog) InfoWithFields(additionalInfos log.Fields, msg string) { additionalInfos["module"] = m.module.GetName() log.InfoWithFields(additionalInfos, msg) diff --git a/backend/pkg/exporter/modules/dashboard_data.go b/backend/pkg/exporter/modules/dashboard_data.go index f7bb7d3f8..5981dbc70 100644 --- a/backend/pkg/exporter/modules/dashboard_data.go +++ b/backend/pkg/exporter/modules/dashboard_data.go @@ -1,93 +1,35 @@ package modules import ( - "database/sql" + "context" "fmt" - "math" "net/http" - "regexp" - "sort" - "strconv" - "strings" + "reflect" "sync" + "sync/atomic" "time" - "github.com/attestantio/go-eth2-client/spec/phase0" - "github.com/gobitfly/beaconchain/pkg/commons/db" "github.com/gobitfly/beaconchain/pkg/commons/metrics" "github.com/gobitfly/beaconchain/pkg/commons/utils" "github.com/gobitfly/beaconchain/pkg/consapi/network" constypes "github.com/gobitfly/beaconchain/pkg/consapi/types" - edb "github.com/gobitfly/beaconchain/pkg/exporter/db" "github.com/gobitfly/beaconchain/pkg/monitoring/constants" - "github.com/pkg/errors" - "github.com/prysmaticlabs/go-bitfield" - "golang.org/x/sync/errgroup" -) - -// -------------- DEBUG FLAGS ---------------- -// Normally rolling aggregation is only done when headEpochQueue exporter is near head so the exporter can catch up faster if behind, but for debugging purposes we can force it to be done every epoch -const debugAggregateMidEveryEpoch = false // prod: false - -// If set to 0 exporter will backfill to node finalized head, use any other value to backfill up to that specific epoch -const debugTargetBackfillEpoch = uint64(0) // prod: 0 - -// Once backfill is done the exporter will start to listen for new head epochs to export. You can disable this behavior by setting this flag to false -const debugSetBackfillCompleted = true // prod: true - -// Old epoch data is cleared from the database to save space and improve performance. This can be disabled for debugging purposes -const debugSkipOldEpochClear = false // prod: false - -// If set to true some tables like the epoch based table or hourly based table will be manually added to AlloyDBs Column Engine -const debugAddToColumnEngine = false // prod: true? - -// During backfill we can attempt to bootstrap the rolling tables on each UTC boundary day (as no tail fetching is needed here). So setting this will update the rolling tables every -// 225 epochs (for ETH mainnet) but at the slight cost of increased aggregation time for this particular boundary epoch. -const debugAggregateRollingWindowsDuringBackfillUTCBoundEpoch = true // prod: true + "github.com/google/uuid" -const debugDeadlockBandaid = true // prod: fix root cause then set to false + //"github.com/fjl/memsize/memsizeui" -// This flag can be used to force a bootstrap of the rolling tables. This is done once, after the bootstrap completes it switches back to off and normal rolling aggregation. -// Can be used to fix a corrupted rolling table. -var debugForceBootstrapRollingTables = false // prod: false - -// ----------- END OF DEBUG FLAGS ------------ - -// How many epochs will be fetched in parallel from the node (relevant for backfill and rolling tail fetching). We are fetching the head epoch and -// one epoch for each rolling table (tail), so if you want to fetch all epochs in one go (and your node can handle that) set this to at least 5. -const epochFetchParallelism = 5 - -// Fetching one epoch consists of multiple calls. You can define how many concurrent calls each epoch fetch will do. Keep in mind that -// the total number of concurrent requests is epochFetchParallelism * epochFetchParallelismWithinEpoch -const epochFetchParallelismWithinEpoch = 6 - -// How many epochs will be written in parallel to the database -const epochWriteParallelism = 4 - -// How many epoch aggregations will be executed in parallel (e.g. total, hour, day, each rolling table) -const databaseAggregationParallelism = 4 - -const nonRollingdatabaseAggregationParallelism = 1 // 1 for now to see if this fixes the "deadlocks" - -// How many epochFetchParallelism iterations will be written before a new aggregation will be triggered during backfill. This can speed up backfill as writing epochs to db is fast and we can delay -// aggregation for a couple iterations. Don't set too high or else epoch table will grow to large and will be a bottleneck. -// Set to 0 to disable and write after every iteration. Recommended value for this is 1 or maybe 2. -// Try increasing this one by one if node_fetch_time < agg_and_storage_time until it targets roughly agg_and_storage_time = node_fetch_time -// Try 0 if agg_and_storage_time is < node_fetch_time -const backfillMaxUnaggregatedIterations = 1 + "golang.org/x/sync/errgroup" + "golang.org/x/sync/semaphore" +) type dashboardData struct { ModuleContext log ModuleLog signingDomain []byte - epochWriter *epochWriter - epochToTotal *epochToTotalAggregator - epochToHour *epochToHourAggregator - epochToDay *epochToDayAggregator - dayUp *dayUpAggregator headEpochQueue chan uint64 backFillCompleted bool - responseCache ResponseCache + phase0HotfixMutex sync.Mutex + latestSafeEpoch atomic.Int64 } func NewDashboardDataModule(moduleContext ModuleContext) ModuleInterface { @@ -96,17 +38,6 @@ func NewDashboardDataModule(moduleContext ModuleContext) ModuleInterface { } temp.log = ModuleLog{module: temp} - // When a new epoch gets exported the very first step is to export it to the db via epochWriter - temp.epochWriter = newEpochWriter(temp) - - // Then those aggregators below use the epoch data to aggregate it into the respective tables - temp.epochToTotal = newEpochToTotalAggregator(temp) - temp.epochToHour = newEpochToHourAggregator(temp) - temp.epochToDay = newEpochToDayAggregator(temp) - - // Once an epoch is aggregated to its respective UTC day, we can use the UTC day table to aggregate up to the rolling window tables (7d, 30d, 90d) - temp.dayUp = newDayUpAggregator(temp) - // This channel is used to queue up epochs from chain head that need to be exported temp.headEpochQueue = make(chan uint64, 100) @@ -114,1786 +45,724 @@ func NewDashboardDataModule(moduleContext ModuleContext) ModuleInterface { // and the exporter can start listening for new head epochs to be processed temp.backFillCompleted = false - temp.responseCache = ResponseCache{ - cache: make(map[string]any), - } return temp } +type Task struct { + UUID uuid.UUID `db:"uuid"` + Hostname string `db:"hostname"` + Priority int64 `db:"priority"` + StartTs time.Time `db:"start_ts"` + EndTs time.Time `db:"end_ts"` + Status string `db:"status"` +} + func (d *dashboardData) Init() error { - go func() { - _, err := db.AlloyWriter.Exec("SET work_mem TO '128MB';") + // blocking loop trying to init d.latestSafeEpoch + for { + err := updateSafeEpoch(d) if err != nil { - d.log.Fatal(err, "failed to set work_mem", 0) - } - - start := time.Now() - for { - var upToEpochPtr *uint64 = nil // nil will backfill back to head - if debugTargetBackfillEpoch > 0 { - upToEpoch := debugTargetBackfillEpoch - upToEpochPtr = &upToEpoch - } - - result, err := d.backfillHeadEpochData(upToEpochPtr) - if err != nil { - d.log.Error(err, "failed to backfill epoch data", 0) - metrics.Errors.WithLabelValues("exporter_v2dash_backfill_fail").Inc() - time.Sleep(10 * time.Second) - continue - } - - if result.BackfilledToHead { - d.log.Infof("dashboard data up to date, starting head export") - if debugSetBackfillCompleted { - if time.Since(start) > time.Hour { - utils.SendMessage(fmt.Sprintf("🎉🎉🎉 v2 Dashboard %s - Reached head, exporting from head now", utils.Config.Chain.Name), &utils.Config.InternalAlerts) - } - d.backFillCompleted = true - } - break - } - time.Sleep(1 * time.Second) + d.log.Error(err, "failed to update safe epoch", 0) + time.Sleep(10 * time.Second) + continue } - - d.processHeadQueue() - }() + break + } + go d.insertTask() // does all the inserting of the data + go d.maintenanceTask() // does all the transferring of the data + go d.rollingTask() // does all the rolling of the data return nil } -func (d *dashboardData) processHeadQueue() { - reachedHead := false - for { - epoch := <-d.headEpochQueue - - // After initial sync or long downtime first head processing might take a long time, so by the time we finished - // the queue might have filled up significantly. To get back on head more quickly we skip some epochs and let the backfill handle those - // before processing the more recent epoch - for len(d.headEpochQueue) > 1 { - epoch = <-d.headEpochQueue - } - if len(d.headEpochQueue) == 0 && !reachedHead { - d.log.Infof("exporter is at head of the chain") - reachedHead = true - } - - startTime := time.Now() - d.log.Infof("exporting dashboard epoch data for epoch %d", epoch) - stage := 0 - doRollingAggregate := false - for { // retry this epoch until no errors occur - currentFinalizedEpoch, err := d.CL.GetFinalityCheckpoints("head") - if err != nil { - d.log.Error(err, "failed to get finalized checkpoint", 0) - metrics.Errors.WithLabelValues("exporter_v2dash_node_get_finalize_fail").Inc() - time.Sleep(time.Second * 10) - continue - } - - // Back fill to epoch -1 if necessary - var backfillResult backfillResult - if stage <= 0 { - targetEpoch := epoch - 1 - backfillResult, err = d.backfillHeadEpochData(&targetEpoch) - if err != nil { - d.log.Error(err, "failed to backfill head epoch data", 0, map[string]interface{}{"epoch": epoch}) - metrics.Errors.WithLabelValues("exporter_v2dash_backfill_fail").Inc() - time.Sleep(time.Second * 10) - continue - } - stage = 1 - } - - // Get epoch data from node and write to database - if stage <= 1 { - doRollingAggregate = currentFinalizedEpoch.Data.Finalized.Epoch <= epoch+1 // only near head - err := d.exportEpochAndTails(epoch, debugAggregateMidEveryEpoch || doRollingAggregate) - if err != nil { - d.log.Error(err, "failed to export epoch tail data", 0, map[string]interface{}{"epoch": epoch}) - metrics.Errors.WithLabelValues("exporter_v2dash_export_epoch_tail_fail").Inc() - time.Sleep(time.Second * 10) - continue - } - stage = 2 - } - - // Run aggregations - if stage <= 2 { - err := d.aggregatePerEpoch(debugAggregateMidEveryEpoch || doRollingAggregate, backfillResult.DidPerformBackfill) // keep epoch data if backfill was needed - if err != nil { - d.log.Error(err, "failed to aggregate", 0, map[string]interface{}{"epoch": epoch}) - metrics.Errors.WithLabelValues("exporter_v2dash_agg_fail").Inc() - time.Sleep(time.Second * 10) - continue - } - stage = 3 - } - - break - } +var EpochsWritten int +var FirstEpochWritten *time.Time - d.log.Infof("[time] completed dashboard epoch data for epoch %d in %v", epoch, time.Since(startTime)) - } +func (d *dashboardData) OnFinalizedCheckpoint(t *constypes.StandardFinalizedCheckpointResponse) error { + return nil } -// exports the provided headEpoch plus any tail epochs that are needed for rolling aggregation -// fE a tail epoch for rolling 1 day aggregation (225 epochs) for head 227 on ethereum would correspond to two tail epochs [0,1] -func (d *dashboardData) exportEpochAndTails(headEpoch uint64, fetchRollingTails bool) error { - missingTails := make([]uint64, 0) - var err error - if fetchRollingTails { - // for 24h aggregation - missingTails, err = d.epochToDay.getMissingRolling24TailEpochs(headEpoch) - if err != nil { - return errors.Wrap(err, "failed to get missing 24h tail epochs") - } - - d.log.Infof("missing 24h tails: %v", missingTails) - - // day aggregation - daysMissingTails, err := d.dayUp.getMissingRollingDayTailEpochs(headEpoch) - if err != nil { - return errors.Wrap(err, "failed to get missing day tail epochs") - } - - dayMissingHeads, err := d.dayUp.getMissingRollingDayHeadEpochs(headEpoch) - if err != nil { - return errors.Wrap(err, "failed to get missing day head epochs") - } - - // merge - missingTails = append(missingTails, utils.Deduplicate(append(daysMissingTails, dayMissingHeads...))...) - - if len(missingTails) > 10 { - d.log.Infof("This might take a bit longer than usual as exporter is catching up quite a lot old epochs, usually happens after downtime or after initial sync") - } - - // sort asc - sort.Slice(missingTails, func(i, j int) bool { - return missingTails[i] < missingTails[j] - }) - } - - hasHeadAlreadyExported, err := edb.HasDashboardDataForEpoch(headEpoch) +func updateSafeEpoch(d *dashboardData) error { + res, err := d.CL.GetFinalityCheckpoints("head") if err != nil { - return errors.Wrap(err, "failed to check if head epoch has dashboard data") + return err } - // append head - if !hasHeadAlreadyExported { - missingTails = append(missingTails, headEpoch) - d.log.Infof("fetch missing tail/head epochs: %v | fetch head: %d", len(missingTails)-1, headEpoch) - } else { - if len(missingTails) == 0 { - return nil // nothing to do - } - d.log.Infof("fetch missing tail/head epochs: %v | fetch head: -", len(missingTails)) - } + finalized := res.Data.Finalized.Epoch + safe := int64(res.Data.Finalized.Epoch) - 2 - var nextDataChan chan []DataEpochProcessed = make(chan []DataEpochProcessed, 1) - go func() { - d.epochDataFetcher(missingTails, epochFetchParallelism, nextDataChan) - }() + metrics.State.WithLabelValues("dashboard_data_exporter_latest_safe_epoch").Set(float64(safe)) + metrics.State.WithLabelValues("dashboard_data_exporter_latest_finalized_epoch").Set(float64(finalized)) - for { - datas := <-nextDataChan + d.latestSafeEpoch.Store(safe) + return nil +} - d.writeEpochDatas(datas) +func (d *dashboardData) GetName() string { + return "Dashboard-Data" +} - // has written last entry in gaps - if containsEpoch(datas, missingTails[len(missingTails)-1]) { - break - } - } +func (d *dashboardData) GetMonitoringEventId() constants.Event { + return constants.Event_ExporterModuleDashboardData +} - d.log.Infof("backfilling tail epochs for aggregation finished") +func (d *dashboardData) OnHead(event *constypes.StandardEventHeadResponse) error { + // you may ask, why here and not OnFinalizedCheckpoint? + // because due to our loadbalanced node architecture we sometimes receive the finalized checkpoint event + // before the node we hit has updated its own finalized checkpoint, causing us to be off by 1 epoch sometimes + // so we simply check more often. the request overhead is minimal anyways + err := updateSafeEpoch(d) + if err != nil { + return err + } return nil } -// fetches and processes epoch data and provides them via the nextDataChan -// expects ordered epochs in ascending order -func (d *dashboardData) epochDataFetcher(epochs []uint64, epochFetchParallelism int, nextDataChan chan []DataEpochProcessed) { - // group epochs into parallel worker groups - groups := getEpochParallelGroups(epochs, epochFetchParallelism) - numberOfEpochsToFetch := len(epochs) - epochsFetched := 0 +func (d *dashboardData) OnChainReorg(event *constypes.StandardEventChainReorg) error { + return nil +} - for _, gapGroup := range groups { - errGroup := &errgroup.Group{} +type MultiEpochData struct { + // needs sorting + epochBasedData struct { + epochs []uint64 + tarIndices []int + tarOffsets []int + validatorStates map[int64]constypes.LightStandardValidatorsResponse // epoch => state + rewards struct { + attestationRewards map[uint64][]constypes.AttestationReward // epoch => validator index => reward + attestationIdealRewards map[uint64]map[uint64]constypes.AttestationIdealReward // epoch => effective balance => reward + } + } + validatorBasedData struct { + // mapping pubkey => validator index + validatorIndices map[string]uint64 + } + syncPeriodBasedData struct { + // sync committee period => assignments + SyncAssignments map[uint64][]uint64 + // sync committee period => state + SyncStateEffectiveBalances map[uint64][]uint64 + } + slotBasedData struct { + blocks map[uint64]constypes.LightAnySignedBlock // slotOffset => block, if nil = missed. will include blocks for one more epoch than needed because attestations can be included an epoch later + assignments struct { + attestationAssignments map[uint64][][]uint64 // slotOffset => committee index => validator index + blockAssignments map[uint64]uint64 // slotOffset => validator index + } + rewards struct { + syncCommitteeRewards map[uint64]constypes.StandardSyncCommitteeRewardsResponse // slotOffset => sync committee rewards + blockRewards map[uint64]constypes.StandardBlockRewardsResponse // slotOffset => block reward data + } + } +} + +// factory +func NewMultiEpochData(epochCount int) MultiEpochData { + // allocate all maps + data := MultiEpochData{} + data.epochBasedData.validatorStates = make(map[int64]constypes.LightStandardValidatorsResponse, epochCount) + data.epochBasedData.tarIndices = make([]int, epochCount) + data.epochBasedData.tarOffsets = make([]int, epochCount) + data.epochBasedData.rewards.attestationRewards = make(map[uint64][]constypes.AttestationReward, epochCount) + data.epochBasedData.rewards.attestationIdealRewards = make(map[uint64]map[uint64]constypes.AttestationIdealReward, epochCount) + slotCount := epochCount * int(utils.Config.Chain.ClConfig.SlotsPerEpoch) + data.slotBasedData.blocks = make(map[uint64]constypes.LightAnySignedBlock, slotCount) + data.slotBasedData.assignments.attestationAssignments = make(map[uint64][][]uint64, slotCount) + data.slotBasedData.assignments.blockAssignments = make(map[uint64]uint64, slotCount) + data.slotBasedData.rewards.syncCommitteeRewards = make(map[uint64]constypes.StandardSyncCommitteeRewardsResponse, slotCount) + data.slotBasedData.rewards.blockRewards = make(map[uint64]constypes.StandardBlockRewardsResponse, slotCount) + data.validatorBasedData.validatorIndices = make(map[string]uint64) + return data +} + +func (d *dashboardData) getDataForEpochRange(epochStart, epochEnd uint64, tar *MultiEpochData) error { + g1 := &errgroup.Group{} + start := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_fetch_overall").Observe(time.Since(start).Seconds()) + }() + // prefill epochBasedData.epochs + for i := epochStart; i <= epochEnd; i++ { + tar.epochBasedData.epochs = append(tar.epochBasedData.epochs, i) + } + heavyRequestsSemMap := sync.Map{} + weights := make(map[string]int64) + weights["heavy"] = 8 + weights["medium"] = 18 + weights["light"] = 128 + orderedKeyList := []string{"heavy", "medium", "light"} + for k, v := range weights { + a := semaphore.NewWeighted(v) + heavyRequestsSemMap.Store(k, a) + } + // debug timer that prints the size of the queue for each node every 10 seconds + // should be stopped once function is done + timer := time.NewTicker(3 * time.Second) + defer timer.Stop() + go func() { + for { + _, ok := <-timer.C + if !ok { + return + } + for _, k := range orderedKeyList { + heavyRequestsSem, _ := heavyRequestsSemMap.Load(k) + // read cur, size, len(waiters) using reflection + v := reflect.ValueOf(heavyRequestsSem) + cur := v.Elem().FieldByName("cur").Int() + size := v.Elem().FieldByName("size").Int() + // waiters is a struct that has a len field + waiters := v.Elem().FieldByName("waiters").FieldByName("len").Int() + d.log.Debugf("%s: cur: %d, size: %d, waiters: %d", k, cur, size, waiters) + } + } + }() - datas := make([]*Data, 0, epochFetchParallelism) + // epoch based Data + g1.Go(func() error { start := time.Now() - - // identifies unique sync periods in this epoch group - var syncCommitteePeriods = make(map[uint64]bool) - - // Step 1: fetch epoch data raw - for _, gap := range gapGroup.Epochs { - gap := gap - syncCommitteePeriods[utils.SyncPeriodOfEpoch(gap)] = true - - errGroup.Go(func() error { - for { - // just in case we ask again before exporting since some time may have been passed - hasEpoch, err := edb.HasDashboardDataForEpoch(gap) - if err != nil { - d.log.Error(err, "failed to check if epoch has dashboard data", 0, map[string]interface{}{"epoch": gap}) - time.Sleep(time.Second * 10) - continue - } - if hasEpoch { - time.Sleep(time.Second * 1) - continue - } - - d.log.Infof("epoch data fetcher, retrieving data for epoch %d", gap) - - // for sequential improve performance by skipping some calls for all epochs that are not the start epoch - // and provide the startBalance and the missed slots for every following epoch in this sequence - // with the data from the previous epoch. Do not do this for non sequential working groups - data, err := d.GetEpochDataRaw(gap, gap != gapGroup.Epochs[0] && gapGroup.Sequential) - if err != nil { - d.log.Error(err, "failed to get epoch data", 0, map[string]interface{}{"epoch": gap}) - metrics.Errors.WithLabelValues("exporter_v2dash_node_fail").Inc() - time.Sleep(time.Second * 10) - continue + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_fetch_epoch_based_data_overall").Observe(time.Since(start).Seconds()) + }() + // get states + g2 := &errgroup.Group{} + slots := make([]uint64, 0) + // first slot of the first epoch + firstEpochToFetch := epochStart + for i := firstEpochToFetch; i <= epochEnd+1; i++ { + if i == 0 { + slots = append(slots, 0) + continue + } + slots = append(slots, uint64(i)*utils.Config.Chain.ClConfig.SlotsPerEpoch-1) + } + writeMutex := &sync.Mutex{} + d.log.Debugf("fetching states for epochs %d to %d using slots %v", epochStart, epochEnd, slots) + tar.epochBasedData.validatorStates = make(map[int64]constypes.LightStandardValidatorsResponse, len(slots)) + startEpoch := int64(epochStart) - 1 + for i, s := range slots { + slot := s + virtualEpoch := startEpoch + int64(i) + g2.Go(func() error { + // acquiring semaphore + heavyRequestsSem, _ := heavyRequestsSemMap.Load("heavy") + err := heavyRequestsSem.(*semaphore.Weighted).Acquire(context.Background(), 1) + if err != nil { + return err + } + defer heavyRequestsSem.(*semaphore.Weighted).Release(1) + start := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_fetch_epoch_based_data_single").Observe(time.Since(start).Seconds()) + }() + var valis *constypes.StandardValidatorsResponse + if slot == 0 { + valis, err = d.CL.GetValidators("genesis", nil, nil) + } else { + valis, err = d.CL.GetValidators(slot, nil, nil) + } + if err != nil { + d.log.Error(err, "can not get validators state", 0, map[string]interface{}{"slot": slot}) + return err + } + // convert to light validators + var lightValis constypes.LightStandardValidatorsResponse + lightValis.Data = make([]constypes.LightStandardValidator, len(valis.Data)) + for i, val := range valis.Data { + lightValis.Data[i] = constypes.LightStandardValidator{ + Index: val.Index, + Balance: val.Balance, + Status: val.Status, + Pubkey: val.Validator.Pubkey, + EffectiveBalance: val.Validator.EffectiveBalance, + Slashed: val.Validator.Slashed, } - - datas = append(datas, data) - - break } + writeMutex.Lock() + tar.epochBasedData.validatorStates[virtualEpoch] = lightValis + // quick update validatorBasedData.validatorIndices + for _, val := range lightValis.Data { + tar.validatorBasedData.validatorIndices[string(val.Pubkey)] = val.Index + } + writeMutex.Unlock() + // free up memory + valis = nil return nil }) } - - // Step 2: fetch sync committee data - d.getSyncCommitteesData(errGroup, syncCommitteePeriods) - - _ = errGroup.Wait() // no need to catch error since it will retry unless all clear without errors - - // Clear old sync committee cache entries that are not relevant for this group - d.clearOldCache(syncCommitteePeriods) - - // sort datas first, epoch asc - sort.Slice(datas, func(i, j int) bool { - return datas[i].epoch < datas[j].epoch - }) - - // Half Step: for sequential since we used skipSerialCalls we must provide startBalance and missedSlots for every epoch except FromEpoch - // with the data from the previous epoch. This is done to improve performance by skipping some calls - if gapGroup.Sequential { - // provide data from the previous epochs - for i := 1; i < len(datas); i++ { - datas[i].lastEpochStateEnd = datas[i-1].currentEpochStateEnd - - for slot := range datas[i-1].missedslots { - datas[i].missedslots[slot] = true - } + err := g2.Wait() + if err != nil { + return fmt.Errorf("error in epochBasedData: %w", err) + } + return nil + }) + // syncPeriodBasedData + g1.Go(func() error { + start := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_fetch_sync_period_based_data_overall").Observe(time.Since(start).Seconds()) + }() + // get sync committee assignments + g2 := &errgroup.Group{} + syncPeriodAssignmentsToFetch := make([]uint64, 0) + snycPeriodStatesToFetch := make([]uint64, 0) + for i := epochStart; i <= epochEnd; i++ { + if i < utils.Config.Chain.ClConfig.AltairForkEpoch { + d.log.Tracef("skipping sync committee assignments for epoch %d (before altair)", i) + continue } + syncPeriod := utils.SyncPeriodOfEpoch(i) + // if we dont have the assignment yet fetch it + if len(syncPeriodAssignmentsToFetch) == 0 || syncPeriodAssignmentsToFetch[len(syncPeriodAssignmentsToFetch)-1] != syncPeriod { + syncPeriodAssignmentsToFetch = append(syncPeriodAssignmentsToFetch, syncPeriod) + } + if utils.FirstEpochOfSyncPeriod(syncPeriod) == i { + snycPeriodStatesToFetch = append(snycPeriodStatesToFetch, syncPeriod) + } + } + d.log.Infof("fetching sync committee assignments and states for sync periods %v", syncPeriodAssignmentsToFetch) + writeMutex := &sync.Mutex{} + tar.syncPeriodBasedData.SyncAssignments = make(map[uint64][]uint64, len(syncPeriodAssignmentsToFetch)) + tar.syncPeriodBasedData.SyncStateEffectiveBalances = make(map[uint64][]uint64, len(snycPeriodStatesToFetch)) + // assignments + for _, s := range syncPeriodAssignmentsToFetch { + syncPeriod := s + g2.Go(func() error { + // acquiring semaphore + heavyRequestsSem, _ := heavyRequestsSemMap.Load("medium") + err := heavyRequestsSem.(*semaphore.Weighted).Acquire(context.Background(), 1) + if err != nil { + return err + } + defer heavyRequestsSem.(*semaphore.Weighted).Release(1) + start := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_fetch_sync_period_based_data_assignments_single").Observe(time.Since(start).Seconds()) + }() + relevantSlot := utils.FirstEpochOfSyncPeriod(syncPeriod) * utils.Config.Chain.ClConfig.SlotsPerEpoch + assignments, err := d.CL.GetSyncCommitteesAssignments(nil, relevantSlot) + if err != nil { + d.log.Error(err, "can not get sync committee assignments", 0, map[string]interface{}{"syncPeriod": syncPeriod}) + return err + } + writeMutex.Lock() + tar.syncPeriodBasedData.SyncAssignments[syncPeriod] = make([]uint64, len(assignments.Data.Validators)) + for i, a := range assignments.Data.Validators { + tar.syncPeriodBasedData.SyncAssignments[syncPeriod][i] = uint64(a) + } + writeMutex.Unlock() + return nil + }) } - - processed := make([]DataEpochProcessed, len(datas)) - - // Step 2: process data - errGroup = &errgroup.Group{} - errGroup.SetLimit(int(math.Max(epochWriteParallelism/2, 2.0))) // mitigate short ram spike - for i := 0; i < len(datas); i++ { - i := i - errGroup.Go(func() error { - for { - d.log.Infof("epoch data fetcher, processing data for epoch %d", datas[i].epoch) - start := time.Now() - - result, err := d.ProcessEpochData(datas[i]) - if err != nil { - d.log.Error(err, "failed to process epoch data", 0, map[string]interface{}{"epoch": datas[i].epoch}) - time.Sleep(time.Second * 10) - continue - } - - err = storeClBlockRewards(datas[i].beaconBlockRewardData) - if err != nil { - d.log.Error(err, "failed to store cl block rewards", 0, map[string]interface{}{"epoch": datas[i].epoch}) - time.Sleep(time.Second * 10) - continue - } - - processed[i] = DataEpochProcessed{ - Epoch: datas[i].epoch, - Data: result, + // states + for _, s := range snycPeriodStatesToFetch { + syncPeriod := s + g2.Go(func() error { + slot := utils.FirstEpochOfSyncPeriod(syncPeriod) * utils.Config.Chain.ClConfig.SlotsPerEpoch + // acquiring semaphore + heavyRequestsSem, _ := heavyRequestsSemMap.Load("heavy") + err := heavyRequestsSem.(*semaphore.Weighted).Acquire(context.Background(), 1) + if err != nil { + return err + } + defer heavyRequestsSem.(*semaphore.Weighted).Release(1) + start := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_fetch_sync_period_based_data_states_single").Observe(time.Since(start).Seconds()) + }() + valis, err := d.CL.GetValidators(slot, nil, nil) + if err != nil { + d.log.Error(err, "can not get sync committee state", 0, map[string]interface{}{"syncPeriod": syncPeriod}) + return err + } + // convert to light validators + dat := make([]uint64, len(valis.Data)) + for i, val := range valis.Data { + if val.Status.IsActive() { + dat[i] = val.Validator.EffectiveBalance } - - d.log.Infof("epoch data fetcher, processed data for epoch %d in %v", datas[i].epoch, time.Since(start)) - break } + writeMutex.Lock() + tar.syncPeriodBasedData.SyncStateEffectiveBalances[syncPeriod] = dat + writeMutex.Unlock() return nil }) } - - _ = errGroup.Wait() // no need to catch error since it will retry unless all clear without errors - datas = nil // clear raw data, not needed any more - - { - epochsFetched += len(processed) - remaining := numberOfEpochsToFetch - epochsFetched - remainingTimeEst := time.Duration(0) - if remaining > 0 { - remainingTimeEst = time.Duration(time.Since(start).Nanoseconds() / int64(len(processed)) * int64(remaining)) - } - - d.log.Infof("[time] epoch data fetcher, fetched %v epochs %v in %v. Remaining: %v (%v)", len(processed), gapGroup.Epochs, time.Since(start), remaining, remainingTimeEst) - metrics.TaskDuration.WithLabelValues("exporter_v2dash_fetch_epochs").Observe(time.Since(start).Seconds()) - metrics.TaskDuration.WithLabelValues("exporter_v2dash_fetch_epochs_per_epochs").Observe(time.Since(start).Seconds() / float64(len(processed))) + err := g2.Wait() + if err != nil { + return fmt.Errorf("error in syncPeriodBasedData: %w", err) } + return nil + }) - nextDataChan <- processed - } -} + // blocks + g1.Go(func() error { + start := time.Now() + defer func() { + //metrics.TaskDuration.With(prometheus.Labels{"pkg": "exporter", "module": "dashboard_data", "function": "getDataForEpochRange", "task": "slotBasedData", "duration_type": "total"}).Observe(time.Since(start).Seconds()) + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_fetch_slot_based_data_overall").Observe(time.Since(start).Seconds()) + }() + // get blocks + g2 := &errgroup.Group{} + slots := make([]uint64, 0) + // first slot of the previous epoch + firstSlotToFetch := (epochStart) * utils.Config.Chain.ClConfig.SlotsPerEpoch + if epochStart == 0 { + firstSlotToFetch = 0 + } + lastSlotToFetch := ((epochEnd + 2) * utils.Config.Chain.ClConfig.SlotsPerEpoch) - 1 + for i := firstSlotToFetch; i <= lastSlotToFetch; i++ { + slots = append(slots, i) + } + writeMutex := &sync.Mutex{} + tar.slotBasedData.blocks = make(map[uint64]constypes.LightAnySignedBlock, len(slots)) + for _, s := range slots { + slot := s + epoch := slot / utils.Config.Chain.ClConfig.SlotsPerEpoch + g2.Go(func() error { + d.log.Tracef("fetching block at slot %d", slot) + heavyRequestsSem, _ := heavyRequestsSemMap.Load("light") + err := heavyRequestsSem.(*semaphore.Weighted).Acquire(context.Background(), 1) + if err != nil { + return err + } + defer heavyRequestsSem.(*semaphore.Weighted).Release(1) + start := time.Now() + defer func() { + //metrics.TaskDuration.With(prometheus.Labels{"pkg": "exporter", "module": "dashboard_data", "function": "getDataForEpochRange", "task": "slotBasedData", "duration_type": "single"}).Observe(time.Since(start).Seconds()) + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_fetch_slot_based_data_single").Observe(time.Since(start).Seconds()) + }() -// Fetches sync committee assignments of provided periods -func (d *dashboardData) getSyncCommitteesData(errGroup *errgroup.Group, syncCommitteePeriods map[uint64]bool) { - for syncPeriod := range syncCommitteePeriods { - syncPeriod := syncPeriod - // -- Get current sync committee members and cache it - { - if found := d.responseCache.GetSyncCommittee(syncPeriod); found == nil { - errGroup.Go(func() error { - for { - start := time.Now() - data, err := d.CL.GetSyncCommitteesAssignments(nil, utils.FirstEpochOfSyncPeriod(syncPeriod)*utils.Config.Chain.ClConfig.SlotsPerEpoch) - if err != nil { - d.log.Error(err, "cannot get sync committee assignments", 0, map[string]interface{}{"syncPeriod": syncPeriod}) - metrics.Errors.WithLabelValues("exporter_v2dash_node_committee_fail").Inc() - time.Sleep(time.Second * 10) - continue - } - d.responseCache.SetSyncCommittee(syncPeriod, data) - d.log.Infof("retrieved sync committee members for sync period %d in %v", syncPeriod, time.Since(start)) - break + block, err := d.CL.GetSlot(slot) + if err != nil { + httpErr := network.SpecificError(err) + if httpErr != nil && httpErr.StatusCode == http.StatusNotFound { + d.log.Tracef("no block at slot %d", slot) + return nil } - return nil - }) - } - } - } -} - -func (d *dashboardData) clearOldCache(syncCommitteePeriods map[uint64]bool) { - // delete old sync committee election cache entries - for key := range d.responseCache.cache { - stillNeeded := false - - if strings.Contains(key, RawSyncCommitteeCacheKey) { - for syncPeriod := range syncCommitteePeriods { - syncCommitteeCacheKey := d.responseCache.GetSyncCommitteeCacheKey(syncPeriod) - if key == syncCommitteeCacheKey { - stillNeeded = true - break + d.log.Error(err, "can not get block", 0, map[string]interface{}{"slot": slot}) + return err } - } - } - - if !stillNeeded { - delete(d.responseCache.cache, key) - d.log.Infof("deleted stale sync committee cache entry %s", key) - } - } -} - -// breaks epoch down in groups of size parallelism -// for example epochs: 1,2,3,5,7,9,10,11,12,13,20,30 with parallelism = 4 breaks down to groups: -// 0 = 1,2,3 sequential true -// 1 = 5,7 sequential false -// 2 = 9,10,11,12 sequential true -// 3 = 13,20,30 sequential false -// expects ordered epochs in ascending order -func getEpochParallelGroups(epochs []uint64, parallelism int) []EpochParallelGroup { - parallelGroups := make([]EpochParallelGroup, 0, len(epochs)/4) + // header + header, err := d.CL.GetBlockHeader(slot) + if err != nil { + d.log.Error(err, "can not get block header", 0, map[string]interface{}{"slot": slot}) + return err + } + var lightBlock constypes.LightAnySignedBlock + lightBlock.Slot = block.Data.Message.Slot + lightBlock.BlockRoot = header.Data.Root + lightBlock.ParentRoot = header.Data.Header.Message.ParentRoot + lightBlock.ProposerIndex = block.Data.Message.ProposerIndex + lightBlock.Attestations = block.Data.Message.Body.Attestations + // deposits + lightBlock.Deposits = append(lightBlock.Deposits, block.Data.Message.Body.Deposits...) + // withdrawals + if epoch >= utils.Config.Chain.ClConfig.CapellaForkEpoch { + for _, w := range block.Data.Message.Body.ExecutionPayload.Withdrawals { + lightBlock.Withdrawals = append(lightBlock.Withdrawals, constypes.LightWithdrawal{ + Amount: w.Amount, + ValidatorIndex: w.ValidatorIndex, + }) + } + } + // AttesterSlashings + for _, s := range block.Data.Message.Body.AttesterSlashings { + lightBlock.SlashedIndices = append(lightBlock.SlashedIndices, s.GetSlashedIndices()...) + } + // ProposerSlashings + for _, s := range block.Data.Message.Body.ProposerSlashings { + lightBlock.SlashedIndices = append(lightBlock.SlashedIndices, s.SignedHeader1.Message.ProposerIndex) + } + if epoch >= utils.Config.Chain.ClConfig.AltairForkEpoch { + // sync + lightBlock.SyncAggregate = block.Data.Message.Body.SyncAggregate + } + // free up memory + block = nil - // 1. Group sequential epochs in parallel groups - for i := 0; i < len(epochs); i++ { - group := EpochParallelGroup{ - Epochs: []uint64{epochs[i]}, - Sequential: true, + writeMutex.Lock() + tar.slotBasedData.blocks[slot] = lightBlock + writeMutex.Unlock() + return nil + }) } - - var j int - for j = 1; j < parallelism && i+j < len(epochs); j++ { - if group.Epochs[len(group.Epochs)-1]+1 == epochs[i+j] { // only group sequential epochs - group.Epochs = append(group.Epochs, epochs[i+j]) - } else { - break - } + err := g2.Wait() + if err != nil { + return fmt.Errorf("error in slotBasedData: %w", err) } - i += j - 1 - - parallelGroups = append(parallelGroups, group) - } - - groups := make([]EpochParallelGroup, 0) - - // 2. Find non sequential (len(Epochs) == 1) epochs and group them as non sequential groups - for i := 0; i < len(parallelGroups); i++ { - group := parallelGroups[i] - - if len(group.Epochs) == 1 { - var j int - for j = 1; j < parallelism && i+j < len(parallelGroups); j++ { - nextGroup := parallelGroups[i+j] - if len(nextGroup.Epochs) == 1 { - group.Sequential = false - group.Epochs = append(group.Epochs, nextGroup.Epochs[0]) - } else { - break + return nil + }) + // block rewards + g1.Go(func() error { + start := time.Now() + defer func() { + // metrics.TaskDuration.With(prometheus.Labels{"pkg": "exporter", "module": "dashboard_data", "function": "getDataForEpochRange", "task": "blockRewards", "duration_type": "total"}).Observe(time.Since(start).Seconds()) + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_fetch_slot_based_data_block_rewards_overall").Observe(time.Since(start).Seconds()) + }() + // get block rewards + g2 := &errgroup.Group{} + writeMutex := &sync.Mutex{} + // we will fetch more than the requested epoch range because there is an "median cl reward" column for missed proposals + // slots: + buffer := utils.Config.Chain.ClConfig.SlotsPerEpoch / 2 + firstSlotToFetch := (epochStart) * utils.Config.Chain.ClConfig.SlotsPerEpoch + if firstSlotToFetch >= buffer { + firstSlotToFetch -= buffer + } + lastSlotToFetch := ((epochEnd + 1) * utils.Config.Chain.ClConfig.SlotsPerEpoch) + buffer - 1 + for i := firstSlotToFetch; i <= lastSlotToFetch; i++ { + slot := i + g2.Go(func() error { + // acquiring semaphore + heavyRequestsSem, _ := heavyRequestsSemMap.Load("light") + err := heavyRequestsSem.(*semaphore.Weighted).Acquire(context.Background(), 1) + if err != nil { + return err } - } - i += j - 1 + defer heavyRequestsSem.(*semaphore.Weighted).Release(1) + start := time.Now() + defer func() { + //metrics.TaskDuration.With(prometheus.Labels{"pkg": "exporter", "module": "dashboard_data", "function": "getDataForEpochRange", "task": "blockRewards", "duration_type": "single"}).Observe(time.Since(start).Seconds()) + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_fetch_slot_based_data_block_rewards_single").Observe(time.Since(start).Seconds()) + }() + data, err := d.CL.GetPropoalRewards(slot) + if err != nil { + httpErr := network.SpecificError(err) + if httpErr != nil && httpErr.StatusCode == http.StatusNotFound { + d.log.Infof("no block rewards for slot %d", slot) + return nil + } + d.log.Error(err, "can not get block rewards", 0, map[string]interface{}{"slot": slot}) + return err + } + writeMutex.Lock() + tar.slotBasedData.rewards.blockRewards[slot] = *data + writeMutex.Unlock() + return nil + }) } - - groups = append(groups, group) - } - - return groups -} - -var unaggregatedWrites = 0 - -type backfillResult struct { - BackfilledToHead bool // if backfill finished and current chain state is head (only set of backfill to head was requested, so only when upToEpoch = null) - DidPerformBackfill bool // whether a backfill was performed at all, false if backfill was not necessary -} - -// can be used to start a backfill up to epoch -// if upToEpoch is nil, it will backfill until the latest finalized epoch -func (d *dashboardData) backfillHeadEpochData(upToEpoch *uint64) (backfillResult, error) { - var result = backfillResult{} - backfillToChainFinalizedHead := upToEpoch == nil - if upToEpoch == nil { - res, err := d.CL.GetFinalityCheckpoints("head") + err := g2.Wait() if err != nil { - return result, errors.Wrap(err, "failed to get finalized checkpoint") + return fmt.Errorf("error in block rewards: %w", err) } - if utils.IsByteArrayAllZero(res.Data.Finalized.Root) { - return result, errors.New("network not finalized yet") + return nil + }) + // GetSyncRewards + g1.Go(func() error { + start := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_fetch_slot_based_data_sync_rewards_overall").Observe(time.Since(start).Seconds()) + }() + // get sync rewards + g2 := &errgroup.Group{} + writeMutex := &sync.Mutex{} + firstSlotToFetch := (epochStart) * utils.Config.Chain.ClConfig.SlotsPerEpoch + lastSlotToFetch := ((epochEnd + 1) * utils.Config.Chain.ClConfig.SlotsPerEpoch) - 1 + for i := firstSlotToFetch; i <= lastSlotToFetch; i++ { + slot := i + epoch := slot / utils.Config.Chain.ClConfig.SlotsPerEpoch + // check if slot is post hardfork + if epoch < utils.Config.Chain.ClConfig.AltairForkEpoch { + d.log.Tracef("skipping sync rewards for slot %d (before altair)", slot) + continue + } + g2.Go(func() error { + // acquiring semaphore + heavyRequestsSem, _ := heavyRequestsSemMap.Load("medium") + err := heavyRequestsSem.(*semaphore.Weighted).Acquire(context.Background(), 1) + if err != nil { + return err + } + defer heavyRequestsSem.(*semaphore.Weighted).Release(1) + start := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_fetch_slot_based_data_sync_rewards_single").Observe(time.Since(start).Seconds()) + }() + data, err := d.CL.GetSyncRewards(slot) + if err != nil { + httpErr := network.SpecificError(err) + if httpErr != nil && httpErr.StatusCode == http.StatusNotFound { + d.log.Tracef("no sync rewards for slot %d", slot) + return nil + } + d.log.Error(err, "can not get sync rewards", 0, map[string]interface{}{"slot": slot}) + return err + } + writeMutex.Lock() + tar.slotBasedData.rewards.syncCommitteeRewards[slot] = *data + writeMutex.Unlock() + return nil + }) } - upToEpoch = &res.Data.Finalized.Epoch - d.log.Infof("backfilling head epoch data up to epoch %d", *upToEpoch) - } - - latestExportedEpoch, err := edb.GetLatestDashboardEpoch() - if err != nil { - return result, errors.Wrap(err, "failed to get latest dashboard epoch") - } - - // An unclean shutdown can occur when exporter is shut down in between writing epoch data - // while each epoch is its own transaction and hence writes the complete epoch or nothing, - // it could happen that epochs have been written out of order due to the parallel nature of the exporter. - // Meaning that there is a gap in the last ~epochFetchParallelism epochs - { - uncleanShutdownGaps, err := edb.GetMissingEpochsBetween(int64(latestExportedEpoch-epochFetchParallelism), int64(latestExportedEpoch+1)) + err := g2.Wait() if err != nil { - return result, errors.Wrap(err, "failed to get epoch gaps") + return fmt.Errorf("error in sync rewards: %w", err) } - - if latestExportedEpoch > 0 && len(uncleanShutdownGaps) > 0 { - d.log.Infof("Unclean shutdown detected, backfilling missing epochs %d", uncleanShutdownGaps) - var nextDataChan chan []DataEpochProcessed = make(chan []DataEpochProcessed, 1) - go func() { - d.epochDataFetcher(uncleanShutdownGaps, epochFetchParallelism, nextDataChan) - }() - - for { - datas := <-nextDataChan - done := containsEpoch(datas, uncleanShutdownGaps[len(uncleanShutdownGaps)-1]) - d.writeEpochDatas(datas) - if done { - break - } - } - d.log.Infof("Fixed unclean shutdown gaps") - } - } - - // more epoch partitions than expected would indicate that a rolling aggregation backfill was interrupted or failed. - // We can use ancientEpochsPresent to keep the ancient epochs for a bit longer to prevent repeating fetching work. - var ancientEpochsPresent bool - { - partitions, err := edb.GetPartitionNamesOfTable(edb.EpochWriterTableName) - if err != nil { - return result, errors.Wrap(err, "failed to get partitions") - } - - epochsInDb := len(partitions) * PartitionEpochWidth - epochsExpectedInDb := int(float64(d.epochWriter.getRetentionEpochDuration()) * 1.2) // 20% buffer - maxAncientEpochs := int(float64(4*utils.EpochsPerDay()) * 0.75) // 18h (24h x 4 rolling tables, 75%). Makes sure we keep not too many which would degrade db performance - ancientEpochsPresent = epochsInDb > epochsExpectedInDb && epochsInDb < maxAncientEpochs - d.log.Infof("Checked for ancient epochs. Epochs in db: %d, expected: %d, maxAncientEpochs: %d, ancientEpochsPresent: %v", epochsInDb, epochsExpectedInDb, maxAncientEpochs, ancientEpochsPresent) - } - - gaps, err := edb.GetMissingEpochsBetween(int64(latestExportedEpoch), int64(*upToEpoch+1)) - if err != nil { - return result, errors.Wrap(err, "failed to get epoch gaps") - } - - if len(gaps) > 0 { - // Aggregate (non rolling) before exporting more epochs - // This is just a precaution so that the aggregated epochs tables are up to date - // before exporting new epochs - if latestExportedEpoch > 0 { - err = d.aggregatePerEpoch(false, true) - if err != nil { - return result, errors.Wrap(err, "failed to aggregate") - } - } - - d.log.Infof("Epoch dashboard data %d gaps found, backfilling gaps in the range fom epoch %d to %d", len(gaps), gaps[0], gaps[len(gaps)-1]) - - // get epochs data - var nextDataChan chan []DataEpochProcessed = make(chan []DataEpochProcessed, 1) - go func() { - d.epochDataFetcher(gaps, epochFetchParallelism, nextDataChan) + return nil + }) + // block assignments + g1.Go(func() error { + start := time.Now() + defer func() { + //metrics.TaskDuration.With(prometheus.Labels{"pkg": "exporter", "module": "dashboard_data", "function": "getDataForEpochRange", "task": "blockAssignments", "duration_type": "total"}).Observe(time.Since(start).Seconds()) + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_fetch_slot_based_data_block_assignments_overall").Observe(time.Since(start).Seconds()) }() - - // save epochs data - for { - d.log.Info("storage waiting for data from fetcher") - datas := <-nextDataChan - - done := containsEpoch(datas, gaps[len(gaps)-1]) // if the last epoch to fetch is in the result set, mark as job completed - lastEpoch := datas[len(datas)-1].Epoch - - // If one epoch containing an UTC boundary epoch we do a bootstrap aggregation for the rolling windows - // since this an exact utc bounds (and hence also an hour bounds) we do not need any tail epochs to calculate the rolling window - if debugAggregateRollingWindowsDuringBackfillUTCBoundEpoch { - for i, data := range datas { - if _, utcEndBound := getDayAggregateBounds(data.Epoch); utcEndBound-1 == data.Epoch { - d.log.Infof("BOOTSTRAPPING ROLLING WINDOWS! Epoch %d contains an UTC boundary epoch", data.Epoch) - // write subset - d.writeEpochDatas(datas[:i+1]) - for { - err = d.aggregatePerEpoch(true, false) // if we do a bootstrap rolling we don't have to prevent any epoch cleanup - if err != nil { - metrics.Errors.WithLabelValues("exporter_v2dash_agg_per_epoch_fail").Inc() - d.log.Error(err, "backfill, failed to aggregate", 0, map[string]interface{}{"epoch start": datas[0].Epoch, "epoch end": datas[len(datas)-1].Epoch}) - time.Sleep(time.Second * 10) - continue - } - break - } - - { - utcDayStart, _ := getDayAggregateBounds(data.Epoch - 1) - utcDay := utils.EpochToTime(utcDayStart).Format("2006-01-02") - utils.SendMessage(fmt.Sprintf("🗡🧙‍♂️ v2 Dashboard %s - Completed UTC day `%s` & Updated Rolling tables (24h, 7d, 30d, 90d) to epoch %v", utils.Config.Chain.Name, utcDay, data.Epoch), &utils.Config.InternalAlerts) - } - - if len(datas) > i+1 { - datas = datas[i+1:] - } else { - datas = nil - } - break - } - } - } - - if len(datas) > 0 { - d.log.Info("storage got data, writing epoch data") - d.writeEpochDatas(datas) - unaggregatedWrites += len(datas) - - if unaggregatedWrites > backfillMaxUnaggregatedIterations*epochFetchParallelism || lastEpoch%225 < epochFetchParallelism { - unaggregatedWrites = 0 - d.log.Info("storage writing done, aggregate") - for { - // This handles the case where we fetch a lot of ancient epochs, something goes wrong, export restarts, backfills (here) to head, - // but delete all the ancient epochs along the way. And then the head export must fetch them again to do the rollings. - // So we keep them in the backfill export and let head export clean them up after done work. - var preventClearOldEpochs bool - - // We target max 24h epochs to be in db to prevent performance degradation. - // ancientEpochsPresent targets max 18h + adding 6h (1/4 of 24h) = 24h - isSmallBackfill := len(gaps) < int(utils.EpochsPerDay()/4) - - // prevent cleaning old epochs in case head export got interrupted and we are already done in this backfill iteration - // or prevent if previous rolling aggregation has been interrupted. - preventClearOldEpochs = done || ancientEpochsPresent && isSmallBackfill - - err = d.aggregatePerEpoch(false, preventClearOldEpochs) - if err != nil { - d.log.Error(err, "backfill, failed to aggregate", 0, map[string]interface{}{"epoch start": datas[0].Epoch, "epoch end": lastEpoch}) - metrics.Errors.WithLabelValues("exporter_v2dash_agg_per_epoch_fail").Inc() - time.Sleep(time.Second * 10) - continue - } - break - } - d.log.InfoWithFields(map[string]interface{}{"epoch start": datas[0].Epoch, "epoch end": lastEpoch}, "backfill, aggregated epoch data") + // get block assignments + g2 := &errgroup.Group{} + writeMutex := &sync.Mutex{} + for e := epochStart; e <= epochEnd; e++ { + epoch := e + g2.Go(func() error { + heavyRequestsSem, _ := heavyRequestsSemMap.Load("medium") + err := heavyRequestsSem.(*semaphore.Weighted).Acquire(context.Background(), 1) + if err != nil { + return err } - - metrics.State.WithLabelValues("exporter_v2dash_last_exported_epoch").Set(float64(lastEpoch)) - } - - if lastEpoch%225 < epochFetchParallelism { - upToEpoch := *upToEpoch - utils.SendMessage(fmt.Sprintf("<:stonks:820252887094394901> v2 Dashboard %s - Epoch progress %d/%d [%.2f%%]", utils.Config.Chain.Name, lastEpoch, upToEpoch, float64(lastEpoch*100)/float64(upToEpoch)), &utils.Config.InternalAlerts) - } - - // has written last entry in gaps - if done { - break - } - } - } - - result.DidPerformBackfill = len(gaps) > 0 - - // Return with "complete" only if task was to sync to chain finalized head and we finished - if backfillToChainFinalizedHead { - res, err := d.CL.GetFinalityCheckpoints("head") - if err != nil { - return result, errors.Wrap(err, "failed to get finalized checkpoint") - } - if utils.IsByteArrayAllZero(res.Data.Finalized.Root) { - return result, errors.New("network not finalized yet") - } - - result.BackfilledToHead = res.Data.Finalized.Epoch-1 <= *upToEpoch - return result, nil - } - - return result, nil -} - -func containsEpoch(d []DataEpochProcessed, epoch uint64) bool { - for i := 0; i < len(d); i++ { - if d[i].Epoch == epoch { - return true - } - } - return false -} - -// stores all passed epoch data, blocks until all data is written without error -func (d *dashboardData) writeEpochDatas(datas []DataEpochProcessed) { - totalStart := time.Now() - defer func() { - d.log.Infof("[time] storage, wrote all epoch data in %v", time.Since(totalStart)) - metrics.TaskDuration.WithLabelValues("exporter_v2dash_write_epochs").Observe(time.Since(totalStart).Seconds()) - }() - - errGroup := &errgroup.Group{} - errGroup.SetLimit(epochWriteParallelism) - for i := 0; i < len(datas); i++ { - data := datas[i] - errGroup.Go(func() error { - for { - d.log.Infof("storage, writing epoch data for epoch %v", data.Epoch) + defer heavyRequestsSem.(*semaphore.Weighted).Release(1) start := time.Now() - - // retry this epoch until no errors occur - err := d.epochWriter.WriteEpochData(data.Epoch, data.Data) + defer func() { + //metrics.TaskDuration.With(prometheus.Labels{"pkg": "exporter", "module": "dashboard_data", "function": "getDataForEpochRange", "task": "blockAssignments", "duration_type": "single"}).Observe(time.Since(start).Seconds()) + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_fetch_slot_based_data_block_assignments_single").Observe(time.Since(start).Seconds()) + }() + data, err := d.CL.GetPropoalAssignments(epoch) if err != nil { - d.log.Error(err, "storage, failed to write epoch data", 0, map[string]interface{}{"epoch": data.Epoch}) - time.Sleep(time.Second * 10) - continue + d.log.Error(err, "can not get block assignments", 0, map[string]interface{}{"epoch": epoch}) + return err } - - d.log.Infof("storage, wrote epoch data %d in %v", data.Epoch, time.Since(start)) - - break - } - return nil - }) - } - - _ = errGroup.Wait() // no errors to handle since it will retry until it resolves without err -} - -// Contains all aggregation logic that should happen for every new exported epoch -// forceAggregate triggers an aggregation, use this when calling on head. -// updateRollingWindows specifies whether we should update rolling windows -func (d *dashboardData) aggregatePerEpoch(updateRollingWindows bool, preventClearOldEpochs bool) error { - currentExportedEpoch, err := edb.GetLatestDashboardEpoch() - if err != nil { - return errors.Wrap(err, "failed to get last exported epoch") - } - - // Performance improvement for backfilling, no need to aggregate day after each epoch, we can update once per hour - start := time.Now() - - // important to do this before hour aggregate as hour aggregate deletes old epochs - errGroup := &errgroup.Group{} - errGroup.SetLimit(nonRollingdatabaseAggregationParallelism) - errGroup.Go(func() error { - err := d.epochToTotal.aggregateTotal(currentExportedEpoch) - if err != nil { - return errors.Wrap(err, "failed to aggregate total") - } - return nil - }) - - // below: so this could be parallel IF we dont need to bootstrap. Room for improvement? - errGroup.Go(func() error { // can run in parallel with aggregateRollingWindows as long as no bootstrap is required, otherwise must be sequential - err := d.epochToHour.aggregate1h(currentExportedEpoch) // will aggregate last hour too if it hasn't completed yet - if err != nil { - return errors.Wrap(err, "failed to aggregate 1h") - } - return nil - }) - - errGroup.Go(func() error { // can run in parallel with aggregateRollingWindowsas long as no bootstrap is required, otherwise must be sequential - err := d.epochToDay.dayAggregate(currentExportedEpoch) - if err != nil { - return errors.Wrap(err, "failed to aggregate day") - } - return nil - }) - - err = errGroup.Wait() - if err != nil { - metrics.Errors.WithLabelValues("exporter_v2dash_agg_non_rolling_fail").Inc() - return errors.Wrap(err, "failed to aggregate") - } - d.log.Infof("[time] all of epoch based aggregation took %v", time.Since(start)) - metrics.TaskDuration.WithLabelValues("exporter_v2dash_agg_non_rolling").Observe(time.Since(start).Seconds()) - - if updateRollingWindows { - // todo you could add it to the err group above IF no bootstrap is needed. - - err = d.aggregateRollingWindows(currentExportedEpoch) - if err != nil { - metrics.Errors.WithLabelValues("exporter_v2dash_agg_non_fail").Inc() - return errors.Wrap(err, "failed to aggregate rolling windows") - } - - debugForceBootstrapRollingTables = false // reset flag after first run - - err = refreshMaterializedSlashedByCounts() - if err != nil { - return errors.Wrap(err, "failed to refresh slashed by counts") - } - } - - if !preventClearOldEpochs { - d.log.Infof("cleaning old epochs") - err = d.epochWriter.clearOldEpochs(int64(currentExportedEpoch - d.epochWriter.getRetentionEpochDuration())) - if err != nil { - return errors.Wrap(err, "failed to clear old epochs") - } - } - - metrics.State.WithLabelValues("exporter_v2dash_last_exported_epoch").Set(float64(currentExportedEpoch)) - - // clear old hourly aggregated epochs, do not remove epochs from epoch table here as these are needed for Mid aggregation - err = d.epochToHour.clearOldHourAggregations(int64(currentExportedEpoch - d.epochToHour.getHourRetentionDurationEpochs())) - if err != nil { - return errors.Wrap(err, "failed to clear old hours") - } - - err = d.epochToDay.clearOldDayAggregations(d.epochToDay.getDayRetentionDurationDays()) - if err != nil { - return errors.Wrap(err, "failed to clear old days") - } - - return nil -} - -// This function contains more heavy aggregation like rolling 7d, 30d, 90d -// This function assumes that epoch aggregation is finished before calling THOUGH it could run in parallel -// as long as the rolling tables do not require a bootstrap -func (d *dashboardData) aggregateRollingWindows(currentExportedEpoch uint64) error { - start := time.Now() - defer func() { - d.log.Infof("[time] all of mid aggregation took %v", time.Since(start)) - metrics.TaskDuration.WithLabelValues("exporter_v2dash_agg_roling").Observe(time.Since(start).Seconds()) - }() - - errGroup := &errgroup.Group{} - errGroup.SetLimit(databaseAggregationParallelism) - - errGroup.Go(func() error { - err := d.epochToDay.rolling24hAggregate(currentExportedEpoch) - if err != nil { - return errors.Wrap(err, "failed to rolling 24h aggregate") - } - d.log.Infof("finished dayAggregate rolling 24h") - return nil - }) - - errGroup.Go(func() error { - err := d.dayUp.rolling7dAggregate(currentExportedEpoch) - if err != nil { - return errors.Wrap(err, "failed to aggregate 7d") - } - return nil - }) - - errGroup.Go(func() error { - err := d.dayUp.rolling30dAggregate(currentExportedEpoch) - if err != nil { - return errors.Wrap(err, "failed to aggregate 30d") + writeMutex.Lock() + for _, p := range data.Data { + tar.slotBasedData.assignments.blockAssignments[uint64(p.Slot)] = p.ValidatorIndex + } + writeMutex.Unlock() + return nil + }) } - return nil - }) - - errGroup.Go(func() error { - err := d.dayUp.rolling90dAggregate(currentExportedEpoch) + err := g2.Wait() if err != nil { - return errors.Wrap(err, "failed to aggregate 90d") + return fmt.Errorf("error in block assignments: %w", err) } return nil }) - err := errGroup.Wait() - if err != nil { - return errors.Wrap(err, "failed to aggregate") - } - - return nil -} - -func (d *dashboardData) OnFinalizedCheckpoint(_ *constypes.StandardFinalizedCheckpointResponse) error { - if !d.backFillCompleted { - return nil // nothing to do, wait for backfill to finish - } - - // Note that "StandardFinalizedCheckpointResponse" event contains the current justified epoch, not the finalized one - // An epoch becomes finalized once the next epoch gets justified - // Hence we just listen for new justified epochs here and fetch the latest finalized one from the node - // Do not assume event.Epoch -1 is finalized by default as it could be that it is not justified - res, err := d.CL.GetFinalityCheckpoints("head") - if err != nil { - return err - } - - latestExported, err := edb.GetLatestDashboardEpoch() - if err != nil { - return err - } - - if latestExported != 0 { - if res.Data.Finalized.Epoch <= latestExported { - d.log.Infof("dashboard epoch data already exported for epoch %d", res.Data.Finalized.Epoch) - return nil - } - } - - metrics.State.WithLabelValues("exporter_v2dash_last_finalized_epoch").Set(float64(res.Data.Finalized.Epoch)) - - d.headEpochQueue <- res.Data.Finalized.Epoch - - return nil -} - -func (d *dashboardData) GetName() string { - return "Dashboard-Data" -} - -func (d *dashboardData) GetMonitoringEventId() constants.Event { - return constants.Event_ExporterModuleDashboardData -} - -func (d *dashboardData) OnHead(event *constypes.StandardEventHeadResponse) error { - return nil -} - -func (d *dashboardData) OnChainReorg(event *constypes.StandardEventChainReorg) error { - return nil -} - -func (d *dashboardData) GetEpochDataRaw(epoch uint64, skipSerialCalls bool) (*Data, error) { - data, err := d.getData(epoch, utils.Config.Chain.ClConfig.SlotsPerEpoch, skipSerialCalls) - if err != nil { - return nil, err - } - if data == nil { - return nil, errors.New("can not get data") - } - - return data, nil -} - -func (d *dashboardData) ProcessEpochData(data *Data) ([]*validatorDashboardDataRow, error) { - if d.signingDomain == nil { - domain, err := utils.GetSigningDomain() - if err != nil { - return nil, err - } - d.log.Infof("initialized signing domain to %x", domain) - d.signingDomain = domain - } - - return d.process(data, d.signingDomain) -} - -func isPartitionAttached(pTable string, partition string) (bool, error) { - var attached bool - err := db.AlloyWriter.QueryRow(fmt.Sprintf(` - SELECT EXISTS ( - SELECT 1 - FROM pg_partitioned_table pgt - JOIN pg_inherits pi ON pgt.partrelid = pi.inhparent - JOIN pg_class pc ON pc.oid = pi.inhrelid - WHERE pgt.partrelid = '%s'::regclass - AND pc.relname = '%s' - ) - `, - pTable, partition, - )).Scan(&attached) - - if err != nil { - return false, errors.Wrap(err, "failed to check if partition is already attached") - } - - return attached, nil -} - -// Returns the epoch where the sync committee election for the given epoch took place -func getSyncCommitteeElectionEpochOf(period uint64) uint64 { - syncElectionPeriod := int64(period) - 1 - firstEpoch := utils.FirstEpochOfSyncPeriod(uint64(syncElectionPeriod)) - if syncElectionPeriod < 0 || firstEpoch < utils.Config.Chain.ClConfig.AltairForkEpoch { - // first sync committee is identical to second sync committee - // See https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/fork.md#upgrading-the-state - return utils.Config.Chain.ClConfig.AltairForkEpoch - } - return firstEpoch -} - -type Data struct { - lastEpochStateEnd *constypes.StandardValidatorsResponse - currentEpochStateEnd *constypes.StandardValidatorsResponse - proposerAssignments *constypes.StandardProposerAssignmentsResponse - attestationRewards []constypes.AttestationReward - idealAttestationRewards map[int64]constypes.AttestationIdealReward // effective-balance -> ideal reward - beaconBlockData map[uint64]*constypes.StandardBeaconSlotResponse - beaconBlockRewardData map[uint64]*constypes.StandardBlockRewardsResponse - syncCommitteeRewardData map[uint64]*constypes.StandardSyncCommitteeRewardsResponse - attestationAssignments map[uint64]uint32 - missedslots map[uint64]bool - genesis bool - epoch uint64 - - // Contains the validator state of the epoch where the current sync committee election took place - syncCommitteeElectedState *constypes.StandardValidatorsResponse -} - -const MAX_EFFECTIVE_BALANCE = 32e9 - -// Data for a single validator -// use skipSerialCalls = false if you are not sure what you are doing. This flag is mainly -// to gain performance improvements when exporting a couple sequential epochs in a row -func (d *dashboardData) getData(epoch, slotsPerEpoch uint64, skipSerialCalls bool) (*Data, error) { - var result Data - var err error - - firstSlotOfEpoch := epoch * slotsPerEpoch - - lastSlotOfPreviousEpoch := int64(firstSlotOfEpoch) - 1 - lastSlotOfEpoch := firstSlotOfEpoch + slotsPerEpoch - 1 - - result.beaconBlockData = make(map[uint64]*constypes.StandardBeaconSlotResponse, slotsPerEpoch) - result.beaconBlockRewardData = make(map[uint64]*constypes.StandardBlockRewardsResponse, slotsPerEpoch) - result.syncCommitteeRewardData = make(map[uint64]*constypes.StandardSyncCommitteeRewardsResponse, slotsPerEpoch) - result.attestationAssignments = make(map[uint64]uint32) - result.idealAttestationRewards = make(map[int64]constypes.AttestationIdealReward) - result.missedslots = make(map[uint64]bool, slotsPerEpoch*2) - result.epoch = epoch - - cl := d.CL - - errGroup := &errgroup.Group{} - errGroup.SetLimit(epochFetchParallelismWithinEpoch) - - totalStart := time.Now() - - errGroup.Go(func() error { - // retrieve proposer assignments for the epoch in order to attribute missed slots + // attestation rewards + g1.Go(func() error { start := time.Now() - var err error - result.proposerAssignments, err = cl.GetPropoalAssignments(epoch) - if err != nil { - d.log.Error(err, "can not get proposer assignments", 0, map[string]interface{}{"epoch": epoch}) - return err - } - d.log.Debugf("retrieved proposer assignments in %v", time.Since(start)) - return nil - }) - - // As of dencun you can attest up until the end of the following epoch - min := lastSlotOfEpoch - utils.Config.Chain.ClConfig.SlotsPerEpoch - if lastSlotOfEpoch < utils.Config.Chain.ClConfig.SlotsPerEpoch { - min = 0 - } - - aaMutex := &sync.Mutex{} - // executes twice, one with lastSlotOf this epoch and then lastSlotOf last epoch - for slot := lastSlotOfEpoch; slot >= min; slot -= utils.Config.Chain.ClConfig.SlotsPerEpoch { - slot := slot - errGroup.Go(func() error { - data, err := cl.GetCommittees(slot, nil, nil, nil) - if err != nil { - d.log.Error(err, "can not get attestation assignments", 0, map[string]interface{}{"slot": slot}) - return err - } - - for _, committee := range data.Data { - for i, valIndex := range committee.Validators { - k := utils.FormatAttestorAssignmentKeyLowMem(committee.Slot, uint16(committee.Index), uint32(i)) - aaMutex.Lock() - result.attestationAssignments[k] = uint32(valIndex) - aaMutex.Unlock() + defer func() { + //metrics.TaskDuration.With(prometheus.Labels{"pkg": "exporter", "module": "dashboard_data", "function": "getDataForEpochRange", "task": "attestationRewards", "duration_type": "total"}).Observe(time.Since(start).Seconds()) + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_fetch_epoch_based_data_attestation_rewards_overall").Observe(time.Since(start).Seconds()) + }() + // get attestation rewards + g2 := &errgroup.Group{} + writeMutex := &sync.Mutex{} + // once per epoch, no extra epochs needed + for e := epochStart; e <= epochEnd; e++ { + epoch := e + g2.Go(func() error { + // acquiring semaphore + heavyRequestsSem, _ := heavyRequestsSemMap.Load("heavy") + err := heavyRequestsSem.(*semaphore.Weighted).Acquire(context.Background(), 1) + if err != nil { + return err } - } - return nil - }) - if slot < utils.Config.Chain.ClConfig.SlotsPerEpoch { - break // special case for epoch 0 + defer heavyRequestsSem.(*semaphore.Weighted).Release(1) + start := time.Now() + defer func() { + //metrics.TaskDuration.With(prometheus.Labels{"pkg": "exporter", "module": "dashboard_data", "function": "getDataForEpochRange", "task": "attestationRewards", "duration_type": "single"}).Observe(time.Since(start).Seconds()) + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_fetch_epoch_based_data_attestation_rewards_single").Observe(time.Since(start).Seconds()) + }() + data, err := d.CL.GetAttestationRewards(epoch) + if err != nil { + d.log.Error(err, "can not get attestation rewards", 0, map[string]interface{}{"epoch": epoch}) + return err + } + // ideal + ideal := make(map[uint64]constypes.AttestationIdealReward) + for _, idealReward := range data.Data.IdealRewards { + ideal[uint64(idealReward.EffectiveBalance)] = idealReward + } + writeMutex.Lock() + tar.epochBasedData.rewards.attestationRewards[epoch] = data.Data.TotalRewards + tar.epochBasedData.rewards.attestationIdealRewards[epoch] = ideal + writeMutex.Unlock() + return nil + }) } - } - - errGroup.Go(func() error { - // attestation rewards - start := time.Now() - data, err := cl.GetAttestationRewards(epoch) + err := g2.Wait() if err != nil { - d.log.Error(err, "can not get attestation rewards", 0, map[string]interface{}{"epoch": epoch}) - return err - } - - for _, idealReward := range data.Data.IdealRewards { - if _, ok := result.idealAttestationRewards[idealReward.EffectiveBalance]; !ok { - result.idealAttestationRewards[idealReward.EffectiveBalance] = idealReward - } + return fmt.Errorf("error in attestation rewards: %w", err) } - - result.attestationRewards = data.Data.TotalRewards - - d.log.Debugf("retrieved attestation rewards data in %v", time.Since(start)) return nil }) - - errGroup.Go(func() error { - // retrieve the validator balances at the end of the epoch + // attestation assignments + g1.Go(func() error { start := time.Now() - var err error - result.currentEpochStateEnd, err = cl.GetValidators(lastSlotOfEpoch, nil, nil) - if err != nil { - d.log.Error(err, "can not get validators balances", 0, map[string]interface{}{"lastSlotOfEpoch": lastSlotOfEpoch}) - return err - } - d.log.Debugf("retrieved end balances using state at slot %d in %v", lastSlotOfEpoch, time.Since(start)) - return nil - }) - - // if this flag is used the caller must provide startBalance from the previous epoch themselves - // as well as providing the missedslots data from the previous epoch - if !skipSerialCalls { - errGroup.Go(func() error { - // retrieve the validator balances at the start of the epoch - start := time.Now() - var err error - if lastSlotOfPreviousEpoch < 0 { - result.lastEpochStateEnd, err = d.CL.GetValidators("genesis", nil, nil) - result.genesis = true - } else { - result.lastEpochStateEnd, err = d.CL.GetValidators(lastSlotOfPreviousEpoch, nil, nil) - } - if err != nil { - d.log.Error(err, "can not get validators balances", 0, map[string]interface{}{"lastSlotOfPreviousEpoch": lastSlotOfPreviousEpoch}) - return err - } - d.log.Debugf("retrieved start balances using state at slot %d in %v", lastSlotOfPreviousEpoch, time.Since(start)) - return nil - }) - - errGroup.Go(func() error { - start := time.Now() - - if firstSlotOfEpoch > slotsPerEpoch { // handle case for first epoch - // get missed slots of last epoch for optimal inclusion distance - for slot := firstSlotOfEpoch - slotsPerEpoch; slot <= lastSlotOfEpoch-slotsPerEpoch; slot++ { - _, err := cl.GetBlockHeader(slot) - if err != nil { - httpErr := network.SpecificError(err) - if httpErr != nil && httpErr.StatusCode == http.StatusNotFound { - result.missedslots[slot] = true - continue // missed - } - d.log.Fatal(err, "can not get block data", 0, map[string]interface{}{"slot": slot}) - continue - } - } - } - d.log.Debugf("retrieved missed slots in %v", time.Since(start)) - return nil - }) - } - - mutex := &sync.Mutex{} - for slot := firstSlotOfEpoch; slot <= lastSlotOfEpoch; slot++ { - slot := slot - errGroup.Go(func() error { - // retrieve the data for all blocks that were proposed in this epoch - block, err := cl.GetSlot(slot) - if err != nil { - httpErr := network.SpecificError(err) - if httpErr != nil && httpErr.StatusCode == http.StatusNotFound { - mutex.Lock() - result.missedslots[slot] = true - mutex.Unlock() - return nil // missed - } - - return err - } - - mutex.Lock() - result.beaconBlockData[slot] = block - mutex.Unlock() - - blockReward, err := cl.GetPropoalRewards(slot) - if err != nil { - d.log.Error(err, "can not get block reward data", 0, map[string]interface{}{"slot": slot}) - return err - } - mutex.Lock() - result.beaconBlockRewardData[slot] = blockReward - mutex.Unlock() - - syncRewards, err := cl.GetSyncRewards(slot) - if err != nil { - d.log.Error(err, "can not get sync committee reward data", 0, map[string]interface{}{"slot": slot}) - return err - } - mutex.Lock() - result.syncCommitteeRewardData[slot] = syncRewards - mutex.Unlock() - - return nil - }) - } - - currentSyncPeriod := utils.SyncPeriodOfEpoch(epoch) - if epoch == utils.FirstEpochOfSyncPeriod(currentSyncPeriod) { - syncCommitteElectionEpoch := getSyncCommitteeElectionEpochOf(currentSyncPeriod) - syncCommitteeElectedInSlot := syncCommitteElectionEpoch * utils.Config.Chain.ClConfig.SlotsPerEpoch - errGroup.Go(func() error { - start := time.Now() - var err error - result.syncCommitteeElectedState, err = d.CL.GetValidators(syncCommitteeElectedInSlot, nil, []constypes.ValidatorStatus{constypes.Active}) - if err != nil { - d.log.Error(err, "can not get sync committee election state", 0, map[string]interface{}{"slot": syncCommitteeElectedInSlot}) - return err - } - d.log.Infof("retrieved validator state for sync committee period %d (election state is epoch %d) in %v", currentSyncPeriod, syncCommitteElectionEpoch, time.Since(start)) - - return nil - }) - } - - err = errGroup.Wait() - if err != nil { - return nil, err - } - - d.log.Infof("[time] retrieved all data for epoch %d in %v", epoch, time.Since(totalStart)) - - return &result, nil -} - -func (d *dashboardData) process(data *Data, domain []byte) ([]*validatorDashboardDataRow, error) { - validatorsData := make([]*validatorDashboardDataRow, len(data.currentEpochStateEnd.Data)) - - pubkeyToIndexMapNewlyActivatedValidators := make(map[string]int64, 8) - pubkeyToIndexMapOldValidators := make(map[string]int64, len(validatorsData)) - activeTotalEffectiveBalanceETH := int64(0) - - currentSyncPeriod := utils.SyncPeriodOfEpoch(data.epoch) - - postAltair := data.epoch >= utils.Config.Chain.ClConfig.AltairForkEpoch - - // write start & end balances and slashed status - for i := 0; i < len(validatorsData); i++ { - if uint64(i) != data.currentEpochStateEnd.Data[i].Index { // sanity assumption that i = validator index - return nil, errors.New("validator index mismatch") - } - - validatorsData[i] = &validatorDashboardDataRow{} - if i < len(data.lastEpochStateEnd.Data) { - validatorsData[i].BalanceStart = data.lastEpochStateEnd.Data[i].Balance - pubkeyToIndexMapOldValidators[string(data.lastEpochStateEnd.Data[i].Validator.Pubkey)] = int64(i) - - if data.genesis { // Add genesis deposits - validatorsData[i].DepositsCount = utils.NullInt16(1) - validatorsData[i].DepositsAmount = utils.NullInt64(int64(data.lastEpochStateEnd.Data[i].Validator.EffectiveBalance)) - } - } else { - pubkeyToIndexMapNewlyActivatedValidators[string(data.currentEpochStateEnd.Data[i].Validator.Pubkey)] = int64(i) // validators that become active in the epoch - } - - if data.currentEpochStateEnd.Data[i].Status.IsActive() { - activeTotalEffectiveBalanceETH += int64(data.currentEpochStateEnd.Data[i].Validator.EffectiveBalance / 1e9) - validatorsData[i].AttestationsScheduled = utils.NullInt16(1) - } - - validatorsData[i].BalanceEnd = data.currentEpochStateEnd.Data[i].Balance - validatorsData[i].Slashed = data.currentEpochStateEnd.Data[i].Validator.Slashed - } - - // Expected Block Proposal - for _, valData := range data.currentEpochStateEnd.Data { - if valData.Status.IsActive() { - // See https://github.com/ethereum/annotated-spec/blob/master/phase0/beacon-chain.md#compute_proposer_index - proposalChance := float64(valData.Validator.EffectiveBalance/1e9) / float64(activeTotalEffectiveBalanceETH) - validatorsData[valData.Index].BlocksExpectedThisEpoch = proposalChance * float64(utils.Config.Chain.ClConfig.SlotsPerEpoch) - } - } - - // Expected Sync Committees - // Get the total effective balance from the state where the current sync committee was elected - // And then calculate the chance of being in the sync committee from that state - if data.epoch == utils.FirstEpochOfSyncPeriod(currentSyncPeriod) { - if data.syncCommitteeElectedState == nil && postAltair { - return nil, errors.New("sync committee election state not found") - } - - syncCommitteeElectionStateTotalEffectiveBalanceETH := int64(0) - for _, valData := range data.syncCommitteeElectedState.Data { - if valData.Status.IsActive() { - syncCommitteeElectionStateTotalEffectiveBalanceETH += int64(valData.Validator.EffectiveBalance / 1e9) - } - } - - for _, valData := range data.syncCommitteeElectedState.Data { - if valData.Status.IsActive() { - // See https://github.com/ethereum/annotated-spec/blob/master/altair/beacon-chain.md#get_sync_committee_indices - // Note that this formula is not 100% the chance as defined in the spec, but after running simulations we found - // it being precise enough for our purposes with an error margin of less than 0.003% - syncChance := float64(valData.Validator.EffectiveBalance/1e9) / float64(syncCommitteeElectionStateTotalEffectiveBalanceETH) - validatorsData[valData.Index].SyncCommitteesExpectedThisPeriod = syncChance * float64(utils.Config.Chain.ClConfig.SyncCommitteeSize) - } - } - } - - size := uint64(len(validatorsData)) - sizeInt := int64(len(validatorsData)) - size32 := uint32(len(validatorsData)) - - // write scheduled block data - for _, proposerAssignment := range data.proposerAssignments.Data { - proposerIndex := proposerAssignment.ValidatorIndex - if proposerIndex >= size { - return nil, errors.New("proposer index out of range") - } - validatorsData[proposerIndex].BlockScheduled.Int16++ - validatorsData[proposerIndex].BlockScheduled.Valid = true - } - - syncCommitteeAssignments := d.responseCache.GetSyncCommittee(currentSyncPeriod) - if syncCommitteeAssignments == nil && postAltair { - return nil, errors.New("sync committee assignments not found") - } - - // write scheduled sync committee data - for _, validator := range syncCommitteeAssignments.Data.Validators { - validatorIndex := int64(validator) - if validatorIndex >= sizeInt { - return nil, errors.New("proposer index out of range") - } - validatorsData[validatorIndex].SyncScheduled.Int16 = int16(len(data.beaconBlockData)) // take into account missed slots - validatorsData[validatorIndex].SyncScheduled.Valid = true - } - - // write proposer rewards data - for _, reward := range data.beaconBlockRewardData { - if reward.Data.ProposerIndex >= size { - return nil, errors.New("proposer index out of range") - } - - validatorsData[reward.Data.ProposerIndex].BlocksClSyncAggregateReward.Int64 += reward.Data.SyncAggregate - validatorsData[reward.Data.ProposerIndex].BlocksClSyncAggregateReward.Valid = true - - validatorsData[reward.Data.ProposerIndex].BlocksClAttestestationsReward.Int64 += reward.Data.Attestations - validatorsData[reward.Data.ProposerIndex].BlocksClAttestestationsReward.Valid = true - - validatorsData[reward.Data.ProposerIndex].BlocksClReward.Int64 += reward.Data.Attestations + reward.Data.AttesterSlashings + reward.Data.ProposerSlashings + reward.Data.SyncAggregate - validatorsData[reward.Data.ProposerIndex].BlocksClReward.Valid = true - - validatorsData[reward.Data.ProposerIndex].SlasherRewards.Int64 += reward.Data.AttesterSlashings + reward.Data.ProposerSlashings - if reward.Data.AttesterSlashings+reward.Data.ProposerSlashings > 0 { - validatorsData[reward.Data.ProposerIndex].SlasherRewards.Valid = true - } - } - - // write sync committee reward data & sync committee execution stats - for _, rewards := range data.syncCommitteeRewardData { - for _, reward := range rewards.Data { - validator_index := reward.ValidatorIndex - if validator_index >= size { - return nil, errors.New("proposer index out of range") - } - syncReward := reward.Reward - validatorsData[validator_index].SyncReward.Int64 += syncReward - validatorsData[validator_index].SyncReward.Valid = true - - if syncReward > 0 { - validatorsData[validator_index].SyncExecuted.Int16++ - validatorsData[validator_index].SyncExecuted.Valid = true - } - } - } - - // write block specific data - for _, block := range data.beaconBlockData { - if block.Data.Message.Slot != 0 { // Special case to exclude genesis block as Validator 0 did not propose it - validatorsData[block.Data.Message.ProposerIndex].BlocksProposed.Int16++ - validatorsData[block.Data.Message.ProposerIndex].BlocksProposed.Valid = true - validatorsData[block.Data.Message.ProposerIndex].LastSubmittedDutyEpoch = utils.NullInt32(int32(block.Data.Message.Slot / utils.Config.Chain.ClConfig.SlotsPerEpoch)) - } - - for depositIndex, depositData := range block.Data.Message.Body.Deposits { - // Properly verify that deposit is valid: - // if signature is valid I count the deposit towards the balance - // if signature is invalid and the validator was in the state at the beginning of the epoch I count the deposit towards the balance - // if signature is invalid and the validator was NOT in the state at the beginning of the epoch and there were no valid deposits in the block prior I DO NOT count the deposit towards the balance - // if signature is invalid and the validator was NOT in the state at the beginning of the epoch and there was a VALID deposit in the blocks prior I DO COUNT the deposit towards the balance - - err := utils.VerifyDepositSignature(&phase0.DepositData{ - PublicKey: phase0.BLSPubKey(depositData.Data.Pubkey), - WithdrawalCredentials: depositData.Data.WithdrawalCredentials, - Amount: phase0.Gwei(depositData.Data.Amount), - Signature: phase0.BLSSignature(depositData.Data.Signature), - }, domain) - - if err != nil { - d.log.Error(fmt.Errorf("deposit at index %d in slot %v is invalid: %v (domain: %x, PublicKey: %s, WithdrawalCredentials: %s, Amount: %d, Signature: %s)", - depositIndex, block.Data.Message.Slot, err, domain, depositData.Data.Pubkey, depositData.Data.WithdrawalCredentials, depositData.Data.Amount, depositData.Data.Signature), "", 0) - - // if the validator hat a valid deposit prior to the current one, count the invalid towards the balance - if validatorsData[pubkeyToIndexMapNewlyActivatedValidators[string(depositData.Data.Pubkey)]].DepositsCount.Int16 > 0 { - d.log.Infof("validator had a valid deposit in some earlier block of the epoch, count the invalid towards the balance") - } else if _, ok := pubkeyToIndexMapOldValidators[string(depositData.Data.Pubkey)]; ok { - d.log.Infof("validator had a valid deposit in some block prior to the current epoch, count the invalid towards the balance") - } else { - d.log.Infof("validator did not have a prior valid deposit, do not count the invalid towards the balance") - continue + defer func() { + //metrics.TaskDuration.With(prometheus.Labels{"pkg": "exporter", "module": "dashboard_data", "function": "getDataForEpochRange", "task": "attestationAssignments", "duration_type": "total"}).Observe(time.Since(start).Seconds()) + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_fetch_slot_based_data_attestation_assignments_overall").Observe(time.Since(start).Seconds()) + }() + // get attestation assignments + g2 := &errgroup.Group{} + writeMutex := &sync.Mutex{} + for e := epochStart; e <= epochEnd; e++ { + epoch := e + g2.Go(func() error { + // fetch assignment using last fetchSlot in epoch. somehow thats faster than using the first fetchSlot. dont ask why + fetchSlot := (epoch+1)*utils.Config.Chain.ClConfig.SlotsPerEpoch - 1 + heavyRequestsSem, _ := heavyRequestsSemMap.Load("heavy") + err := heavyRequestsSem.(*semaphore.Weighted).Acquire(context.Background(), 1) + if err != nil { + return err } - } - - validator_index, ok := pubkeyToIndexMapNewlyActivatedValidators[string(depositData.Data.Pubkey)] - if !ok { - validator_index, ok = pubkeyToIndexMapOldValidators[string(depositData.Data.Pubkey)] - if !ok { - return nil, errors.New("proposer index out of range") + defer heavyRequestsSem.(*semaphore.Weighted).Release(1) + start := time.Now() + defer func() { + //metrics.TaskDuration.With(prometheus.Labels{"pkg": "exporter", "module": "dashboard_data", "function": "getDataForEpochRange", "task": "attestationAssignments", "duration_type": "single"}).Observe(time.Since(start).Seconds()) + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_fetch_slot_based_data_attestation_assignments_single").Observe(time.Since(start).Seconds()) + }() + data, err := d.CL.GetCommittees(fetchSlot, nil, nil, nil) + if err != nil { + d.log.Error(err, "can not get attestation assignments", 0, map[string]interface{}{"slot": fetchSlot}) + return err } - } - if validator_index >= sizeInt { - return nil, errors.New("proposer index out of range") - } - - validatorsData[validator_index].DepositsAmount.Int64 += int64(depositData.Data.Amount) - validatorsData[validator_index].DepositsAmount.Valid = true - - validatorsData[validator_index].DepositsCount.Int16++ - validatorsData[validator_index].DepositsCount.Valid = true - } - - for _, withdrawal := range block.Data.Message.Body.ExecutionPayload.Withdrawals { - validator_index := withdrawal.ValidatorIndex - - if validator_index >= size { - return nil, errors.New("proposer index out of range") - } - validatorsData[validator_index].WithdrawalsAmount.Int64 += int64(withdrawal.Amount) - validatorsData[validator_index].WithdrawalsAmount.Valid = true - - validatorsData[validator_index].WithdrawalsCount.Int16++ - validatorsData[validator_index].WithdrawalsCount.Valid = true - } - - for _, attestation := range block.Data.Message.Body.Attestations { - aggregationBits := bitfield.Bitlist(attestation.AggregationBits) - - for i := uint64(0); i < aggregationBits.Len(); i++ { - if aggregationBits.BitAt(i) { - validator_index, found := data.attestationAssignments[utils.FormatAttestorAssignmentKeyLowMem(attestation.Data.Slot, attestation.Data.Index, uint32(i))] - if !found { // This should never happen! - d.log.Error(fmt.Errorf("validator not found in attestation assignments"), "validator not found in attestation assignments", 0, map[string]interface{}{"slot": attestation.Data.Slot, "index": attestation.Data.Index, "i": i}) - return nil, fmt.Errorf("validator not found in attestation assignments") + writeMutex.Lock() + for _, committee := range data.Data { + // todo replace with single alloc variant that uses config values (config has 0 when the code hits here) + if _, ok := tar.slotBasedData.assignments.attestationAssignments[committee.Slot]; !ok { + tar.slotBasedData.assignments.attestationAssignments[committee.Slot] = make([][]uint64, committee.Index+1) } - if validator_index >= size32 { - return nil, errors.New("proposer index out of range") + // if not long enough, extend + if l := len(tar.slotBasedData.assignments.attestationAssignments[committee.Slot]); l < int(committee.Index)+1 { + tar.slotBasedData.assignments.attestationAssignments[committee.Slot] = append( + tar.slotBasedData.assignments.attestationAssignments[committee.Slot], + make([][]uint64, int(committee.Index)+1-l)..., + ) } - - validatorsData[validator_index].InclusionDelaySum = utils.NullInt16(int16(block.Data.Message.Slot - attestation.Data.Slot - 1)) - - validatorsData[validator_index].LastSubmittedDutyEpoch = utils.NullInt32(int32(attestation.Data.Slot / utils.Config.Chain.ClConfig.SlotsPerEpoch)) - - optimalInclusionDistance := 0 - for i := attestation.Data.Slot + 1; i < block.Data.Message.Slot; i++ { - if _, ok := data.missedslots[i]; ok { - optimalInclusionDistance++ - } else { - break - } + // if not enough space for validators, allocate + if l := len(tar.slotBasedData.assignments.attestationAssignments[committee.Slot][committee.Index]); l < len(committee.Validators) { + tar.slotBasedData.assignments.attestationAssignments[committee.Slot][committee.Index] = make([]uint64, len(committee.Validators)) + } + // ass + for i, valIndex := range committee.Validators { + tar.slotBasedData.assignments.attestationAssignments[committee.Slot][committee.Index][i] = uint64(valIndex) } - - validatorsData[validator_index].OptimalInclusionDelay = utils.NullInt16(int16(optimalInclusionDistance)) } - } - } - - // attester slashings "done" by proposer (slashed by) - for _, data := range block.Data.Message.Body.AttesterSlashings { - slashedIndices := data.GetSlashedIndices() - for _, index := range slashedIndices { - validatorsData[index].SlashedBy = utils.NullInt32(int32(block.Data.Message.ProposerIndex)) - validatorsData[index].Slashed = true - validatorsData[index].SlashedViolation = utils.NullInt16(SLASHED_VIOLATION_ATTESTATION) - } - } - - // proposer slashings "done" by proposer (slashed by) - for _, data := range block.Data.Message.Body.ProposerSlashings { - validatorsData[data.SignedHeader1.Message.ProposerIndex].SlashedBy = utils.NullInt32(int32(block.Data.Message.ProposerIndex)) - validatorsData[data.SignedHeader1.Message.ProposerIndex].Slashed = true - validatorsData[data.SignedHeader1.Message.ProposerIndex].SlashedViolation = utils.NullInt16(SLASHED_VIOLATION_PROPOSER) - } - } - - // write attestation rewards data - for _, attestationReward := range data.attestationRewards { - validator_index := attestationReward.ValidatorIndex - if validator_index >= size { - return nil, errors.New("proposer index out of range") - } - - validatorsData[validator_index].AttestationsHeadReward = utils.NullInt32(attestationReward.Head) - validatorsData[validator_index].AttestationsSourceReward = utils.NullInt32(attestationReward.Source) - validatorsData[validator_index].AttestationsTargetReward = utils.NullInt32(attestationReward.Target) - validatorsData[validator_index].AttestationsInactivityPenalty = utils.NullInt32(attestationReward.Inactivity) - validatorsData[validator_index].AttestationsInclusionsReward = utils.NullInt32(attestationReward.InclusionDelay) - validatorsData[validator_index].AttestationReward = utils.NullInt64(int64(attestationReward.Head + attestationReward.Source + attestationReward.Target + attestationReward.Inactivity + attestationReward.InclusionDelay)) - - idealRewardsOfValidator, ok := data.idealAttestationRewards[int64(data.currentEpochStateEnd.Data[validator_index].Validator.EffectiveBalance)] - if !ok { - return nil, errors.New("ideal reward not found") - } - - validatorsData[validator_index].AttestationsIdealHeadReward = utils.NullInt32(idealRewardsOfValidator.Head) - validatorsData[validator_index].AttestationsIdealTargetReward = utils.NullInt32(idealRewardsOfValidator.Target) - validatorsData[validator_index].AttestationsIdealSourceReward = utils.NullInt32(idealRewardsOfValidator.Source) - validatorsData[validator_index].AttestationsIdealInactivityPenalty = utils.NullInt32(idealRewardsOfValidator.Inactivity) - validatorsData[validator_index].AttestationsIdealInclusionsReward = utils.NullInt32(idealRewardsOfValidator.InclusionDelay) - validatorsData[validator_index].AttestationIdealReward = utils.NullInt64(int64(idealRewardsOfValidator.Head + idealRewardsOfValidator.Source + idealRewardsOfValidator.Target + idealRewardsOfValidator.Inactivity + idealRewardsOfValidator.InclusionDelay)) - - if attestationReward.Head > 0 { - validatorsData[validator_index].AttestationsHeadExecuted = utils.NullInt16(1) - validatorsData[validator_index].AttestationsObserved = utils.NullInt16(1) - } - if attestationReward.Source > 0 { - validatorsData[validator_index].AttestationsSourceExecuted = utils.NullInt16(1) - validatorsData[validator_index].AttestationsObserved = utils.NullInt16(1) - } - if attestationReward.Target > 0 { - validatorsData[validator_index].AttestationsTargetExecuted = utils.NullInt16(1) - validatorsData[validator_index].AttestationsObserved = utils.NullInt16(1) + writeMutex.Unlock() + return nil + }) } - } - - return validatorsData, nil -} - -func storeClBlockRewards(data map[uint64]*constypes.StandardBlockRewardsResponse) error { - tx, err := db.AlloyWriter.Beginx() - if err != nil { - return errors.Wrap(err, "failed to start cl blocks transaction") - } - defer utils.Rollback(tx) - - for slot, rewards := range data { - _, err := tx.Exec(` - INSERT INTO consensus_payloads (slot, cl_attestations_reward, cl_sync_aggregate_reward, cl_slashing_inclusion_reward) - VALUES ($1, $2, $3, $4) - ON CONFLICT (slot) DO NOTHING - `, slot, rewards.Data.Attestations, rewards.Data.SyncAggregate, rewards.Data.AttesterSlashings+rewards.Data.ProposerSlashings) + err := g2.Wait() if err != nil { - return errors.Wrap(err, "failed to insert cl blocks data") + return fmt.Errorf("error in attestation assignments: %w", err) } - } - - if err := tx.Commit(); err != nil { - return errors.Wrap(err, "failed to commit cl blocks transaction") - } - - return nil -} - -func parseEpochRange(pattern, partition string) (uint64, uint64, error) { - // Compile the regular expression pattern - regex := regexp.MustCompile(pattern) - - // Find the matches in the partition string - matches := regex.FindStringSubmatch(partition) - - // Check if the partition string matches the pattern - if len(matches) != 3 { - return 0, 0, fmt.Errorf("invalid partition string: %s", partition) - } - - // Parse the epoch range values - epochFrom, err := strconv.ParseUint(matches[1], 10, 64) - if err != nil { - return 0, 0, fmt.Errorf("failed to parse epoch from: %w", err) - } - - epochTo, err := strconv.ParseUint(matches[2], 10, 64) - if err != nil { - return 0, 0, fmt.Errorf("failed to parse epoch to: %w", err) - } - - return epochFrom, epochTo, nil -} - -type DataEpoch struct { - DataRaw *Data - Epoch uint64 -} - -type DataEpochProcessed struct { - Data []*validatorDashboardDataRow - Epoch uint64 -} - -type EpochParallelGroup struct { - Epochs []uint64 - Sequential bool -} - -type validatorDashboardDataRow struct { - AttestationsSourceReward sql.NullInt32 //done - AttestationsTargetReward sql.NullInt32 //done - AttestationsHeadReward sql.NullInt32 //done - AttestationsInactivityPenalty sql.NullInt32 //done - AttestationsInclusionsReward sql.NullInt32 //done - AttestationReward sql.NullInt64 //done - AttestationsIdealSourceReward sql.NullInt32 //done - AttestationsIdealTargetReward sql.NullInt32 //done - AttestationsIdealHeadReward sql.NullInt32 //done - AttestationsIdealInactivityPenalty sql.NullInt32 //done - AttestationsIdealInclusionsReward sql.NullInt32 //done - AttestationIdealReward sql.NullInt64 //done - - AttestationsScheduled sql.NullInt16 //done - AttestationsObserved sql.NullInt16 //done - AttestationsHeadExecuted sql.NullInt16 //done - AttestationsSourceExecuted sql.NullInt16 //done - AttestationsTargetExecuted sql.NullInt16 //done - - LastSubmittedDutyEpoch sql.NullInt32 // does not include sync committee duty slots - - BlockScheduled sql.NullInt16 // done - BlocksProposed sql.NullInt16 // done - BlocksExpectedThisEpoch float64 // done - SyncCommitteesExpectedThisPeriod float64 // done - - BlocksClReward sql.NullInt64 // done - BlocksClAttestestationsReward sql.NullInt64 // done - BlocksClSyncAggregateReward sql.NullInt64 // done - - SyncScheduled sql.NullInt16 // done - SyncExecuted sql.NullInt16 // done - SyncReward sql.NullInt64 // done - - SlasherRewards sql.NullInt64 // done - Slashed bool // done - SlashedBy sql.NullInt32 // done - SlashedViolation sql.NullInt16 // done - - BalanceStart uint64 // done - BalanceEnd uint64 // done - - DepositsCount sql.NullInt16 // done - DepositsAmount sql.NullInt64 // done - - WithdrawalsCount sql.NullInt16 // done - WithdrawalsAmount sql.NullInt64 // done - - InclusionDelaySum sql.NullInt16 // done - OptimalInclusionDelay sql.NullInt16 // done -} - -const SLASHED_VIOLATION_ATTESTATION = 1 -const SLASHED_VIOLATION_PROPOSER = 2 - -const RawSyncCommitteeCacheKey = "sync_committee_period_" - -type ResponseCache struct { - cache map[string]any -} - -func (r *ResponseCache) SetSyncCommittee(period uint64, data *constypes.StandardSyncCommitteesResponse) { - r.cache[r.GetSyncCommitteeCacheKey(period)] = data -} - -func (r *ResponseCache) GetSyncCommittee(period uint64) *constypes.StandardSyncCommitteesResponse { - temp, ok := r.cache[r.GetSyncCommitteeCacheKey(period)] - if !ok { return nil - } - return temp.(*constypes.StandardSyncCommitteesResponse) -} - -func (r *ResponseCache) GetSyncCommitteeCacheKey(period uint64) string { - return fmt.Sprintf("%s%d", RawSyncCommitteeCacheKey, period) -} - -func refreshMaterializedSlashedByCounts() error { - tx, err := db.AlloyWriter.Beginx() - if err != nil { - return errors.Wrap(err, "failed to start transaction") - } - defer utils.Rollback(tx) - - _, err = tx.Exec(` - CREATE MATERIALIZED VIEW IF NOT EXISTS validator_dashboard_data_rolling_total_slashedby_count AS - SELECT slashed_by, COUNT(*) as slashed_amount - FROM validator_dashboard_data_rolling_total - WHERE slashed = true AND slashed_by IS NOT NULL - GROUP BY slashed_by; - CREATE INDEX IF NOT EXISTS idx_validator_dashboard_data_rolling_total_slashedby_count_slashed_by ON validator_dashboard_data_rolling_total_slashedby_count(slashed_by); - - CREATE MATERIALIZED VIEW IF NOT EXISTS validator_dashboard_data_rolling_daily_slashedby_count AS - SELECT slashed_by, COUNT(*) as slashed_amount - FROM validator_dashboard_data_rolling_daily - WHERE slashed = true AND slashed_by IS NOT NULL - GROUP BY slashed_by; - CREATE INDEX IF NOT EXISTS idx_validator_dashboard_data_rolling_daily_slashedby_count_slashed_by ON validator_dashboard_data_rolling_daily_slashedby_count(slashed_by); - - CREATE MATERIALIZED VIEW IF NOT EXISTS validator_dashboard_data_rolling_weekly_slashedby_count AS - SELECT slashed_by, COUNT(*) as slashed_amount - FROM validator_dashboard_data_rolling_weekly - WHERE slashed = true AND slashed_by IS NOT NULL - GROUP BY slashed_by; - CREATE INDEX IF NOT EXISTS idx_validator_dashboard_data_rolling_weekly_slashedby_count_slashed_by ON validator_dashboard_data_rolling_weekly_slashedby_count(slashed_by); - - CREATE MATERIALIZED VIEW IF NOT EXISTS validator_dashboard_data_rolling_monthly_slashedby_count AS - SELECT slashed_by, COUNT(*) as slashed_amount - FROM validator_dashboard_data_rolling_monthly - WHERE slashed = true AND slashed_by IS NOT NULL - GROUP BY slashed_by; - CREATE INDEX IF NOT EXISTS idx_validator_dashboard_data_rolling_monthly_slashedby_count_slashed_by ON validator_dashboard_data_rolling_monthly_slashedby_count(slashed_by); - - CREATE MATERIALIZED VIEW IF NOT EXISTS validator_dashboard_data_epoch_slashedby_count AS - SELECT epoch, slashed_by, COUNT(*) as slashed_amount - FROM validator_dashboard_data_epoch - WHERE slashed = true AND slashed_by IS NOT NULL - GROUP BY epoch, slashed_by; - CREATE INDEX IF NOT EXISTS idx_validator_dashboard_data_epoch_slashedby_count_epoch_slashed_by ON validator_dashboard_data_epoch_slashedby_count(epoch, slashed_by); - - REFRESH MATERIALIZED VIEW validator_dashboard_data_rolling_total_slashedby_count; - REFRESH MATERIALIZED VIEW validator_dashboard_data_rolling_daily_slashedby_count; - REFRESH MATERIALIZED VIEW validator_dashboard_data_rolling_weekly_slashedby_count; - REFRESH MATERIALIZED VIEW validator_dashboard_data_rolling_monthly_slashedby_count; - REFRESH MATERIALIZED VIEW validator_dashboard_data_epoch_slashedby_count; - `) + }) + err := g1.Wait() if err != nil { - return errors.Wrap(err, "failed to refresh materialized views") + return fmt.Errorf("error in getDataForEpochRange: %w", err) } - return tx.Commit() + return nil } - -// Can be used to backfill old missing cl block rewards -// Commented out since this is a one time operation, kept in in case we need it again -// func (d *dashboardData) backfillCLBlockRewards() { -// upTo := 1731488 -// startFrom := 1328805 -// batchSize := 8 -// parallelization := 8 - -// blocksChan := make(chan map[uint64]*constypes.StandardBlockRewardsResponse, 1) - -// err := db.AlloyWriter.Get(&startFrom, "SELECT last_slot FROM meta_slot_export") -// if err != nil { -// d.log.Error(err, "failed to get last slot from meta_slot_export", 0) -// return -// } - -// // re export 317201 +- 20000 - -// go func() { -// for i := startFrom; i < upTo+batchSize; i += batchSize { -// batched := map[uint64]*constypes.StandardBlockRewardsResponse{} -// mutex := &sync.Mutex{} - -// errgroup := &errgroup.Group{} -// errgroup.SetLimit(parallelization) - -// for slot := i; slot < i+batchSize; slot++ { -// slot := slot -// errgroup.Go(func() error { -// blockReward, err := d.CL.GetPropoalRewards(slot) -// if err != nil { -// httpErr := network.SpecificError(err) -// if httpErr != nil && httpErr.StatusCode == http.StatusNotFound { -// return nil -// } - -// d.log.Error(err, "can not get block reward data", 0, map[string]interface{}{"slot": slot}) -// return err -// } - -// mutex.Lock() -// batched[uint64(slot)] = blockReward -// mutex.Unlock() - -// return nil -// }) -// } - -// err := errgroup.Wait() -// if err != nil { -// d.log.Error(err, "failed to backfill cl block rewards", 0) -// close(blocksChan) -// } - -// blocksChan <- batched -// } -// }() - -// go func() { -// for blockReward := range blocksChan { -// for { -// err := storeClBlockRewards(blockReward) -// if err != nil { -// d.log.Error(err, "failed to store cl block rewards", 0) -// continue -// } -// break -// } -// highestSlot := uint64(0) -// for slot := range blockReward { -// if slot > highestSlot { -// highestSlot = slot -// } -// } - -// if highestSlot%100 < uint64(batchSize) { -// d.log.Infof("processed blocks, height: %d", highestSlot) -// } - -// if highestSlot%10000 < uint64(batchSize) { -// _, err := db.AlloyWriter.Exec("UPDATE meta_slot_export SET last_slot = $1", highestSlot) -// if err != nil { -// d.log.Error(err, "failed to update last slot in meta_slot_export", 0) -// } -// } -// } -// }() -// } diff --git a/backend/pkg/exporter/modules/dashboard_data_insert.go b/backend/pkg/exporter/modules/dashboard_data_insert.go new file mode 100644 index 000000000..336fb695d --- /dev/null +++ b/backend/pkg/exporter/modules/dashboard_data_insert.go @@ -0,0 +1,226 @@ +package modules + +import ( + "fmt" + "slices" + "time" + + "github.com/gobitfly/beaconchain/pkg/commons/db" + "github.com/gobitfly/beaconchain/pkg/commons/log" + "github.com/gobitfly/beaconchain/pkg/commons/metrics" + "github.com/gobitfly/beaconchain/pkg/commons/utils" + edb "github.com/gobitfly/beaconchain/pkg/exporter/db" + "github.com/gobitfly/beaconchain/pkg/exporter/types" + "github.com/google/uuid" + "golang.org/x/exp/maps" + "golang.org/x/sync/errgroup" + + "github.com/pkg/errors" +) + +// | handleIncompleteInserts +// | - GetIncompleteInsertEpochs +// | - FetchEpochs A +// | - PushEpochs B +// | - PushEpochMetadata (successful_insert) C +// | handlePendingInserts +// | - GetPendingInsertEpochs +// | - allocate insert batch ids +// | - PushEpochMetadata (insert_batch_id) +// | - FetchEpochs A +// | - PushEpochs B +// | - PushEpochMetadata (successful_insert) C + +func (d *dashboardData) insertTask() { + for { + // loop to complete incomplete epochs + err := d.handleIncompleteInserts() + if err != nil { + d.log.Error(err, "failed to handle incomplete inserts", 0) + time.Sleep(10 * time.Second) + continue + } + err = d.handlePendingInserts() + if err != nil { + d.log.Error(err, "failed to handle pending inserts", 0) + time.Sleep(10 * time.Second) + continue + } + time.Sleep(1 * time.Second) + } +} + +func (d *dashboardData) handleIncompleteInserts() error { + start := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_handle_incomplete_inserts").Observe(time.Since(start).Seconds()) + }() + // get latest unsafe epoch + latestUnsafeEpoch, err := edb.GetLatestUnsafeEpoch() + if err != nil { + return errors.Wrap(err, "failed to get latest unsafe epoch") + } + metrics.State.WithLabelValues("dashboard_data_exporter_latest_unsafe_epoch").Set(float64(latestUnsafeEpoch)) + defer func() { + latestUnsafeEpoch, err := edb.GetLatestUnsafeEpoch() + if err != nil { + d.log.Error(err, "failed to get latest unsafe epoch", 0) + return + } + metrics.State.WithLabelValues("dashboard_data_exporter_latest_unsafe_epoch").Set(float64(latestUnsafeEpoch)) + }() + + incomplete, err := edb.GetIncompleteInsertEpochs() + if err != nil { + return errors.Wrap(err, "failed to get incomplete insert epochs") + } + if len(incomplete) == 0 { + d.log.Debugf("handleIncompleteInserts, no incomplete insert epochs") + return nil + } + d.log.Infof("handleIncompleteInserts, found %d incomplete insert epochs", len(incomplete)) + // we do grouping here even tho fetchAndInsertEpochs does grouping again + // this is because we can in theory have incomplete epochs that aren't next to each other + // and fetchAndInsert wants to do only one fetch of all the epochs + // also means incomplete uses lower memory than pending, which seems like a sane option + insertBatchEpochs := make(map[uuid.UUID][]edb.EpochMetadata) + for _, e := range incomplete { + if e.InsertBatchID == nil { + return fmt.Errorf("insert batch id is nil for epoch %v", e) + } + id := *e.InsertBatchID + insertBatchEpochs[id] = append(insertBatchEpochs[id], e) + } + for _, epochs := range insertBatchEpochs { + err = d.fetchAndInsertEpochs(epochs) + // fail as soon as we run into an error. outer loop will call us again and + // since the insert id (likely) isnt marked as successful, we will automagically try again + if err != nil { + return fmt.Errorf("failed to fetch and insert epochs: %v", err) + } + } + + return nil +} + +func (d *dashboardData) handlePendingInserts() error { + start := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_handle_pending_inserts").Observe(time.Since(start).Seconds()) + }() + + safeEpoch := d.latestSafeEpoch.Load() + pending, err := edb.GetPendingInsertEpochs(safeEpoch, utils.Config.DashboardExporter.FetchAtOnceLimit) + if err != nil { + return errors.Wrap(err, "failed to get pending insert epochs") + } + if len(pending) == 0 { + d.log.Debugf("handlePendingInserts, no pending insert epochs") + return nil + } + d.log.Infof("handlePendingInserts, found %d pending insert epochs", len(pending)) + + // allocate insert batch ids + batchIDS := make([]uuid.UUID, 0) + for i := range pending { + if int64(i)%utils.Config.DashboardExporter.InsertAtOnceLimit == 0 { + id := uuid.New() + batchIDS = append(batchIDS, id) + } + pending[i].InsertBatchID = &batchIDS[len(batchIDS)-1] + } + + err = edb.PushEpochMetadata(pending) + if err != nil { + return errors.Wrap(err, "failed to push epoch metadata") + } + + // fetch and insert in parallel. do a single fetch for all epochs + err = d.fetchAndInsertEpochs(pending) + if err != nil { + return errors.Wrap(err, "failed to fetch and insert pending epochs") + } + + return nil +} + +// a-c func +func (d *dashboardData) fetchAndInsertEpochs(epochs []edb.EpochMetadata) error { + // sanity check that epochs are in order, have no gaps + slices.SortFunc(epochs, func(a, b edb.EpochMetadata) int { + return int(a.Epoch) - int(b.Epoch) + }) + // check for gaps + for i := 1; i < len(epochs); i++ { + if epochs[i].Epoch-epochs[i-1].Epoch != 1 { + return fmt.Errorf("epoch gap between %d and %d", epochs[i-1].Epoch, epochs[i].Epoch) + } + } + insertBatchEpochs := make(map[uuid.UUID][]edb.EpochMetadata) + for _, e := range epochs { + if e.InsertBatchID == nil { + return fmt.Errorf("insert batch id is nil for epoch %v", e) + } + id := *e.InsertBatchID + insertBatchEpochs[id] = append(insertBatchEpochs[id], e) + } + + // fetch + rawData := NewMultiEpochData(len(epochs)) + err := d.getDataForEpochRange(epochs[0].Epoch, epochs[len(epochs)-1].Epoch, &rawData) + if err != nil { + return errors.Wrap(err, "failed to get data for epochs") + } + + // process + processedData := make([]types.VDBDataEpochColumns, len(maps.Keys(insertBatchEpochs))) // grouped by insert batch id + err = d.processRunner(&rawData, &processedData, epochs) + if err != nil { + return errors.Wrap(err, "failed to process data") + } + // insert step wooho + eg := &errgroup.Group{} + eg.SetLimit(int(utils.Config.DashboardExporter.InsertInParallel)) + now := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_insert_overall").Observe(time.Since(now).Seconds()) + }() + // loop processed data, its grouped by insert batch id already and ready to be inserted + for i := range processedData { + data := processedData[i] + insertId := data.InsertBatchID[0] + epochs, ok := insertBatchEpochs[insertId] + if !ok { + return fmt.Errorf("insert batch id %s not found in insert batch epochs", insertId) + } + eg.Go(func() error { + d.log.Infof("doing insert batch %s", insertId) + now := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_insert_batch").Observe(time.Since(now).Seconds()) + }() + err := db.UltraFastDumpToClickhouse(&data, edb.EpochWriterSink, insertId.String()) + if err != nil { + d.log.Error(err, "failed to insert epochs", 0, log.Fields{"epochs": data}) + return errors.Wrap(err, "failed to insert epochs") + } + // mark as successful + sfts := time.Now() + for i := range epochs { + epochs[i].SuccessfulInsert = &sfts + } + + err = edb.PushEpochMetadata(epochs) + if err != nil { + d.log.Error(err, "failed to push epoch metadata", 0, log.Fields{"epochs": insertBatchEpochs[insertId]}) + return errors.Wrap(err, "failed to push epoch metadata") + } + return nil + }) + } + err = eg.Wait() + if err != nil { + return errors.Wrap(err, "failed to insert all epochs") + } + return nil +} diff --git a/backend/pkg/exporter/modules/dashboard_data_maintenance.go b/backend/pkg/exporter/modules/dashboard_data_maintenance.go new file mode 100644 index 000000000..7ff914383 --- /dev/null +++ b/backend/pkg/exporter/modules/dashboard_data_maintenance.go @@ -0,0 +1,140 @@ +package modules + +import ( + "fmt" + "time" + + "github.com/gobitfly/beaconchain/pkg/commons/log" + "github.com/gobitfly/beaconchain/pkg/commons/metrics" + "github.com/gobitfly/beaconchain/pkg/commons/utils" + edb "github.com/gobitfly/beaconchain/pkg/exporter/db" + "github.com/google/uuid" + + "github.com/pkg/errors" + "golang.org/x/sync/errgroup" +) + +// | handleIncompleteTransfers +// | - GetIncompleteTransferEpochs +// | - TransferEpochs A +// | - PushEpochMetadata (successful_transfer) B +// | handlePendingTransfers +// | - GetPendingTransferEpochs +// | - allocate transfer batch ids +// | - PushEpochMetadata (transfer_batch_id) +// | - TransferEpochs A +// | - PushEpochMetadata (successful_transfer) B + +func (d *dashboardData) maintenanceTask() { + for { + // loop to complete incomplete epochs + err := d.handleIncompleteTransfers() + if err != nil { + d.log.Error(err, "failed to handle incomplete transfers", 0) + time.Sleep(10 * time.Second) + continue + } + err = d.handlePendingTransfers() + if err != nil { + d.log.Error(err, "failed to handle pending transfers", 0) + time.Sleep(10 * time.Second) + continue + } + time.Sleep(1 * time.Second) + } +} +func (d *dashboardData) handleIncompleteTransfers() error { + incomplete, err := edb.GetIncompleteTransferEpochs() + if err != nil { + return errors.Wrap(err, "failed to get incomplete transfer epochs") + } + if len(incomplete) == 0 { + d.log.Debugf("handleIncompleteTransfers, no incomplete transfer epochs") + return nil + } + d.log.Infof("handleIncompleteTransfers, found %d incomplete transfer epochs", len(incomplete)) + + err = d.transferEpochs(incomplete) + if err != nil { + return errors.Wrap(err, "failed to transfer incomplete epochs") + } + return nil +} + +func (d *dashboardData) handlePendingTransfers() error { + pending, err := edb.GetPendingTransferEpochs(utils.Config.DashboardExporter.TransferAtOnce * utils.Config.DashboardExporter.TransferInParallel) + if err != nil { + return errors.Wrap(err, "failed to get pending transfer epochs") + } + if len(pending) == 0 { + d.log.Debugf("handlePendingTransfers, no pending transfer epochs") + return nil + } + d.log.Infof("handlePendingTransfers, found %d pending transfer epochs", len(pending)) + + // allocate transfer batch ids + batchIds := make([]uuid.UUID, 0) + for i := range pending { + if i%int(utils.Config.DashboardExporter.TransferAtOnce) == 0 { + id := uuid.New() + batchIds = append(batchIds, id) + } + pending[i].TransferBatchId = &batchIds[len(batchIds)-1] + } + + err = edb.PushEpochMetadata(pending) + if err != nil { + return errors.Wrap(err, "failed to push epoch metadata") + } + + err = d.transferEpochs(pending) + if err != nil { + return errors.Wrap(err, "failed to transfer pending epochs") + } + return nil +} + +func (d *dashboardData) transferEpochs(epochs []edb.EpochMetadata) error { + now := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_transfer_overall").Observe(time.Since(now).Seconds()) + }() + transferBatchEpochs := make(map[uuid.UUID][]edb.EpochMetadata) + for _, e := range epochs { + if e.TransferBatchId == nil { + return fmt.Errorf("transfer batch id is nil for epoch %v", e) + } + id := *e.TransferBatchId + transferBatchEpochs[id] = append(transferBatchEpochs[id], e) + } + eg := &errgroup.Group{} + eg.SetLimit(int(utils.Config.DashboardExporter.TransferInParallel)) + for id, epochs := range transferBatchEpochs { + epochs := epochs + id := id + eg.Go(func() error { + d.log.Infof("doing transfer batch %s", id) + err := edb.TransferEpochs(epochs) + if err != nil { + d.log.Error(err, "failed to transfer epochs", 0, log.Fields{"epochs": epochs}) + return errors.Wrap(err, "failed to transfer epochs") + } + + now := time.Now() + for i := range epochs { + epochs[i].SuccessfulTransfer = &now + } + err = edb.PushEpochMetadata(epochs) + if err != nil { + d.log.Error(err, "failed to push epoch metadata", 0, log.Fields{"epochs": epochs}) + return errors.Wrap(err, "failed to push epoch metadata") + } + return nil + }) + } + err := eg.Wait() + if err != nil { + return errors.Wrap(err, "failed to complete transfer batches") + } + return nil +} diff --git a/backend/pkg/exporter/modules/dashboard_data_process.go b/backend/pkg/exporter/modules/dashboard_data_process.go new file mode 100644 index 000000000..1bcc4a3a1 --- /dev/null +++ b/backend/pkg/exporter/modules/dashboard_data_process.go @@ -0,0 +1,1065 @@ +package modules + +import ( + "bytes" + "fmt" + "slices" + "time" + + "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/gobitfly/beaconchain/pkg/commons/metrics" + "github.com/gobitfly/beaconchain/pkg/commons/utils" + edb "github.com/gobitfly/beaconchain/pkg/exporter/db" + "github.com/gobitfly/beaconchain/pkg/exporter/types" + "github.com/google/uuid" + "golang.org/x/sync/errgroup" + + "github.com/pkg/errors" +) + +func (d *dashboardData) processRunner(data *MultiEpochData, tar *[]types.VDBDataEpochColumns, epochs []edb.EpochMetadata) error { + d.log.Info("starting processRunner") + start := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_process_overall").Observe(time.Since(start).Seconds()) + }() + var err error + seenInsertIds := []uuid.UUID{*epochs[0].InsertBatchID} + currentTarIndex := 0 + bulkValiCount := 0 + bulkEpochsContained := make([]uint64, 0) + // sanity check, epochs in data.epochsBaseData.epochs should be identical to []epochs.Epoch + for i, epoch := range data.epochBasedData.epochs { + if epoch != epochs[i].Epoch { + return fmt.Errorf("epoch mismatch, expected %d, got %d", epochs[i].Epoch, epoch) + } + } + + // prepare tar + for i, epoch := range epochs { + insertId := *epoch.InsertBatchID + // debug log + d.log.Infof("epoch %d, insertId %s", epoch.Epoch, insertId) + if insertId != seenInsertIds[len(seenInsertIds)-1] { + d.log.Infof("allocating tar insert id %s with %d entries", seenInsertIds[len(seenInsertIds)-1], bulkValiCount) + (*tar)[currentTarIndex], err = types.NewVDBDataEpochColumns(bulkValiCount) + if err != nil { + return errors.Wrap(err, "failed to allocate new VDBDataEpochColumns") + } + (*tar)[currentTarIndex].EpochsContained = bulkEpochsContained + (*tar)[currentTarIndex].InsertBatchID = []uuid.UUID{seenInsertIds[len(seenInsertIds)-1]} // important to use insertID because pointers + currentTarIndex++ + bulkEpochsContained = make([]uint64, 0) + bulkValiCount = 0 + seenInsertIds = append(seenInsertIds, insertId) + } + d.log.Infof("adding epoch %d to tar insert id %s", epoch.Epoch, insertId) + bulkEpochsContained = append(bulkEpochsContained, epoch.Epoch) + data.epochBasedData.tarIndices[i] = currentTarIndex + data.epochBasedData.tarOffsets[i] = bulkValiCount + bulkValiCount += len(data.epochBasedData.validatorStates[int64(epoch.Epoch)].Data) + } + if bulkValiCount > 0 { + d.log.Infof("leftover valis, allocating tar index %d with %d entries", currentTarIndex, bulkValiCount) + (*tar)[currentTarIndex], err = types.NewVDBDataEpochColumns(bulkValiCount) + if err != nil { + return errors.Wrap(err, "failed to allocate new VDBDataEpochColumns") + } + (*tar)[currentTarIndex].EpochsContained = bulkEpochsContained + (*tar)[currentTarIndex].InsertBatchID = []uuid.UUID{seenInsertIds[len(seenInsertIds)-1]} // important to use insertID because pointers + } + for i := range *tar { + d.log.Warnf("tar %d, insertId %s, epochs %v, seendInsertIds %v", i, (*tar)[i].InsertBatchID, (*tar)[i].EpochsContained, seenInsertIds) + } + d.log.Infof("prepared tar with %d entries", len(*tar)) + // errgroup + g := &errgroup.Group{} + g.Go(func() error { + start := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_process_validator_states_overall").Observe(time.Since(start).Seconds()) + }() + err := d.processValidatorStates(data, tar) + if err != nil { + return fmt.Errorf("error in processValidatorStates: %w", err) + } + return nil + }) + g.Go(func() error { + start := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_process_scheduled_attestations_overall").Observe(time.Since(start).Seconds()) + }() + err := d.processScheduledAttestations(data, tar) + if err != nil { + return fmt.Errorf("error in processScheduledAttestations: %w", err) + } + return nil + }) + g.Go(func() error { + start := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_process_blocks_overall").Observe(time.Since(start).Seconds()) + }() + err := d.processBlocks(data, tar) + if err != nil { + return fmt.Errorf("error in processBlocks: %w", err) + } + return nil + }) + g.Go(func() error { + start := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_process_deposits_overall").Observe(time.Since(start).Seconds()) + }() + err := d.processDeposits(data, tar) + if err != nil { + return fmt.Errorf("error in processDeposits: %w", err) + } + return nil + }) + // force sequential operation of attestation rewards and proposal rewards + if data.epochBasedData.epochs[len(data.epochBasedData.epochs)-1] < utils.Config.Chain.ClConfig.AltairForkEpoch { + d.phase0HotfixMutex.Lock() + } + g.Go(func() error { + start := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_process_attestation_rewards_overall").Observe(time.Since(start).Seconds()) + }() + err := d.processAttestationRewards(data, tar) + if err != nil { + return fmt.Errorf("error in processAttestationRewards: %w", err) + } + return nil + }) + g.Go(func() error { + start := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_process_proposal_rewards_overall").Observe(time.Since(start).Seconds()) + }() + err := d.processWithdrawals(data, tar) + if err != nil { + return fmt.Errorf("error in processWithdrawals: %w", err) + } + return nil + }) + g.Go(func() error { + start := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_process_attestations_overall").Observe(time.Since(start).Seconds()) + }() + err := d.processAttestations(data, tar) + if err != nil { + return fmt.Errorf("error in processAttestations: %w", err) + } + return nil + }) + // processExpectedSyncPeriods + g.Go(func() error { + start := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_process_expected_sync_periods_overall").Observe(time.Since(start).Seconds()) + }() + err := d.processExpectedSyncPeriods(data, tar) + if err != nil { + return fmt.Errorf("error in processExpectedSyncPeriods: %w", err) + } + return nil + }) + // processSyncVotes + g.Go(func() error { + start := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_process_sync_votes_overall").Observe(time.Since(start).Seconds()) + }() + err := d.processSyncVotes(data, tar) + if err != nil { + return fmt.Errorf("error in processSyncVotes: %w", err) + } + d.log.Infof("processed sync votes in %v", time.Since(start)) + return nil + }) + // processProposalRewards + g.Go(func() error { + start := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_process_proposal_rewards_overall").Observe(time.Since(start).Seconds()) + }() + err := d.processProposalRewards(data, tar) + if err != nil { + return fmt.Errorf("error in processProposalRewards: %w", err) + } + return nil + }) + // processSyncCommitteeRewards + g.Go(func() error { + start := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_process_sync_committee_rewards_overall").Observe(time.Since(start).Seconds()) + }() + err := d.processSyncCommitteeRewards(data, tar) + if err != nil { + return fmt.Errorf("error in processSyncCommitteeRewards: %w", err) + } + return nil + }) + // processBlocksExpected + g.Go(func() error { + start := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_process_blocks_expected_overall").Observe(time.Since(start).Seconds()) + }() + err := d.processBlocksExpected(data, tar) + if err != nil { + return fmt.Errorf("error in processBlocksExpected: %w", err) + } + return nil + }) + + err = g.Wait() + if err != nil { + return fmt.Errorf("error in processRunner: %w", err) + } + + return nil +} + +func (d *dashboardData) processValidatorStates(data *MultiEpochData, tar *[]types.VDBDataEpochColumns) error { + g := &errgroup.Group{} + for i, e := range data.epochBasedData.epochs { + epoch := e + iEpoch := int64(epoch) + tI := data.epochBasedData.tarIndices[i] + tO := data.epochBasedData.tarOffsets[i] + startSate := data.epochBasedData.validatorStates[iEpoch-1] + endState := data.epochBasedData.validatorStates[iEpoch] + ts := utils.EpochToTime(epoch) + g.Go(func() error { + start := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_process_validator_states_single").Observe(time.Since(start).Seconds()) + }() + for j := range endState.Data { + (*tar)[tI].ValidatorIndex[tO+j] = uint64(j) + (*tar)[tI].Epoch[tO+j] = iEpoch + (*tar)[tI].EpochTimestamp[tO+j] = &ts + (*tar)[tI].BalanceEnd[tO+j] = int64(endState.Data[j].Balance) + (*tar)[tI].BalanceEffectiveEnd[tO+j] = int64(endState.Data[j].EffectiveBalance) + // do NOT set the slashed flag here. it is set by the block processing + } + if epoch == 0 { + // we do not set the start state for epoch 0, this is because validators that join + // after the genesis epoch will always have a start balance of 0 (they didn't exist + // in the previous epoch yet) and we want to emulate the same behavior + return nil + } + for j := range startSate.Data { + (*tar)[tI].BalanceStart[tO+j] = int64(startSate.Data[j].Balance) + (*tar)[tI].BalanceEffectiveStart[tO+j] = int64(startSate.Data[j].EffectiveBalance) + } + return nil + }) + } + return g.Wait() +} + +func (d *dashboardData) processScheduledAttestations(data *MultiEpochData, tar *[]types.VDBDataEpochColumns) error { + g := &errgroup.Group{} + for i, e := range data.epochBasedData.epochs { + epoch := e + tI := data.epochBasedData.tarIndices[i] + tO := data.epochBasedData.tarOffsets[i] + startSlot := epoch * utils.Config.Chain.ClConfig.SlotsPerEpoch + endSlot := startSlot + utils.Config.Chain.ClConfig.SlotsPerEpoch + g.Go(func() error { + now := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_process_scheduled_attestations_single").Observe(time.Since(now).Seconds()) + }() + // pre-init should not be required, is pointer and defaults to null + for slot := startSlot; slot < endSlot; slot++ { + // fetch from attestati on assignments + if _, ok := data.slotBasedData.assignments.attestationAssignments[slot]; !ok { + // error, should never happen, we should always have assignments + return fmt.Errorf("no attestation assignments for slot %d", slot) + } + for committee, validatorIndices := range data.slotBasedData.assignments.attestationAssignments[slot] { + for committeeIndex, validatorIndex := range validatorIndices { + (*tar)[tI].AttestationsScheduled[tO+int(validatorIndex)]++ + (*tar)[tI].AttestationAssignmentsSlot[tO+int(validatorIndex)] = append( + (*tar)[tI].AttestationAssignmentsSlot[tO+int(validatorIndex)], + int64(slot), + ) + (*tar)[tI].AttestationAssignmentsCommittee[tO+int(validatorIndex)] = append( + (*tar)[tI].AttestationAssignmentsCommittee[tO+int(validatorIndex)], + int64(committee), + ) + (*tar)[tI].AttestationAssignmentsIndex[tO+int(validatorIndex)] = append( + (*tar)[tI].AttestationAssignmentsIndex[tO+int(validatorIndex)], + int64(committeeIndex), + ) + } + } + } + return nil + }) + } + return g.Wait() +} + +func (d *dashboardData) processBlocks(data *MultiEpochData, tar *[]types.VDBDataEpochColumns) error { + g := &errgroup.Group{} + for i, e := range data.epochBasedData.epochs { + epoch := e + tI := data.epochBasedData.tarIndices[i] + tO := data.epochBasedData.tarOffsets[i] + startSlot := epoch * utils.Config.Chain.ClConfig.SlotsPerEpoch + endSlot := startSlot + utils.Config.Chain.ClConfig.SlotsPerEpoch + g.Go(func() error { + now := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_process_blocks_single").Observe(time.Since(now).Seconds()) + }() + for slot := startSlot; slot < endSlot; slot++ { + // skip slot 0, as nobody proposed it + if slot == 0 { + continue + } + proposer := data.slotBasedData.assignments.blockAssignments[slot] + (*tar)[tI].BlocksStatusSlot[tO+int(proposer)] = append((*tar)[tI].BlocksStatusSlot[tO+int(proposer)], int64(slot)) + + if _, ok := data.slotBasedData.blocks[slot]; !ok { + (*tar)[tI].BlocksStatusProposed[tO+int(proposer)] = append((*tar)[tI].BlocksStatusProposed[tO+int(proposer)], false) + continue + } + + (*tar)[tI].BlocksStatusProposed[tO+int(proposer)] = append((*tar)[tI].BlocksStatusProposed[tO+int(proposer)], true) + + for _, index := range data.slotBasedData.blocks[slot].SlashedIndices { + (*tar)[tI].Slashed[tO+int(index)] = true + (*tar)[tI].BlocksSlashingCount[uint64(tO)+data.slotBasedData.blocks[slot].ProposerIndex]++ + } + } + return nil + }) + } + return g.Wait() +} + +func (d *dashboardData) processDeposits(data *MultiEpochData, tar *[]types.VDBDataEpochColumns) error { + if d.signingDomain == nil { + domain, err := utils.GetSigningDomain() + if err != nil { + return err + } + d.signingDomain = domain + } + g := &errgroup.Group{} + for i, e := range data.epochBasedData.epochs { + epoch := e + tI := data.epochBasedData.tarIndices[i] + tO := data.epochBasedData.tarOffsets[i] + startSlot := epoch * utils.Config.Chain.ClConfig.SlotsPerEpoch + endSlot := startSlot + utils.Config.Chain.ClConfig.SlotsPerEpoch + g.Go(func() error { + now := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_process_deposits_single").Observe(time.Since(now).Seconds()) + }() + // genesis deposits + if epoch == 0 { + for j := range data.epochBasedData.validatorStates[0].Data { + (*tar)[tI].DepositsCount[tO+j] = 1 + (*tar)[tI].DepositsAmount[tO+j] = int64(data.epochBasedData.validatorStates[0].Data[j].Balance) + } + } + for jj := startSlot; jj < endSlot; jj++ { + slot := jj + if _, ok := data.slotBasedData.blocks[slot]; !ok { + // nothing to do + continue + } + + for depositIndex, depositData := range data.slotBasedData.blocks[slot].Deposits { + index, indexExists := data.validatorBasedData.validatorIndices[string(depositData.Data.Pubkey)] + if !indexExists { + // skip + d.log.Infof("validator not found for deposit at index %d in slot %v", depositIndex, data.slotBasedData.blocks[slot].Slot) + continue + } + err := utils.VerifyDepositSignature(&phase0.DepositData{ + PublicKey: phase0.BLSPubKey(depositData.Data.Pubkey), + WithdrawalCredentials: depositData.Data.WithdrawalCredentials, + Amount: phase0.Gwei(depositData.Data.Amount), + Signature: phase0.BLSSignature(depositData.Data.Signature), + }, d.signingDomain) + if err != nil { + d.log.Error(fmt.Errorf("deposit at index %d in slot %v is invalid: %v (signature: %s)", depositIndex, data.slotBasedData.blocks[slot].Slot, err, depositData.Data.Signature), "", 0) + // this fails if there was no valid deposits in our entire epoch range + // so we can skip this deposit + if !indexExists { + d.log.Infof("validator did not have a valid deposit within epoch range - assuming discard of deposit") + continue + } + // validator has been created. check if it already exists in the current epoch. if yes, all deposits can be assumed as valid + // check if it existed in the previous state + var existedPreviousEpoch bool + if uint64(len(data.epochBasedData.validatorStates[int64(epoch)].Data)) > index { + existedPreviousEpoch = true + } + + if (*tar)[tI].DepositsCount[uint64(tO)+index] == 0 && !existedPreviousEpoch { + d.log.Infof("validator did not have a valid deposit before current deposit within epoch and did not exist in previous epoch - assuming discard of deposit") + continue + } + } + (*tar)[tI].DepositsAmount[uint64(tO)+index] += int64(depositData.Data.Amount) + + (*tar)[tI].DepositsCount[uint64(tO)+index]++ + d.log.Infof("processed deposit at index %d in slot %v", depositIndex, data.slotBasedData.blocks[slot].Slot) + } + } + return nil + }) + } + + return g.Wait() +} + +func (d *dashboardData) processAttestationRewards(data *MultiEpochData, tar *[]types.VDBDataEpochColumns) error { + if data.epochBasedData.epochs[len(data.epochBasedData.epochs)-1] < utils.Config.Chain.ClConfig.AltairForkEpoch { + d.phase0HotfixMutex.Lock() + defer d.phase0HotfixMutex.Unlock() + } + g := &errgroup.Group{} + for i, e := range data.epochBasedData.epochs { + epoch := e + tI := data.epochBasedData.tarIndices[i] + tO := data.epochBasedData.tarOffsets[i] + if data.epochBasedData.rewards.attestationRewards[epoch] == nil { + return fmt.Errorf("no rewards for epoch %d", epoch) + } + g.Go(func() error { + now := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_process_attestation_rewards_single").Observe(time.Since(now).Seconds()) + }() + // calculate max per committee x attestation slot + // array of slotsperepoch length, then array of committee per slot length. no maps because maps are slow + hyperlocalizedMax := make([][]int64, int(utils.Config.Chain.ClConfig.SlotsPerEpoch)) + if utils.Config.Chain.ClConfig.TargetCommitteeSize == 0 { + utils.Config.Chain.ClConfig.TargetCommitteeSize = 64 + } + // pre-init committees + for slot := 0; slot < int(utils.Config.Chain.ClConfig.SlotsPerEpoch); slot++ { + hyperlocalizedMax[slot] = make([]int64, utils.Config.Chain.ClConfig.TargetCommitteeSize) + } + // validator_index => [slot, committee] mapping + validatorSlotMap := make([]*struct { + Slot int64 + Committee int64 + }, len(data.epochBasedData.validatorStates[int64(epoch)].Data)) + startSlot := epoch * utils.Config.Chain.ClConfig.SlotsPerEpoch + endSlot := startSlot + utils.Config.Chain.ClConfig.SlotsPerEpoch + for slot := startSlot; slot < endSlot; slot++ { + if _, ok := data.slotBasedData.assignments.attestationAssignments[slot]; !ok { + return fmt.Errorf("no attestation assignments for slot %d", slot) + } + for committee, validatorIndices := range data.slotBasedData.assignments.attestationAssignments[slot] { + for _, validatorIndex := range validatorIndices { + validatorSlotMap[validatorIndex] = &struct { + Slot int64 + Committee int64 + }{ + Slot: int64(slot - startSlot), + Committee: int64(committee), + } + } + } + } + + for _, ar := range data.epochBasedData.rewards.attestationRewards[epoch] { + // there is a bug in lighthouse which causes the rewards api to return rewards for validators + // that have been deposited but arent active yet for some reason + // we can safely ignore these + if ar.ValidatorIndex >= uint64(len(validatorSlotMap)) { + d.log.Tracef("skipping reward for validator %d in epoch %d", ar.ValidatorIndex, epoch) + continue + } + valiIndextO := uint64(tO) + ar.ValidatorIndex + // ideal rewards + idealReward, ok := data.epochBasedData.rewards.attestationIdealRewards[epoch][data.epochBasedData.validatorStates[int64(epoch)].Data[ar.ValidatorIndex].EffectiveBalance] + if !ok { + return fmt.Errorf("no ideal reward for validator %d in epoch %d", valiIndextO, epoch) + } + (*tar)[tI].AttestationsIdealHeadReward[valiIndextO] = int64(idealReward.Head) + (*tar)[tI].AttestationsIdealSourceReward[valiIndextO] = int64(idealReward.Source) + (*tar)[tI].AttestationsIdealTargetReward[valiIndextO] = int64(idealReward.Target) + (*tar)[tI].AttestationsIdealInclusionReward[valiIndextO] = int64(idealReward.InclusionDelay) + (*tar)[tI].AttestationsIdealInactivityReward[valiIndextO] = int64(idealReward.Inactivity) + + (*tar)[tI].AttestationsHeadReward[valiIndextO] = int64(ar.Head) + (*tar)[tI].AttestationsSourceReward[valiIndextO] = int64(ar.Source) + (*tar)[tI].AttestationsTargetReward[valiIndextO] = int64(ar.Target) + // phase0 hotfix - cap inclusion delay reward at ideal inclusion delay reward + if epoch < utils.Config.Chain.ClConfig.AltairForkEpoch && int64(ar.InclusionDelay) > int64(idealReward.InclusionDelay) { + ar.InclusionDelay = idealReward.InclusionDelay + } + (*tar)[tI].AttestationsInclusionReward[valiIndextO] = int64(ar.InclusionDelay) + (*tar)[tI].AttestationsInactivityReward[valiIndextO] = int64(ar.Inactivity) + // total + total := int64(0) + for _, r := range []int64{int64(ar.Head), int64(ar.Source), int64(ar.Target), int64(ar.InclusionDelay), int64(ar.Inactivity)} { + total += r + } + // generate hyper localized max + attData := validatorSlotMap[ar.ValidatorIndex] + if attData == nil { + // happens when the validator has been added to state but doesnt have a duty yet or has been slashed + // fine to ignore + continue + } + if hyperlocalizedMax[attData.Slot][attData.Committee] < total { + hyperlocalizedMax[attData.Slot][attData.Committee] = total + } + } + // localize to slot + localizedMax := make([]int64, int(utils.Config.Chain.ClConfig.SlotsPerEpoch)) + for slot := 0; slot < int(utils.Config.Chain.ClConfig.SlotsPerEpoch); slot++ { + r := int64(0) + for _, d := range hyperlocalizedMax[slot] { + if d > r { + r = d + } + } + localizedMax[slot] = r + } + // write hyper localized max by iterating over all validators and looking up the max + for _, ar := range data.epochBasedData.rewards.attestationRewards[epoch] { + if ar.ValidatorIndex >= uint64(len(validatorSlotMap)) { + continue + } + valiSlot := validatorSlotMap[ar.ValidatorIndex] + + // happens when the validator has been added to state but doesnt have a duty yet or has been slashed + // fine to ignore + if valiSlot == nil { + continue + } + valiIndextO := uint64(tO) + ar.ValidatorIndex + (*tar)[tI].AttestationsHyperLocalizedMaxReward[valiIndextO] = hyperlocalizedMax[valiSlot.Slot][valiSlot.Committee] + (*tar)[tI].AttestationsLocalizedMaxReward[valiIndextO] = localizedMax[valiSlot.Slot] + } + return nil + }) + } + return g.Wait() +} + +// withdrawals +func (d *dashboardData) processWithdrawals(data *MultiEpochData, tar *[]types.VDBDataEpochColumns) error { + g := &errgroup.Group{} + for i, e := range data.epochBasedData.epochs { + epoch := e + if epoch < utils.Config.Chain.ClConfig.CapellaForkEpoch { + // d.log.Infof("skipping withdrawals for epoch %d (before capella)", epoch) + // no withdrawals before cappella + continue + } + tI := data.epochBasedData.tarIndices[i] + tO := uint64(data.epochBasedData.tarOffsets[i]) + startSlot := epoch * utils.Config.Chain.ClConfig.SlotsPerEpoch + endSlot := startSlot + utils.Config.Chain.ClConfig.SlotsPerEpoch + g.Go(func() error { + now := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_process_withdrawals_single").Observe(time.Since(now).Seconds()) + }() + for j := startSlot; j < endSlot; j++ { + if _, ok := data.slotBasedData.blocks[j]; !ok { + // nothing to do + continue + } + for _, withdrawal := range data.slotBasedData.blocks[j].Withdrawals { + (*tar)[tI].WithdrawalsAmount[tO+withdrawal.ValidatorIndex] = int64(withdrawal.Amount) + (*tar)[tI].WithdrawalsCount[tO+withdrawal.ValidatorIndex]++ + } + } + return nil + }) + } + return g.Wait() +} + +// Function to filter an array of integers using a bit mask with reduced allocations +func filterArrayUsingBitMask(arr []uint64, bitmask []byte) []uint64 { + result := make([]uint64, 0, len(arr)) + + for i := 0; i < len(arr); i++ { + byteIndex := i / 8 + bitIndex := i % 8 + if byteIndex < len(bitmask) { + if (bitmask[byteIndex] & (1 << bitIndex)) != 0 { + result = append(result, arr[i]) + } + } + } + + return result +} + +func IntegerSquareRoot(n uint64) uint64 { + x := n + y := (x + 1) / 2 + for y < x { + x = y + y = (x + n/x) / 2 + } + return x +} + +// attestations +func (d *dashboardData) processAttestations(data *MultiEpochData, tar *[]types.VDBDataEpochColumns) error { + g := &errgroup.Group{} + blockRoots := make(map[uint64]hexutil.Bytes) + blockValidityMap := make(map[uint64]int64, len(data.slotBasedData.blocks)) + // loop through slots. if a slot is missing reuse the previous slot. skip if the first slot is missing just skip + var lastBlockHash *hexutil.Bytes + var previousValidHash hexutil.Bytes + var toBeFilled []uint64 + for _, e := range data.epochBasedData.epochs { + epoch := e + start := epoch * utils.Config.Chain.ClConfig.SlotsPerEpoch + end := start + utils.Config.Chain.ClConfig.SlotsPerEpoch + for j := start; j < end; j++ { + if _, ok := data.slotBasedData.blocks[j]; !ok { + blockValidityMap[j] = 0 + // check if we have a previous block + if lastBlockHash == nil { + toBeFilled = append(toBeFilled, j) + continue + } + blockRoots[j] = *lastBlockHash + continue + } + blockValidityMap[j] = 1 + if lastBlockHash == nil { + previousValidHash = data.slotBasedData.blocks[j].ParentRoot + } + a := data.slotBasedData.blocks[j].BlockRoot + lastBlockHash = &a + blockRoots[j] = *lastBlockHash + } + } + // do blockValidityMap for n+1 epoch data + lookaheadEpoch := data.epochBasedData.epochs[len(data.epochBasedData.epochs)-1] + 1 + start := lookaheadEpoch * utils.Config.Chain.ClConfig.SlotsPerEpoch + end := start + utils.Config.Chain.ClConfig.SlotsPerEpoch + for j := start; j < end; j++ { + if _, ok := data.slotBasedData.blocks[j]; !ok { + blockValidityMap[j] = 0 + continue + } + blockValidityMap[j] = 1 + } + if lastBlockHash == nil { + return fmt.Errorf("no valid slots found") + } + // fill toBeFilled + for _, slot := range toBeFilled { + blockRoots[slot] = previousValidHash + } + + for i, e := range data.epochBasedData.epochs { + epoch := e + //debugCounters := make(map[string]int) + tI := data.epochBasedData.tarIndices[i] + tO := uint64(data.epochBasedData.tarOffsets[i]) + startSlot := epoch * utils.Config.Chain.ClConfig.SlotsPerEpoch + endSlot := startSlot + utils.Config.Chain.ClConfig.SlotsPerEpoch + squareSlotsPerEpoch := IntegerSquareRoot(utils.Config.Chain.ClConfig.SlotsPerEpoch) + //debugCounters := make(map[string]int) + g.Go(func() error { + start := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_process_attestations_single").Observe(time.Since(start).Seconds()) + }() + for j := startSlot; j < startSlot+(utils.Config.Chain.ClConfig.SlotsPerEpoch*2); j++ { + if _, ok := data.slotBasedData.blocks[j]; !ok { + // nothing to do + continue + } + // d.log.Infof("processing attestations for epoch %d in slot %d", epoch, j) + // attestations + for _, att := range data.slotBasedData.blocks[j].Attestations { + // ignore if slot is not within our epoch of interest + if att.Data.Slot < startSlot || att.Data.Slot >= endSlot { + //d.log.Infof("ignoring attestation in slot %d because its for a different epoch %d while we are processing epoch %d", att.Data.Slot, att.Data.Slot/utils.Config.Chain.ClConfig.SlotsPerEpoch, epoch) + //debugCounters["skipped_different_epoch"]++ + continue + } else { + //debugCounters["processed_attestations"]++ + //d.log.Infof("processing attestation in slot %d during epoch %d", att.Data.Slot, epoch) + } + // precalculate integer squareroot of slots per epoch + v := filterArrayUsingBitMask(data.slotBasedData.assignments.attestationAssignments[att.Data.Slot][att.Data.Index], att.AggregationBits) + if len(v) == 0 { + //debugCounters["skipped_no_validators"]++ + continue + } + inclusion_delay := int64(j - att.Data.Slot) + if inclusion_delay < 1 { + return fmt.Errorf("inclusion delay is less than 1 in slot %d for attestation of slot %d", j, att.Data.Slot) + } + // https://eips.ethereum.org/EIPS/eip-7045 + if epoch < utils.Config.Chain.ClConfig.AltairForkEpoch && inclusion_delay > 32 { + //debugCounters["skipped_inclusion_delay"]++ + continue + } + optimalInclusionDelay := int64(0) + for k := att.Data.Slot + 1; k < j; k++ { + c, ok := blockValidityMap[k] + if !ok { + return fmt.Errorf("no block validity found for slot %d", k) + } + optimalInclusionDelay += c + } + is_matching_source := true // enforced by the spec + br, ok := blockRoots[att.Data.Target.Epoch*utils.Config.Chain.ClConfig.SlotsPerEpoch] + if !ok { + return fmt.Errorf("no block root found for epoch %d", att.Data.Target.Epoch) + } + is_matching_target := bytes.Equal(att.Data.Target.Root, br) + br, ok = blockRoots[att.Data.Slot] + if !ok { + return fmt.Errorf("no block root found for slot %d", att.Data.Slot) + } + is_matching_head := is_matching_target && bytes.Equal(att.Data.BeaconBlockRoot, br) + sourceValue := int64(0) + targetValue := int64(0) + headValue := int64(0) + switch utils.ForkVersionAtEpoch(epoch).CurrentVersion { + case utils.Config.Chain.ClConfig.GenesisForkVersion: + // genesis, head source and target only care about having accurate hashes + if is_matching_source { + sourceValue = 1 + } + if is_matching_target { + targetValue = 1 + } + if is_matching_head { + headValue = 1 + } + default: + if is_matching_source && inclusion_delay <= int64(squareSlotsPerEpoch) { + sourceValue = 1 + } + if is_matching_target { // 32 slot filter on target for pre altair forks is done above + targetValue = 1 + } + if is_matching_head && inclusion_delay == 1 { + headValue = 1 + } + } + + for _, valiIndex := range v { + valiIndex += tO + // check if it was already processed. if yes skip + if (*tar)[tI].AttestationsObserved[valiIndex] > 0 { + continue + } + // executed + (*tar)[tI].AttestationsObserved[valiIndex]++ + // inclusion delay + (*tar)[tI].InclusionDelaySum[valiIndex] += inclusion_delay - 1 + (*tar)[tI].OptimalInclusionDelaySum[valiIndex] += optimalInclusionDelay + // metrics + (*tar)[tI].AttestationsSourceExecuted[valiIndex] += sourceValue + if is_matching_source { + (*tar)[tI].AttestationsSourceMatched[valiIndex]++ + } + (*tar)[tI].AttestationsTargetExecuted[valiIndex] += targetValue + if is_matching_target { + (*tar)[tI].AttestationsTargetMatched[valiIndex]++ + } + (*tar)[tI].AttestationsHeadExecuted[valiIndex] += headValue + if is_matching_head { + (*tar)[tI].AttestationsHeadMatched[valiIndex]++ + } + } + } + } + return nil + }) + } + return g.Wait() +} + +// sync odds +func (d *dashboardData) processExpectedSyncPeriods(data *MultiEpochData, tar *[]types.VDBDataEpochColumns) error { + g := &errgroup.Group{} + for i, e := range data.epochBasedData.epochs { + epoch := e + if utils.FirstEpochOfSyncPeriod(utils.SyncPeriodOfEpoch(epoch)) != epoch { + // d.log.Infof("skipping epoch %d as it is not the first epoch of a sync period", epoch) + continue + } + tI := data.epochBasedData.tarIndices[i] + tO := uint64(data.epochBasedData.tarOffsets[i]) + g.Go(func() error { + now := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_process_expected_sync_periods_single").Observe(time.Since(now).Seconds()) + }() + iEpoch := int64(epoch) + + totalEffective := int64(0) + for _, valData := range data.epochBasedData.validatorStates[iEpoch].Data { + if valData.Status.IsActive() { + totalEffective += int64(valData.EffectiveBalance / 1e9) + } + } + fTotalEffective := float64(totalEffective) + fCommitteeSize := float64(utils.Config.Chain.ClConfig.SyncCommitteeSize) + + for _, valData := range data.epochBasedData.validatorStates[iEpoch].Data { + if valData.Status.IsActive() { + // See https://github.com/ethereum/annotated-spec/blob/master/altair/beacon-chain.md#get_sync_committee_indices + // Note that this formula is not 100% the chance as defined in the spec, but after running simulations we found + // it being precise enough for our purposes with an error margin of less than 0.003% + syncChance := float64(valData.EffectiveBalance/1e9) / fTotalEffective + (*tar)[tI].SyncCommitteesExpected[tO+valData.Index] = syncChance * fCommitteeSize + } + } + + return nil + }) + } + return g.Wait() +} + +// blocks expected +func (d *dashboardData) processBlocksExpected(data *MultiEpochData, tar *[]types.VDBDataEpochColumns) error { + g := &errgroup.Group{} + for i, e := range data.epochBasedData.epochs { + epoch := e + tI := data.epochBasedData.tarIndices[i] + tO := uint64(data.epochBasedData.tarOffsets[i]) + g.Go(func() error { + defe := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_process_blocks_expected_single").Observe(time.Since(defe).Seconds()) + }() + iEpoch := int64(epoch) + totalEffective := int64(0) + for _, valData := range data.epochBasedData.validatorStates[iEpoch].Data { + if valData.Status.IsActive() { + totalEffective += int64(valData.EffectiveBalance / 1e9) + } + } + fTotalEffective := float64(totalEffective) + fSlotsPerEpoch := float64(utils.Config.Chain.ClConfig.SlotsPerEpoch) + if epoch == 0 { + // cant propose slot 0 + fSlotsPerEpoch-- + } + for _, valData := range data.epochBasedData.validatorStates[iEpoch].Data { + if valData.Status.IsActive() { + // See https://github.com/ethereum/annotated-spec/blob/master/phase0/beacon-chain.md#compute_proposer_index + proposalChance := float64(valData.EffectiveBalance/1e9) / fTotalEffective + (*tar)[tI].BlocksExpected[tO+valData.Index] = proposalChance * fSlotsPerEpoch + } + } + return nil + }) + } + return g.Wait() +} + +func (d *dashboardData) processSyncVotes(data *MultiEpochData, tar *[]types.VDBDataEpochColumns) error { + g := &errgroup.Group{} + for i, e := range data.epochBasedData.epochs { + epoch := e + if epoch < utils.Config.Chain.ClConfig.AltairForkEpoch { + //d.log.Infof("skipping sync votes for epoch %d (before altair)", epoch) + // no sync votes before altair + continue + } + syncPeriod := utils.SyncPeriodOfEpoch(epoch) + iSyncPeriod := int64(syncPeriod) + tI := data.epochBasedData.tarIndices[i] + tO := uint64(data.epochBasedData.tarOffsets[i]) + startSlot := epoch * utils.Config.Chain.ClConfig.SlotsPerEpoch + endSlot := startSlot + utils.Config.Chain.ClConfig.SlotsPerEpoch + g.Go(func() error { + now := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_process_sync_votes_single").Observe(time.Since(now).Seconds()) + }() + // safety check, check if we have the sync period data + if _, ok := data.syncPeriodBasedData.SyncAssignments[syncPeriod]; !ok { + return fmt.Errorf("no sync assignments for sync period %d", syncPeriod) + } + for i, valiIndex := range data.syncPeriodBasedData.SyncAssignments[syncPeriod] { + (*tar)[tI].SyncCommitteeAssignmentsPeriod[tO+valiIndex] = append((*tar)[tI].SyncCommitteeAssignmentsPeriod[tO+valiIndex], iSyncPeriod) + (*tar)[tI].SyncCommitteeAssignmentsIndex[tO+valiIndex] = append((*tar)[tI].SyncCommitteeAssignmentsIndex[tO+valiIndex], int64(i)) + } + for j := startSlot; j < endSlot; j++ { + for _, valiIndex := range data.syncPeriodBasedData.SyncAssignments[syncPeriod] { + (*tar)[tI].SyncStatusSlot[tO+valiIndex] = append((*tar)[tI].SyncStatusSlot[tO+valiIndex], int64(j)) + (*tar)[tI].SyncStatusExecuted[tO+valiIndex] = append((*tar)[tI].SyncStatusExecuted[tO+valiIndex], false) + } + if _, ok := data.slotBasedData.blocks[j]; !ok { + // nothing to do + continue + } + // you cant vote for slot 0 + if j == 0 { + continue + } + // sync votes + v := filterArrayUsingBitMask( + data.syncPeriodBasedData.SyncAssignments[syncPeriod], + data.slotBasedData.blocks[j].SyncAggregate.SyncCommitteeBits) + for _, valiIndex := range v { + (*tar)[tI].SyncStatusExecuted[tO+valiIndex][len((*tar)[tI].SyncStatusExecuted[tO+valiIndex])-1] = true + } + for _, valiIndex := range data.syncPeriodBasedData.SyncAssignments[syncPeriod] { + (*tar)[tI].SyncScheduled[tO+valiIndex]++ + } + } + return nil + }) + } + return g.Wait() +} + +func (d *dashboardData) processProposalRewards(data *MultiEpochData, tar *[]types.VDBDataEpochColumns) error { + if data.epochBasedData.epochs[len(data.epochBasedData.epochs)-1] < utils.Config.Chain.ClConfig.AltairForkEpoch { + defer d.phase0HotfixMutex.Unlock() + } + g := &errgroup.Group{} + buffer := utils.Config.Chain.ClConfig.SlotsPerEpoch / 2 + for i, e := range data.epochBasedData.epochs { + epoch := e + tI := data.epochBasedData.tarIndices[i] + tO := uint64(data.epochBasedData.tarOffsets[i]) + startSlot := epoch * utils.Config.Chain.ClConfig.SlotsPerEpoch + endSlot := startSlot + utils.Config.Chain.ClConfig.SlotsPerEpoch + g.Go(func() error { + now := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_process_proposal_rewards_single").Observe(time.Since(now).Seconds()) + }() + for j := startSlot; j < endSlot; j++ { + // calculate median reward + medianStartSlot := uint64(0) + if j >= buffer { + medianStartSlot = j - buffer + } + medianEndSlot := j + buffer + // actually, lets do median instead + medianArray := make([]int64, 0) + for k := medianStartSlot; k < medianEndSlot; k++ { + if r, ok := data.slotBasedData.rewards.blockRewards[k]; ok { + rewards := r.Data + reward := rewards.Attestations + rewards.AttesterSlashings + rewards.ProposerSlashings + rewards.SyncAggregate + medianArray = append(medianArray, reward) + } + } + if len(medianArray) == 0 { + // no rewards in buffer + // lets fall back to zero missed rewards. this gets triggered on gnosis during slot 11737794 (and more) + medianArray = append(medianArray, 0) + } + // calculate median + slices.Sort(medianArray) + median := int64(0) + if len(medianArray)%2 == 0 { + // even + median = (medianArray[len(medianArray)/2-1] + medianArray[len(medianArray)/2]) / 2 + } else { + // odd + median = medianArray[len(medianArray)/2] + } + + if _, ok := data.slotBasedData.rewards.blockRewards[j]; !ok { + // assign to validator that missed the slot + proposer := data.slotBasedData.assignments.blockAssignments[j] + (*tar)[tI].BlocksClMissedMedianReward[tO+proposer] += median + continue + } + rewards := data.slotBasedData.rewards.blockRewards[j].Data + (*tar)[tI].BlockRewardsSlot[tO+rewards.ProposerIndex] = append((*tar)[tI].BlockRewardsSlot[tO+rewards.ProposerIndex], int64(j)) + (*tar)[tI].BlockRewardsAttestationsReward[tO+rewards.ProposerIndex] = append((*tar)[tI].BlockRewardsAttestationsReward[tO+rewards.ProposerIndex], rewards.Attestations) + (*tar)[tI].BlockRewardsSyncAggregateReward[tO+rewards.ProposerIndex] = append((*tar)[tI].BlockRewardsSyncAggregateReward[tO+rewards.ProposerIndex], rewards.SyncAggregate) + (*tar)[tI].BlockRewardsSlasherReward[tO+rewards.ProposerIndex] = append((*tar)[tI].BlockRewardsSlasherReward[tO+rewards.ProposerIndex], rewards.AttesterSlashings+rewards.ProposerSlashings) + reward := rewards.Attestations + rewards.AttesterSlashings + rewards.ProposerSlashings + rewards.SyncAggregate + if reward < median { + (*tar)[tI].BlocksClMissedMedianReward[tO+rewards.ProposerIndex] += median - reward + } + if epoch < utils.Config.Chain.ClConfig.AltairForkEpoch { + // hotfix for phase0 blocks + if (*data).epochBasedData.rewards.attestationRewards[epoch][rewards.ProposerIndex].InclusionDelay > int32(reward) { + (*data).epochBasedData.rewards.attestationRewards[epoch][rewards.ProposerIndex].InclusionDelay -= int32(reward) + } + } + } + return nil + }) + } + return g.Wait() +} + +// sync rewards +func (d *dashboardData) processSyncCommitteeRewards(data *MultiEpochData, tar *[]types.VDBDataEpochColumns) error { + g := &errgroup.Group{} + for i, e := range data.epochBasedData.epochs { + epoch := e + if epoch < utils.Config.Chain.ClConfig.AltairForkEpoch { + //d.log.Infof("skipping sync rewards for epoch %d (before altair)", epoch) + // no sync rewards before altair + continue + } + tI := data.epochBasedData.tarIndices[i] + tO := uint64(data.epochBasedData.tarOffsets[i]) + startSlot := epoch * utils.Config.Chain.ClConfig.SlotsPerEpoch + endSlot := startSlot + utils.Config.Chain.ClConfig.SlotsPerEpoch + g.Go(func() error { + now := time.Now() + defer func() { + metrics.TaskDuration.WithLabelValues("dashboard_data_exporter_process_sync_rewards_single").Observe(time.Since(now).Seconds()) + }() + maxRewards := int64(0) + for j := startSlot; j < endSlot; j++ { + if _, ok := data.slotBasedData.blocks[j]; !ok { + // no data is expected for missed blocks + continue + } + for _, reward := range data.slotBasedData.rewards.syncCommitteeRewards[j].Data { + (*tar)[tI].SyncRewardsSlot[tO+reward.ValidatorIndex] = append((*tar)[tI].SyncRewardsSlot[tO+reward.ValidatorIndex], int64(j)) + (*tar)[tI].SyncRewardsReward[tO+reward.ValidatorIndex] = append((*tar)[tI].SyncRewardsReward[tO+reward.ValidatorIndex], reward.Reward) + if reward.Reward > maxRewards { + maxRewards = reward.Reward + } + } + } + /* + // removed safety check as it was triggered in a real scenario on gnosis + if maxRewards <= 0 { + // lets just bork ourselves to be safe + return fmt.Errorf("no rewards in epoch %d", epoch) + } + */ + for j := startSlot; j < endSlot; j++ { + for _, reward := range data.slotBasedData.rewards.syncCommitteeRewards[j].Data { + (*tar)[tI].SyncLocalizedMaxReward[tO+reward.ValidatorIndex] += maxRewards + } + } + return nil + }) + } + return g.Wait() +} diff --git a/backend/pkg/exporter/modules/dashboard_data_rollings.go b/backend/pkg/exporter/modules/dashboard_data_rollings.go new file mode 100644 index 000000000..94927b367 --- /dev/null +++ b/backend/pkg/exporter/modules/dashboard_data_rollings.go @@ -0,0 +1,189 @@ +package modules + +import ( + "fmt" + "time" + + "github.com/gobitfly/beaconchain/pkg/commons/metrics" + "github.com/gobitfly/beaconchain/pkg/commons/utils" + edb "github.com/gobitfly/beaconchain/pkg/exporter/db" + "github.com/pkg/errors" + "golang.org/x/sync/errgroup" +) + +// we wrap all our errors in this codebase + +func (d *dashboardData) rollingTask() { + for { + // loop to complete incomplete epochs + err := d.handleRollings() + if err != nil { + d.log.Error(err, "failed to handle rollings", 0) + time.Sleep(10 * time.Second) + continue + } + time.Sleep(10 * time.Second) + } +} + +func (d *dashboardData) handleRollings() error { + // fork for every rolling we have to do + rollings := []edb.Rollings{ + edb.Rolling1h, + edb.Rolling24h, + edb.Rolling7d, + edb.Rolling30d, + edb.Rolling90d, + edb.RollingTotal, + } + // but lets limit to x rollings + eg := errgroup.Group{} + eg.SetLimit(int(utils.Config.DashboardExporter.RollingsInParallel)) + for _, rolling := range rollings { + rolling := rolling + eg.Go(func() error { + return d.doRollingCheck(rolling) + }) + } + err := eg.Wait() + if err != nil { + return errors.Wrap(err, "failed to do rollings") + } + return nil +} + +func (d *dashboardData) doRollingCheck(rolling edb.Rollings) error { + defer func() { + // GetOldestUnfinishedTransferEpoch + oldestUnfinishedTransferEpoch, err := edb.GetOldestUnfinishedTransferEpoch() + if err != nil { + d.log.Error(err, "failed to get oldest unfinished transfer epoch", 0) + return + } + metrics.State.WithLabelValues("dashboard_data_exporter_oldest_unfinished_transfer_epoch").Set(float64(oldestUnfinishedTransferEpoch)) + }() + + start := time.Now() + finishedEpoch, err := edb.GetLatestFinishedEpoch() + if err != nil { + return errors.Wrap(err, "failed to get latest finished epoch") + } + if finishedEpoch < 0 { + d.log.Infof("no finished epoch yet") + return nil + } + metrics.State.WithLabelValues("dashboard_data_exporter_finished_epoch").Set(float64(finishedEpoch)) + // if finishedEpoch is not the same as the safeepoch we skip updating the rolling so resyncing after falling back is fast + if safeEpoch := d.latestSafeEpoch.Load(); finishedEpoch != safeEpoch { + d.log.Infof("skipping rolling %s update, finished epoch %d, safe epoch %d", rolling, finishedEpoch, safeEpoch) + return nil + } + rollingEpoch, err := edb.GetRollingLastEpoch(rolling) + if err != nil { + return errors.Wrap(err, "failed to get rolling last epoch") + } + metrics.State.WithLabelValues(fmt.Sprintf("dashboard_data_exporter_rolling_%s_epoch", rolling)).Set(float64(rollingEpoch)) + if rollingEpoch >= finishedEpoch { + d.log.Debugf("rolling %s is up to date", rolling) + return nil + } + defer func() { + metrics.TaskDuration.WithLabelValues(fmt.Sprintf("dashboard_data_exporter_rolling_%s_overall", rolling)).Observe(time.Since(start).Seconds()) + }() + d.log.Infof("rolling %s is outdated, latest epoch %d, latest finished epoch %d", rolling, rollingEpoch, finishedEpoch) + // update metric after run + defer func() { + rollingEpoch, err := edb.GetRollingLastEpoch(rolling) + if err != nil { + d.log.Error(err, "failed to get rolling last epoch", 0) + return + } + metrics.State.WithLabelValues(fmt.Sprintf("dashboard_data_exporter_rolling_%s_epoch", rolling)).Set(float64(rollingEpoch)) + }() + now := time.Now() + // next, nuke the unsafe rolling tables to prepare them for us to fill them + err = edb.NukeUnsafeRollingTable(rolling) + if err != nil { + return errors.Wrap(err, "failed to nuke unsafe rolling table") + } + // also create a defer that nukes it - exchange or not, we want it clean so clickhouse doesnt waste compute on it + defer func() { + err := edb.NukeUnsafeRollingTable(rolling) + if err != nil { + d.log.Error(err, "failed to nuke unsafe rolling table", 0) + } + }() + metrics.TaskDuration.WithLabelValues(fmt.Sprintf("dashboard_data_exporter_rolling_%s_nuke_unsafe", rolling)).Observe(time.Since(now).Seconds()) + // now we fetch the start & end for each pre-aggregated table we use + minTs := utils.EpochToTime(uint64(finishedEpoch)).Add(-rolling.GetDuration()) + d.log.Infof("rolling %s, min ts %s", rolling, minTs) + tables := []edb.RollingSources{ + edb.RollingSourceMonthly, + edb.RollingSourceDaily, + edb.RollingSourceHourly, + edb.RollingSourceEpochly, + } + minMaxMap := make(map[edb.RollingSources]*edb.MinMax) + var lowestSeenTs *time.Time + for _, table := range tables { + now := time.Now() + minmax, err := edb.GetMinMaxForRollingSource(table, minTs, lowestSeenTs) + if err != nil { + return errors.Wrap(err, "failed to get min max for rolling source") + } + if minmax == nil { + //d.log.Debug("rolling %s, source %s, no data", rolling, table) + continue + } + minMaxMap[table] = minmax + d.log.Infof("rolling %s, source %s, min %s, max %s", rolling, table, minmax.Min, minmax.Max) + lowestSeenTs = minmax.Min + metrics.TaskDuration.WithLabelValues(fmt.Sprintf("dashboard_data_exporter_rolling_%s_minmax", table)).Observe(time.Since(now).Seconds()) + metrics.TaskDuration.WithLabelValues(fmt.Sprintf("dashboard_data_exporter_rolling_%s_minmax", rolling)).Observe(time.Since(now).Seconds()) + } + // now the transfer logic for each source + eg := errgroup.Group{} + eg.SetLimit(int(utils.Config.DashboardExporter.RollingPartsInParallel)) + now = time.Now() + for source, minmax := range minMaxMap { + if minmax == nil { + continue + } + source := source + minmax := minmax + eg.Go(func() error { + d.log.Infof("transferring rolling source %s to rolling %s", source, rolling) + now := time.Now() + err := edb.TransferRollingSourceToRolling(rolling, source, *minmax) + if err != nil { + return errors.Wrap(err, "failed to transfer rolling source to rolling") + } + // d.log.Infof("transferred rolling source %s to rolling %s in %s", source, rolling, time.Since(now)) + // one metric for the source table and oe for the rolling table + metrics.TaskDuration.WithLabelValues(fmt.Sprintf("dashboard_data_exporter_rolling_%s_transfer", source)).Observe(time.Since(now).Seconds()) + metrics.TaskDuration.WithLabelValues(fmt.Sprintf("dashboard_data_exporter_rolling_%s_transfer", rolling)).Observe(time.Since(now).Seconds()) + return nil + }) + } + err = eg.Wait() + metrics.TaskDuration.WithLabelValues(fmt.Sprintf("dashboard_data_exporter_rolling_%s_transfer_all", rolling)).Observe(time.Since(now).Seconds()) + if err != nil { + return errors.Wrap(err, "failed to transfer all rolling sources") + } + // trigger an optimize final on the table. this should be fine since the size of the table is relatively small and calls are constrained + now = time.Now() + d.log.Infof("optimizing rolling %s", rolling) + err = edb.OptimizeUnsafeRollingTable(rolling) + if err != nil { + return errors.Wrap(err, "failed to optimize rolling table") + } + metrics.TaskDuration.WithLabelValues(fmt.Sprintf("dashboard_data_exporter_rolling_%s_optimize", rolling)).Observe(time.Since(now).Seconds()) + // now we swap the tables + now = time.Now() + err = edb.SwapRollingTables(rolling) + if err != nil { + return errors.Wrap(err, "failed to swap rolling tables") + } + metrics.TaskDuration.WithLabelValues(fmt.Sprintf("dashboard_data_exporter_rolling_%s_swap", rolling)).Observe(time.Since(now).Seconds()) + return nil +} diff --git a/backend/pkg/exporter/modules/dashboard_data_w_1_epoch.go b/backend/pkg/exporter/modules/dashboard_data_w_1_epoch.go deleted file mode 100644 index d363465f2..000000000 --- a/backend/pkg/exporter/modules/dashboard_data_w_1_epoch.go +++ /dev/null @@ -1,276 +0,0 @@ -package modules - -import ( - "context" - "fmt" - "sync" - - "github.com/gobitfly/beaconchain/pkg/commons/db" - "github.com/gobitfly/beaconchain/pkg/commons/utils" - edb "github.com/gobitfly/beaconchain/pkg/exporter/db" - pgxdecimal "github.com/jackc/pgx-shopspring-decimal" - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/stdlib" - "github.com/pkg/errors" -) - -type epochWriter struct { - *dashboardData - mutex *sync.Mutex -} - -func newEpochWriter(d *dashboardData) *epochWriter { - return &epochWriter{ - dashboardData: d, - mutex: &sync.Mutex{}, - } -} - -// How wide each table partition is in epochs -const PartitionEpochWidth = 3 - -// How long epochs will remain in the database is defined in getRetentionEpochDuration. -// For ETH mainnet this will be 9 epochs, as 9 epochs is exactly the range we need in the hour table (roughly one hour). -// This buffer can be used to increase or decrease from that 9 epoch target. A value of 1 will keep exactly those 9 needed epochs in the database. -const retentionBuffer = 1.1 // do not go below 1 - -func (d *epochWriter) getRetentionEpochDuration() uint64 { - return uint64(float64(utils.EpochsPerDay()) / 24 * retentionBuffer) -} - -func (d *epochWriter) getPartitionRange(epoch uint64) (uint64, uint64) { - startOfPartition := epoch / PartitionEpochWidth * PartitionEpochWidth // inclusive - endOfPartition := startOfPartition + PartitionEpochWidth // exclusive - return startOfPartition, endOfPartition -} - -func (d *epochWriter) clearOldEpochs(removeBelowEpoch int64) error { - if debugSkipOldEpochClear { - d.log.Infof("DEBUG MODE, prevented epoch clear below %d", removeBelowEpoch) - return nil - } - - partitions, err := edb.GetPartitionNamesOfTable(edb.EpochWriterTableName) - if err != nil { - return errors.Wrap(err, "failed to get partitions") - } - - for _, partition := range partitions { - epochFrom, epochTo, err := parseEpochRange(fmt.Sprintf(`%s_(\d+)_(\d+)`, edb.EpochWriterTableName), partition) - if err != nil { - return errors.Wrap(err, "failed to parse epoch range") - } - - if int64(epochTo) < removeBelowEpoch { - d.mutex.Lock() - err := d.deleteEpochPartition(epochFrom, epochTo) - d.log.Infof("Deleted old epoch partition %d-%d", epochFrom, epochTo) - d.mutex.Unlock() - if err != nil { - return errors.Wrap(err, "failed to delete epoch partition") - } - } - } - - return nil -} - -func (d *epochWriter) WriteEpochData(epoch uint64, data []*validatorDashboardDataRow) error { - // Create table if needed - startOfPartition, endOfPartition := d.getPartitionRange(epoch) - - d.mutex.Lock() - err := d.createEpochPartition(startOfPartition, endOfPartition) - if epoch == startOfPartition && debugAddToColumnEngine { - err = edb.AddToColumnEngineAllColumns(fmt.Sprintf("%s_%d_%d", edb.EpochWriterTableName, startOfPartition, endOfPartition)) - if err != nil { - d.log.Warnf("Failed to add epoch to column engine: %v", err) - } - } - d.mutex.Unlock() - if err != nil { - return errors.Wrap(err, "failed to create epoch partition") - } - - conn, err := db.AlloyWriter.Conn(context.Background()) - if err != nil { - return fmt.Errorf("error retrieving raw sql connection: %w", err) - } - defer conn.Close() - - err = conn.Raw(func(driverConn interface{}) error { - conn := driverConn.(*stdlib.Conn).Conn() - - pgxdecimal.Register(conn.TypeMap()) - tx, err := conn.Begin(context.Background()) - - if err != nil { - return errors.Wrap(err, "error starting transaction") - } - - defer func() { - err := tx.Rollback(context.Background()) - if err != nil && !errors.Is(err, pgx.ErrTxClosed) { - d.log.Error(err, "error rolling back transaction", 0) - } - }() - - _, err = tx.CopyFrom(context.Background(), pgx.Identifier{edb.EpochWriterTableName}, []string{ - "validator_index", - "epoch", - "attestations_source_reward", - "attestations_target_reward", - "attestations_head_reward", - "attestations_inactivity_reward", - "attestations_inclusion_reward", - "attestations_reward", - "attestations_ideal_source_reward", - "attestations_ideal_target_reward", - "attestations_ideal_head_reward", - "attestations_ideal_inactivity_reward", - "attestations_ideal_inclusion_reward", - "attestations_ideal_reward", - "blocks_scheduled", - "blocks_proposed", - "blocks_cl_reward", - "sync_scheduled", - "sync_executed", - "sync_reward", - "slashed", - "balance_start", - "balance_end", - "deposits_count", - "deposits_amount", - "withdrawals_count", - "withdrawals_amount", - "inclusion_delay_sum", - "blocks_expected", - "attestations_scheduled", - "attestations_observed", - "attestations_head_executed", - "attestations_source_executed", - "attestations_target_executed", - "optimal_inclusion_delay_sum", - "slashed_by", - "slashed_violation", - "slasher_reward", - "last_executed_duty_epoch", - "blocks_cl_attestations_reward", - "blocks_cl_sync_aggregate_reward", - "sync_committees_expected", - }, pgx.CopyFromSlice(len(data), func(i int) ([]interface{}, error) { - return []interface{}{ - i, - epoch, - data[i].AttestationsSourceReward, - data[i].AttestationsTargetReward, - data[i].AttestationsHeadReward, - data[i].AttestationsInactivityPenalty, - data[i].AttestationsInclusionsReward, - data[i].AttestationReward, - data[i].AttestationsIdealSourceReward, - data[i].AttestationsIdealTargetReward, - data[i].AttestationsIdealHeadReward, - data[i].AttestationsIdealInactivityPenalty, - data[i].AttestationsIdealInclusionsReward, - data[i].AttestationIdealReward, - data[i].BlockScheduled, - data[i].BlocksProposed, - data[i].BlocksClReward, - data[i].SyncScheduled, - data[i].SyncExecuted, - data[i].SyncReward, - data[i].Slashed, - data[i].BalanceStart, - data[i].BalanceEnd, - data[i].DepositsCount, - data[i].DepositsAmount, - data[i].WithdrawalsCount, - data[i].WithdrawalsAmount, - data[i].InclusionDelaySum, - data[i].BlocksExpectedThisEpoch, - data[i].AttestationsScheduled, - data[i].AttestationsObserved, - data[i].AttestationsHeadExecuted, - data[i].AttestationsSourceExecuted, - data[i].AttestationsTargetExecuted, - data[i].OptimalInclusionDelay, - data[i].SlashedBy, - data[i].SlashedViolation, - data[i].SlasherRewards, - data[i].LastSubmittedDutyEpoch, - data[i].BlocksClAttestestationsReward, - data[i].BlocksClSyncAggregateReward, - data[i].SyncCommitteesExpectedThisPeriod, - }, nil - })) - - if err != nil { - return errors.Wrap(err, "error copying data") - } - - err = tx.Commit(context.Background()) - if err != nil { - if !utils.IsDuplicatedKeyError(err) { - return errors.Wrap(err, "error committing transaction") - } - } - return nil - }) - - if err != nil { - return errors.Wrap(err, "error writing data") - } - - return nil -} - -func (d *epochWriter) createEpochPartition(epochFrom, epochTo uint64) error { - partitionName := fmt.Sprintf("%s_%d_%d", edb.EpochWriterTableName, epochFrom, epochTo) - _, err := db.AlloyWriter.Exec(fmt.Sprintf(` - CREATE TABLE IF NOT EXISTS %s( - LIKE %s INCLUDING DEFAULTS INCLUDING CONSTRAINTS - ) - `, - partitionName, edb.EpochWriterTableName, - )) - if err != nil { - return errors.Wrap(err, "failed to create epoch partition (1)") - } - - isAttached, err := isPartitionAttached(edb.EpochWriterTableName, partitionName) - if err != nil { - return errors.Wrap(err, "failed to check if partition is attached") - } - - if !isAttached { - _, err = db.AlloyWriter.Exec(fmt.Sprintf(` - ALTER TABLE %s ATTACH PARTITION %s - FOR VALUES FROM (%d) TO (%d) - `, - edb.EpochWriterTableName, partitionName, epochFrom, epochTo, - )) - } - - return err -} - -func (d *epochWriter) deleteEpochPartition(epochFrom, epochTo uint64) error { - _, err := db.AlloyWriter.Exec(fmt.Sprintf(` - ALTER TABLE %[1]s DETACH PARTITION %[1]s_%[2]d_%[3]d; - `, - edb.EpochWriterTableName, epochFrom, epochTo, - )) - - if err != nil { - return err - } - - _, err = db.AlloyWriter.Exec(fmt.Sprintf(` - DROP TABLE IF EXISTS %s_%d_%d - `, - edb.EpochWriterTableName, epochFrom, epochTo, - )) - - return err -} diff --git a/backend/pkg/exporter/modules/dashboard_data_w_2_epoch_hour.go b/backend/pkg/exporter/modules/dashboard_data_w_2_epoch_hour.go deleted file mode 100644 index 75898ae46..000000000 --- a/backend/pkg/exporter/modules/dashboard_data_w_2_epoch_hour.go +++ /dev/null @@ -1,253 +0,0 @@ -package modules - -import ( - "database/sql" - "fmt" - "sync" - "time" - - "github.com/gobitfly/beaconchain/pkg/commons/db" - "github.com/gobitfly/beaconchain/pkg/commons/metrics" - "github.com/gobitfly/beaconchain/pkg/commons/utils" - edb "github.com/gobitfly/beaconchain/pkg/exporter/db" - "github.com/pkg/errors" -) - -type epochToHourAggregator struct { - *dashboardData - mutex *sync.Mutex -} - -// How long aggregated hours will remain in the database is defined in getHourRetentionDurationEpochs. -// For ETH mainnet this will be 225 epochs, as 225 epochs is exactly the range we need in the day table (equals 1 day). -// This buffer can be used to increase or decrease from that 225 epoch target. A value of 1 will keep exactly those 25 (225 / 9) needed hour aggregations in the database. -const hourRetentionBuffer = 1.1 // do not go below 1 - -func getHourAggregateWidth() uint64 { - return utils.EpochsPerDay() / 24 -} - -func newEpochToHourAggregator(d *dashboardData) *epochToHourAggregator { - return &epochToHourAggregator{ - dashboardData: d, - mutex: &sync.Mutex{}, - } -} - -func (d *epochToHourAggregator) clearOldHourAggregations(removeBelowEpoch int64) error { - partitions, err := edb.GetPartitionNamesOfTable(edb.HourWriterTableName) - if err != nil { - return errors.Wrap(err, "failed to get partitions") - } - - for _, partition := range partitions { - epochFrom, epochTo, err := parseEpochRange(fmt.Sprintf(`%s_(\d+)_(\d+)`, edb.HourWriterTableName), partition) - if err != nil { - return errors.Wrap(err, "failed to parse epoch range") - } - - if int64(epochTo) < removeBelowEpoch { - d.mutex.Lock() - err := d.deleteHourlyPartition(epochFrom, epochTo) - d.log.Infof("Deleted old hourly partition %d-%d", epochFrom, epochTo) - d.mutex.Unlock() - if err != nil { - return errors.Wrap(err, "failed to delete hourly partition") - } - } - } - - return nil -} - -// Assumes no gaps in epochs -func (d *epochToHourAggregator) aggregate1h(currentExportedEpoch uint64) error { - d.mutex.Lock() - defer d.mutex.Unlock() - - startTime := time.Now() - d.log.Info("aggregating 1h") - defer func() { - d.log.Infof("aggregate 1h took %v", time.Since(startTime)) - metrics.TaskDuration.WithLabelValues("exporter_v2dash_agg_1h").Observe(time.Since(startTime).Seconds()) - }() - - lastHourExported, err := edb.GetLastExportedHour() - if err != nil && err != sql.ErrNoRows { - return errors.Wrap(err, "failed to get latest dashboard hourly epoch") - } - - differenceToCurrentEpoch := currentExportedEpoch + 1 - lastHourExported.EpochEnd // epochEnd is excl hence the +1 - - if differenceToCurrentEpoch > d.getHourRetentionDurationEpochs() { - d.log.Warnf("difference to current epoch is larger than retention duration, skipping for now: %v", differenceToCurrentEpoch) - return nil - } - - gaps, err := edb.GetMissingEpochsBetween(int64(lastHourExported.EpochEnd), int64(currentExportedEpoch+1)) - if err != nil { - return errors.Wrap(err, "failed to get dashboard epoch gaps") - } - - if len(gaps) > 0 { - return fmt.Errorf("gaps in dashboard epoch, skipping for now: %v", gaps) // sanity, this should never happen - } - - _, currentEndBound := getHourAggregateBounds(currentExportedEpoch) - - for epoch := lastHourExported.EpochStart; epoch <= currentEndBound; epoch += getHourAggregateWidth() { - boundsStart, boundsEnd := getHourAggregateBounds(epoch) - if lastHourExported.EpochEnd == boundsEnd { // no need to update last hour entry if it is complete - d.log.Infof("skipping updating last hour entry since it is complete") - continue - } - - // no need to aggregate epoch data that hasn't been exported yet - if boundsEnd > currentEndBound { - continue - } - - // define start bounds as lastHourExported.EpochEnd for first iteration - if epoch == lastHourExported.EpochStart { - boundsStart = lastHourExported.EpochEnd - } - - // scope down to max currentExportedEpoch (since epoch data is inclusive, add 1) - if currentExportedEpoch+1 >= boundsStart && currentExportedEpoch+1 < boundsEnd { - boundsEnd = currentExportedEpoch + 1 - } - - err = d.aggregate1hWithBounds(boundsStart, boundsEnd) - if err != nil { - return errors.Wrap(err, "failed to aggregate 1h") - } - } - - return nil -} - -// Returns the epoch_start and epoch_end (the epoch bounds of an hourly aggregation) for a given epoch. -// epoch_start is inclusive, epoch_end is exclusive. -func getHourAggregateBounds(epoch uint64) (uint64, uint64) { - offset := utils.GetEpochOffsetGenesis() - epoch += offset // offset to utc - startOfPartition := epoch / getHourAggregateWidth() * getHourAggregateWidth() // inclusive - endOfPartition := startOfPartition + getHourAggregateWidth() // exclusive - if startOfPartition < offset { - startOfPartition = offset - } - return startOfPartition - offset, endOfPartition - offset -} - -func (d *epochToHourAggregator) GetHourPartitionRange(epoch uint64) (uint64, uint64) { - startOfPartition := epoch / (PartitionEpochWidth * getHourAggregateWidth()) * PartitionEpochWidth * getHourAggregateWidth() // inclusive - endOfPartition := startOfPartition + PartitionEpochWidth*getHourAggregateWidth() // exclusive - return startOfPartition, endOfPartition -} - -func (d *epochToHourAggregator) getHourRetentionDurationEpochs() uint64 { - return uint64(float64(utils.EpochsPerDay()) * hourRetentionBuffer) -} - -func (d *epochToHourAggregator) createHourlyPartition(epochStartFrom, epochStartTo uint64) error { - partitionName := fmt.Sprintf("%s_%d_%d", edb.HourWriterTableName, epochStartFrom, epochStartTo) - _, err := db.AlloyWriter.Exec(fmt.Sprintf(` - CREATE TABLE IF NOT EXISTS %s( - LIKE %s INCLUDING DEFAULTS INCLUDING CONSTRAINTS - ) - `, - partitionName, edb.HourWriterTableName, - )) - if err != nil { - return errors.Wrap(err, "failed to create epoch partition (1)") - } - - isAttached, err := isPartitionAttached(edb.HourWriterTableName, partitionName) - if err != nil { - return errors.Wrap(err, "failed to check if partition is attached") - } - - if !isAttached { - _, err = db.AlloyWriter.Exec(fmt.Sprintf(` - ALTER TABLE %s ATTACH PARTITION %s - FOR VALUES FROM (%d) TO (%d) - `, - edb.HourWriterTableName, partitionName, epochStartFrom, epochStartTo, - )) - } - - return err -} - -func (d *epochToHourAggregator) deleteHourlyPartition(epochStartFrom, epochStartTo uint64) error { - _, err := db.AlloyWriter.Exec(fmt.Sprintf(` - ALTER TABLE %[1]s DETACH PARTITION %[1]s_%[2]d_%[3]d; - `, - edb.HourWriterTableName, epochStartFrom, epochStartTo, - )) - - if err != nil { - return err - } - - _, err = db.AlloyWriter.Exec(fmt.Sprintf(` - DROP TABLE IF EXISTS %s_%d_%d - `, - edb.HourWriterTableName, epochStartFrom, epochStartTo, - )) - - return err -} - -// epochStart incl, epochEnd excl -func (d *epochToHourAggregator) aggregate1hWithBounds(epochStart, epochEnd uint64) error { - tx, err := db.AlloyWriter.Beginx() - if err != nil { - return errors.Wrap(err, "failed to start transaction") - } - defer utils.Rollback(tx) - - partitionStartRange, partitionEndRange := d.GetHourPartitionRange(epochStart) - - err = d.createHourlyPartition(partitionStartRange, partitionEndRange) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("failed to create hourly partition, startRange: %d, endRange: %d", partitionStartRange, partitionEndRange)) - } - - boundsStart, _ := getHourAggregateBounds(epochStart) - - if epochStart == partitionStartRange && debugAddToColumnEngine { - err = edb.AddToColumnEngineAllColumns(fmt.Sprintf("%s_%d_%d", edb.HourWriterTableName, partitionStartRange, partitionEndRange)) - if err != nil { - d.log.Warnf("Failed to add epoch to column engine: %v", err) - } - } - - d.log.Infof("aggregating 1h with bounds, startEpoch: %d endEpoch: %d", epochStart, epochEnd) - - err = AddToRollingCustom(tx, CustomRolling{ - StartEpoch: epochStart, - EndEpoch: epochEnd, - StartBoundEpoch: int64(boundsStart), - TableFrom: edb.EpochWriterTableName, - TableTo: edb.HourWriterTableName, - TableFromEpochColumn: "epoch", - Log: d.log, - TailBalancesQuery: fmt.Sprintf(`balance_starts as ( - SELECT validator_index, balance_start FROM %s WHERE epoch = $3 - ),`, edb.EpochWriterTableName), - TailBalancesJoinQuery: `LEFT JOIN balance_starts ON aggregate_head.validator_index = balance_starts.validator_index`, - TailBalancesInsertColumnQuery: `balance_start,`, - TableConflict: "(epoch_start, validator_index)", - }) - - if err != nil { - return errors.Wrap(err, "failed to insert hourly data") - } - - err = tx.Commit() - if err != nil { - return errors.Wrap(err, "failed to commit transaction") - } - return nil -} diff --git a/backend/pkg/exporter/modules/dashboard_data_w_2_epoch_total.go b/backend/pkg/exporter/modules/dashboard_data_w_2_epoch_total.go deleted file mode 100644 index fe03ba2d9..000000000 --- a/backend/pkg/exporter/modules/dashboard_data_w_2_epoch_total.go +++ /dev/null @@ -1,105 +0,0 @@ -package modules - -import ( - "database/sql" - "fmt" - "sync" - "time" - - "github.com/gobitfly/beaconchain/pkg/commons/db" - "github.com/gobitfly/beaconchain/pkg/commons/metrics" - "github.com/gobitfly/beaconchain/pkg/commons/utils" - edb "github.com/gobitfly/beaconchain/pkg/exporter/db" - "github.com/pkg/errors" -) - -type epochToTotalAggregator struct { - *dashboardData - mutex *sync.Mutex -} - -func newEpochToTotalAggregator(d *dashboardData) *epochToTotalAggregator { - return &epochToTotalAggregator{ - dashboardData: d, - mutex: &sync.Mutex{}, - } -} - -// Assumes no gaps in epochs -func (d *epochToTotalAggregator) aggregateTotal(currentExportedEpoch uint64) error { - d.mutex.Lock() - defer d.mutex.Unlock() - - startTime := time.Now() - defer func() { - d.log.Infof("aggregate total took %v", time.Since(startTime)) - metrics.TaskDuration.WithLabelValues("exporter_v2dash_agg_total").Observe(time.Since(startTime).Seconds()) - }() - - lastTotalExported, err := edb.GetLastExportedTotalEpoch() - if err != nil && err != sql.ErrNoRows { - return errors.Wrap(err, "failed to get last exported total epoch") - } - - if currentExportedEpoch < lastTotalExported.EpochEnd { - return errors.Wrap(err, "total export nothing to do, currentEpoch <= lastTotalExported.EpochEnd") - } - - gaps, err := edb.GetMissingEpochsBetween(int64(lastTotalExported.EpochEnd), int64(currentExportedEpoch+1)) - if err != nil { - return errors.Wrap(err, "failed to get dashboard epoch gaps") - } - - if len(gaps) > 0 { - return fmt.Errorf("gaps in dashboard epoch, skipping total for now: %v", gaps) // sanity, this should never happen - } - - err = d.aggregateAndAddToTotal(lastTotalExported.EpochEnd, currentExportedEpoch+1) - if err != nil { - return errors.Wrap(err, "failed to aggregate total") - } - - return nil -} - -// epochStart incl, epochEnd excl -func (d *epochToTotalAggregator) aggregateAndAddToTotal(epochStart, epochEnd uint64) error { - tx, err := db.AlloyWriter.Beginx() - if err != nil { - return err - } - defer utils.Rollback(tx) - - d.log.Infof("aggregating total (from: %d) up to %d", epochStart, epochEnd) - - err = AddToRollingCustom(tx, CustomRolling{ - StartEpoch: epochStart, - EndEpoch: epochEnd, - StartBoundEpoch: 0, - TableFrom: edb.EpochWriterTableName, - TableTo: edb.RollingTotalWriterTableName, - TailBalancesInsertColumnQuery: "0 as balance_start,", // Since all validators start with a 0 balance until deposit is voted in, we can just set it to 0. Genesis validators will be set to 0 to unify the data access approach - TableFromEpochColumn: "epoch", - Log: d.log, - TableConflict: "(validator_index)", - - // This may come in handy at some point so leaving it there if you need the first value in an epoch range for a given validator - - // TailBalancesQuery: ` - // balance_start_epochs as ( - // SELECT validator_index, MIN(epoch) as epoch FROM validator_dashboard_data_epoch WHERE epoch >= $1 AND epoch <= $2 AND balance_start IS NOT NULL - // GROUP BY validator_index - // ), - // balance_starts as ( - // SELECT validator_index, balance_start FROM balance_start_epochs LEFT JOIN validator_dashboard_data_epoch USING (validator_index, epoch) - // ),`, - // TailBalancesJoinQuery: `LEFT JOIN balance_starts ON aggregate_head.validator_index = balance_starts.validator_index`, - //TailBalancesInsertColumnQuery: "balance_start,", - }) - - if err != nil { - return err - } - - return tx.Commit() -} diff --git a/backend/pkg/exporter/modules/dashboard_data_w_3_epoch_day.go b/backend/pkg/exporter/modules/dashboard_data_w_3_epoch_day.go deleted file mode 100644 index 61192600f..000000000 --- a/backend/pkg/exporter/modules/dashboard_data_w_3_epoch_day.go +++ /dev/null @@ -1,510 +0,0 @@ -package modules - -import ( - "database/sql" - "fmt" - "regexp" - "sync" - "time" - - "github.com/gobitfly/beaconchain/pkg/commons/db" - "github.com/gobitfly/beaconchain/pkg/commons/metrics" - "github.com/gobitfly/beaconchain/pkg/commons/utils" - edb "github.com/gobitfly/beaconchain/pkg/exporter/db" - "github.com/jmoiron/sqlx" - "github.com/pkg/errors" -) - -type epochToDayAggregator struct { - *dashboardData - mutex *sync.Mutex - rollingAggregator RollingAggregator -} - -// How long aggregated hours will remain in the database is defined in getDayRetentionDurationDays. -// This depends on the max rolling timeframe we supports, so 90d as of now. -// This buffer can be used to increase or decrease from that 90d day target. A value of 1 will keep exactly those 90d in the database. -const dayRetentionBuffer = 1.2 // do not go below 1 - -const PartitionDayWidth = 6 - -func newEpochToDayAggregator(d *dashboardData) *epochToDayAggregator { - return &epochToDayAggregator{ - dashboardData: d, - mutex: &sync.Mutex{}, - rollingAggregator: RollingAggregator{ - log: d.log, - RollingAggregatorInt: &DayRollingAggregatorImpl{ - log: d.log, - }, - }, - } -} - -func GetDayAggregateWidth() uint64 { - return utils.EpochsPerDay() -} - -func (d *epochToDayAggregator) dayAggregate(currentExportedEpoch uint64) error { - d.mutex.Lock() - defer d.mutex.Unlock() - - err := d.utcDayAggregate(currentExportedEpoch) - if err != nil { - return errors.Wrap(err, "failed to utc day aggregate") - } - - return nil -} - -// used to retrieve missing historic epochs in database for rolling 24h aggregation -// intentedHeadEpoch is the head you currently want to export -func (d *epochToDayAggregator) getMissingRolling24TailEpochs(intendedHeadEpoch uint64) ([]uint64, error) { - return d.rollingAggregator.getMissingRollingTailEpochs(1, intendedHeadEpoch, edb.RollingDailyWriterTable) -} - -func (d *epochToDayAggregator) rolling24hAggregate(currentEpochHead uint64) error { - return d.rollingAggregator.Aggregate(1, edb.RollingDailyWriterTable, currentEpochHead) -} - -// Returns the epoch_start and epoch_end (the epoch bounds of a UTC day) for a given epoch. -// epoch_start is inclusive, epoch_end is exclusive. -func getDayAggregateBounds(epoch uint64) (uint64, uint64) { - offset := utils.GetEpochOffsetGenesis() - epoch += offset // offset to utc - startOfPartition := epoch / GetDayAggregateWidth() * GetDayAggregateWidth() // inclusive - endOfPartition := startOfPartition + GetDayAggregateWidth() // exclusive - if startOfPartition < offset { - startOfPartition = offset - } - return startOfPartition - offset, endOfPartition - offset -} - -func (d *epochToDayAggregator) utcDayAggregate(currentExportedEpoch uint64) error { - startTime := time.Now() - defer func() { - d.log.Infof("utc day aggregate took %v", time.Since(startTime)) - metrics.TaskDuration.WithLabelValues("exporter_v2dash_agg_utc_day").Observe(time.Since(startTime).Seconds()) - }() - - latestExportedDay, err := edb.GetLastExportedDay() - if err != nil && err != sql.ErrNoRows { - return errors.Wrap(err, "failed to get latest daily epoch") - } - - gaps, err := edb.GetMissingEpochsBetween(int64(latestExportedDay.EpochEnd), int64(currentExportedEpoch+1)) - if err != nil { - return errors.Wrap(err, "failed to get dashboard epoch gaps") - } - - if len(gaps) > 0 { - return fmt.Errorf("gaps in dashboard epoch for utc day agg, skipping for now: %v (%v-%v)", gaps, latestExportedDay.EpochEnd, currentExportedEpoch) // sanity, this should never happen - } - - _, currentEndBound := getDayAggregateBounds(currentExportedEpoch) - - for epoch := latestExportedDay.EpochStart; epoch <= currentEndBound; epoch += GetDayAggregateWidth() { - boundsStart, boundsEnd := getDayAggregateBounds(epoch) - if latestExportedDay.EpochEnd == boundsEnd { // no need to update last hour entry if it is complete - d.log.Infof("skipping updating last day entry since it is complete") - continue - } - - // no need to aggregate epoch data that hasn't been exported yet - if boundsEnd > currentEndBound { - continue - } - - // define start bounds as lastHourExported.EpochEnd for first iteration - if epoch == latestExportedDay.EpochStart { - boundsStart = latestExportedDay.EpochEnd - } - - // scope down to max currentExportedEpoch (since epoch data is inclusive, add 1) - if currentExportedEpoch+1 >= boundsStart && currentExportedEpoch+1 < boundsEnd { - boundsEnd = currentExportedEpoch + 1 - } - - err = d.aggregateUtcDayWithBounds(boundsStart, boundsEnd) - if err != nil { - d.log.Error(err, "failed to aggregate utc day with bounds", 0) - return errors.Wrap(err, "failed to aggregate utc day with bounds") - } - } - - return nil -} - -// firstEpochOfDay incl, lastEpochOfDay excl -func (d *epochToDayAggregator) aggregateUtcDayWithBounds(firstEpochOfDay, lastEpochOfDay uint64) error { - d.log.Infof("aggregating day of epoch %d", firstEpochOfDay) - partitionStartRange, partitionEndRange := d.GetDayPartitionRange(lastEpochOfDay) - - err := d.createDayPartition(partitionStartRange, partitionEndRange) - if err != nil { - return errors.Wrap(err, "failed to create day partition") - } - - boundsStart, _ := getDayAggregateBounds(firstEpochOfDay) - - tx, err := db.AlloyWriter.Beginx() - if err != nil { - return errors.Wrap(err, "failed to start transaction") - } - defer utils.Rollback(tx) - - err = AddToRollingCustom(tx, CustomRolling{ - StartEpoch: firstEpochOfDay, - EndEpoch: lastEpochOfDay, - StartBoundEpoch: int64(boundsStart), - TableFrom: edb.EpochWriterTableName, - TableTo: edb.DayWriterTableName, - TableFromEpochColumn: "epoch", - Log: d.log, - TailBalancesQuery: fmt.Sprintf(`balance_starts as ( - SELECT validator_index, balance_start FROM %s WHERE epoch = $3 - ),`, edb.EpochWriterTableName), - TailBalancesJoinQuery: `LEFT JOIN balance_starts ON aggregate_head.validator_index = balance_starts.validator_index`, - TailBalancesInsertColumnQuery: `balance_start,`, - TableDayColum: "day,", - TableDayValue: fmt.Sprintf("'%s' as day,", utils.EpochToTime(boundsStart).Format("2006-01-02")), - TableConflict: "(day, validator_index)", - }) - - if err != nil { - return errors.Wrap(err, "failed to insert daily aggregate") - } - - return tx.Commit() -} - -func (d *epochToDayAggregator) GetDayPartitionRange(epoch uint64) (time.Time, time.Time) { - _, boundsEnd := getDayAggregateBounds(epoch) - startOfPartition := boundsEnd / (PartitionDayWidth * GetDayAggregateWidth()) * PartitionDayWidth * GetDayAggregateWidth() // inclusive - endOfPartition := startOfPartition + PartitionDayWidth*GetDayAggregateWidth() // exclusive - return utils.EpochToTime(startOfPartition), utils.EpochToTime(endOfPartition) -} - -func (d *epochToDayAggregator) createDayPartition(dayFrom, dayTo time.Time) error { - partitionName := fmt.Sprintf("%s_%s_%s", edb.DayWriterTableName, dayToYYMMDDLabel(dayFrom), dayToYYMMDDLabel(dayTo)) - _, err := db.AlloyWriter.Exec(fmt.Sprintf(` - CREATE TABLE IF NOT EXISTS %s( - LIKE %s INCLUDING DEFAULTS INCLUDING CONSTRAINTS - ) - `, - partitionName, edb.DayWriterTableName, - )) - - if err != nil { - return errors.Wrap(err, "failed to create epoch partition (1)") - } - - isAttached, err := isPartitionAttached(edb.DayWriterTableName, partitionName) - if err != nil { - return errors.Wrap(err, "failed to check if partition is attached") - } - - if !isAttached { - _, err = db.AlloyWriter.Exec(fmt.Sprintf(` - ALTER TABLE %s ATTACH PARTITION %s - FOR VALUES FROM ('%s') TO ('%s') - `, - edb.DayWriterTableName, partitionName, dayToDDMMYY(dayFrom), dayToDDMMYY(dayTo), - )) - } - - return err -} - -func dayToYYMMDDLabel(day time.Time) string { - return day.Format("20060102") -} - -func dayToDDMMYY(day time.Time) string { - return day.Format("02-January-2006") -} - -// -- rolling aggregate -- - -type DayRollingAggregatorImpl struct { - log ModuleLog -} - -// returns both start_epochs -// the epoch_start from the the bootstrap tail -// and the epoch_start from the bootstrap head (epoch_start of latestExportedHourEpoch) -func (d *DayRollingAggregatorImpl) getBootstrapBounds(latestExportedHourEpoch uint64, _ uint64) (uint64, uint64) { - currentStartBounds, _ := getHourAggregateBounds(latestExportedHourEpoch) - - dayOldEpoch := int64(currentStartBounds - utils.EpochsPerDay()) - if dayOldEpoch < 0 { - dayOldEpoch = 0 - } - dayOldBoundsStart, _ := getHourAggregateBounds(uint64(dayOldEpoch)) - return dayOldBoundsStart, currentStartBounds -} - -// How many epochs can you be behind in the rolling table without bootstrap -func (d *DayRollingAggregatorImpl) getBootstrapOnEpochsBehind() uint64 { - return getHourAggregateWidth() -} - -// Bootstrap rolling 24h table from hourly table -func (d *DayRollingAggregatorImpl) bootstrap(tx *sqlx.Tx, days int, tableName string) error { - startTime := time.Now() - defer func() { - d.log.Infof("bootstrap 24h aggregate took %v", time.Since(startTime)) - }() - - latestHourlyEpochBounds, err := edb.GetLastExportedHour() - if err != nil && err != sql.ErrNoRows { - return errors.Wrap(err, "failed to get latest dashboard epoch") - } - - dayOldBoundsStart, latestHourlyEpoch := d.getBootstrapBounds(latestHourlyEpochBounds.EpochEnd, 1) - - var found bool - err = db.AlloyWriter.Get(&found, fmt.Sprintf(` - SELECT true FROM %s WHERE epoch_start = $1 LIMIT 1 - `, edb.HourWriterTableName), dayOldBoundsStart) - if err != nil || !found { - return errors.Wrap(err, fmt.Sprintf("failed to check if tail %s epoch_start %v exists", edb.HourWriterTableName, dayOldBoundsStart)) - } - - latestHourlyStartBound, _ := getHourAggregateBounds(latestHourlyEpoch - 1) // excl - d.log.Infof("latestHourlyEpoch (excl): %d, dayOldHourlyEpoch: %d, latestHourlyStartBound (incl): %d", latestHourlyEpoch, dayOldBoundsStart, latestHourlyStartBound) - - _, err = tx.Exec(fmt.Sprintf(`TRUNCATE %s`, edb.RollingDailyWriterTable)) - if err != nil { - return errors.Wrap(err, "failed to delete old rolling 24h aggregate") - } - - _, err = tx.Exec(fmt.Sprintf(` - WITH - epoch_ends as ( - SELECT max(epoch_end) as epoch_end FROM %[2]s WHERE epoch_start = $2 LIMIT 1 - ), - balance_starts as ( - SELECT validator_index, balance_start FROM %[2]s WHERE epoch_start = $1 - ), - balance_ends as ( - SELECT validator_index, balance_end FROM %[2]s WHERE epoch_start = $2 - ), - aggregate as ( - SELECT - validator_index, - SUM(attestations_source_reward) as attestations_source_reward, - SUM(attestations_target_reward) as attestations_target_reward, - SUM(attestations_head_reward) as attestations_head_reward, - SUM(attestations_inactivity_reward) as attestations_inactivity_reward, - SUM(attestations_inclusion_reward) as attestations_inclusion_reward, - SUM(attestations_reward) as attestations_reward, - SUM(attestations_ideal_source_reward) as attestations_ideal_source_reward, - SUM(attestations_ideal_target_reward) as attestations_ideal_target_reward, - SUM(attestations_ideal_head_reward) as attestations_ideal_head_reward, - SUM(attestations_ideal_inactivity_reward) as attestations_ideal_inactivity_reward, - SUM(attestations_ideal_inclusion_reward) as attestations_ideal_inclusion_reward, - SUM(attestations_ideal_reward) as attestations_ideal_reward, - SUM(blocks_scheduled) as blocks_scheduled, - SUM(blocks_proposed) as blocks_proposed, - SUM(blocks_cl_reward) as blocks_cl_reward, - SUM(blocks_cl_attestations_reward) as blocks_cl_attestations_reward, - SUM(blocks_cl_sync_aggregate_reward) as blocks_cl_sync_aggregate_reward, - SUM(sync_scheduled) as sync_scheduled, - SUM(sync_executed) as sync_executed, - SUM(sync_reward) as sync_reward, - bool_or(slashed) as slashed, - SUM(deposits_count) as deposits_count, - SUM(deposits_amount) as deposits_amount, - SUM(withdrawals_count) as withdrawals_count, - SUM(withdrawals_amount) as withdrawals_amount, - SUM(inclusion_delay_sum) as inclusion_delay_sum, - SUM(blocks_expected) as blocks_expected, - SUM(sync_committees_expected) as sync_committees_expected, - SUM(attestations_scheduled) as attestations_scheduled, - SUM(attestations_observed) as attestations_observed, - SUM(attestations_head_executed) as attestations_head_executed, - SUM(attestations_source_executed) as attestations_source_executed, - SUM(attestations_target_executed) as attestations_target_executed, - SUM(optimal_inclusion_delay_sum) as optimal_inclusion_delay_sum, - SUM(slasher_reward) as slasher_reward, - MAX(slashed_by) as slashed_by, - MAX(slashed_violation) as slashed_violation, - MAX(last_executed_duty_epoch) as last_executed_duty_epoch - FROM %[2]s - WHERE epoch_start >= $1 AND epoch_start <= $2 - GROUP BY validator_index - ) - INSERT INTO %[1]s ( - validator_index, - epoch_start, - epoch_end, - attestations_source_reward, - attestations_target_reward, - attestations_head_reward, - attestations_inactivity_reward, - attestations_inclusion_reward, - attestations_reward, - attestations_ideal_source_reward, - attestations_ideal_target_reward, - attestations_ideal_head_reward, - attestations_ideal_inactivity_reward, - attestations_ideal_inclusion_reward, - attestations_ideal_reward, - blocks_scheduled, - blocks_proposed, - blocks_cl_reward, - blocks_cl_attestations_reward, - blocks_cl_sync_aggregate_reward, - sync_scheduled, - sync_executed, - sync_reward, - slashed, - balance_start, - balance_end, - deposits_count, - deposits_amount, - withdrawals_count, - withdrawals_amount, - inclusion_delay_sum, - blocks_expected, - sync_committees_expected, - attestations_scheduled, - attestations_observed, - attestations_head_executed, - attestations_source_executed, - attestations_target_executed, - optimal_inclusion_delay_sum, - slasher_reward, - slashed_by, - slashed_violation, - last_executed_duty_epoch - ) - SELECT - aggregate.validator_index, - $1, - (SELECT epoch_end FROM epoch_ends), - attestations_source_reward, - attestations_target_reward, - attestations_head_reward, - attestations_inactivity_reward, - attestations_inclusion_reward, - attestations_reward, - attestations_ideal_source_reward, - attestations_ideal_target_reward, - attestations_ideal_head_reward, - attestations_ideal_inactivity_reward, - attestations_ideal_inclusion_reward, - attestations_ideal_reward, - blocks_scheduled, - blocks_proposed, - blocks_cl_reward, - blocks_cl_attestations_reward, - blocks_cl_sync_aggregate_reward, - sync_scheduled, - sync_executed, - sync_reward, - slashed, - balance_start, - balance_end, - deposits_count, - deposits_amount, - withdrawals_count, - withdrawals_amount, - inclusion_delay_sum, - blocks_expected, - sync_committees_expected, - attestations_scheduled, - attestations_observed, - attestations_head_executed, - attestations_source_executed, - attestations_target_executed, - optimal_inclusion_delay_sum, - slasher_reward, - slashed_by, - slashed_violation, - last_executed_duty_epoch - FROM aggregate - LEFT JOIN balance_starts ON aggregate.validator_index = balance_starts.validator_index - LEFT JOIN balance_ends ON aggregate.validator_index = balance_ends.validator_index - `, edb.RollingDailyWriterTable, edb.HourWriterTableName), dayOldBoundsStart, latestHourlyStartBound) - - if err != nil { - return errors.Wrap(err, "failed to insert rolling 24h aggregate") - } - - return nil -} - -func (d *epochToDayAggregator) clearOldDayAggregations(removeOlderThanDays uint64) error { - partitions, err := edb.GetPartitionNamesOfTable(edb.DayWriterTableName) - if err != nil { - return errors.Wrap(err, "failed to get partitions") - } - - for _, partition := range partitions { - dateFromString, dateToString, err := parseDayRange(fmt.Sprintf(`%s_(\d+)_(\d+)`, edb.DayWriterTableName), partition) - if err != nil { - return errors.Wrap(err, "failed to parse day range") - } - - dateTo, err := time.Parse("20060102", dateToString) - if err != nil { - return errors.Wrap(err, "failed to parse dateTo") - } - - daysAgo := int64(time.Since(dateTo).Hours() / 24) - - if daysAgo > int64(removeOlderThanDays) { - d.mutex.Lock() - err := d.deleteDayPartition(dateFromString, dateToString) - d.log.Infof("Deleted old day partition %s-%s (%d days ago)", dateFromString, dateToString, daysAgo) - d.mutex.Unlock() - if err != nil { - return errors.Wrap(err, "failed to delete day partition") - } - } - } - - return nil -} - -func parseDayRange(pattern, partition string) (string, string, error) { - // Compile the regular expression pattern - regex := regexp.MustCompile(pattern) - - // Find the matches in the partition string - matches := regex.FindStringSubmatch(partition) - - // Check if the partition string matches the pattern - if len(matches) != 3 { - return "", "", fmt.Errorf("invalid partition string: %s", partition) - } - - return matches[1], matches[2], nil -} - -func (d *epochToDayAggregator) getDayRetentionDurationDays() uint64 { - return 90 * dayRetentionBuffer // max rolling timeframe -} - -func (d *epochToDayAggregator) deleteDayPartition(epochStartFrom, epochStartTo string) error { - _, err := db.AlloyWriter.Exec(fmt.Sprintf(` - ALTER TABLE %[1]s DETACH PARTITION %[1]s_%[2]s_%[3]s; - `, - edb.DayWriterTableName, epochStartFrom, epochStartTo, - )) - - if err != nil { - return err - } - - query := fmt.Sprintf(` - DROP TABLE IF EXISTS %s_%s_%s - `, - edb.DayWriterTableName, epochStartFrom, epochStartTo, - ) - - _, err = db.AlloyWriter.Exec(query) - return err -} diff --git a/backend/pkg/exporter/modules/dashboard_data_w_4_day_weekly-month-90d.go b/backend/pkg/exporter/modules/dashboard_data_w_4_day_weekly-month-90d.go deleted file mode 100644 index 821f430bb..000000000 --- a/backend/pkg/exporter/modules/dashboard_data_w_4_day_weekly-month-90d.go +++ /dev/null @@ -1,333 +0,0 @@ -package modules - -import ( - "database/sql" - "fmt" - "sync" - "time" - - "github.com/gobitfly/beaconchain/pkg/commons/utils" - edb "github.com/gobitfly/beaconchain/pkg/exporter/db" - "github.com/jmoiron/sqlx" - "github.com/pkg/errors" -) - -type dayUpAggregator struct { - *dashboardData - setupMutex *sync.Mutex - mutexes map[string]*sync.Mutex - rollingAggregator RollingAggregator -} - -func newDayUpAggregator(d *dashboardData) *dayUpAggregator { - return &dayUpAggregator{ - dashboardData: d, - mutexes: make(map[string]*sync.Mutex), - setupMutex: &sync.Mutex{}, - rollingAggregator: RollingAggregator{ - log: d.log, - RollingAggregatorInt: &MultipleDaysRollingAggregatorImpl{ - log: d.log, - }, - }, - } -} - -func (d *dayUpAggregator) rolling7dAggregate(currentEpochHead uint64) error { - return d.aggregateRollingXDays(7, edb.RollingWeeklyWriterTable, currentEpochHead) -} - -func (d *dayUpAggregator) rolling30dAggregate(currentEpochHead uint64) error { - return d.aggregateRollingXDays(30, edb.RollingMonthlyWriterTable, currentEpochHead) -} - -func (d *dayUpAggregator) rolling90dAggregate(currentEpochHead uint64) error { - return d.aggregateRollingXDays(90, edb.RollingNinetyDaysWriterTable, currentEpochHead) -} - -// for a given epoch intendedHeadEpoch, what epochs are needed for removal from the rolling tables -func (d *dayUpAggregator) getMissingRollingDayTailEpochs(intendedHeadEpoch uint64) ([]uint64, error) { - week, err := d.getMissingRollingXDaysTailEpochs(7, intendedHeadEpoch, edb.RollingWeeklyWriterTable) - if err != nil { - return nil, errors.Wrap(err, "failed to get missing 7d tail epochs") - } - month, err := d.getMissingRollingXDaysTailEpochs(30, intendedHeadEpoch, edb.RollingMonthlyWriterTable) - if err != nil { - return nil, errors.Wrap(err, "failed to get missing 30d tail epochs") - } - ninety, err := d.getMissingRollingXDaysTailEpochs(90, intendedHeadEpoch, edb.RollingNinetyDaysWriterTable) - if err != nil { - return nil, errors.Wrap(err, "failed to get missing 90d tail epochs") - } - - d.log.Infof("missing tail 7d: %v", week) - d.log.Infof("missing tail 30d: %v", month) - d.log.Infof("missing tail 90d: %v", ninety) - - return utils.Deduplicate(append(append(week, month...), ninety...)), nil -} - -// for a given epoch intendedHeadEpoch, what epochs are needed to add to the rolling table (excluding the current epoch) -func (d *dayUpAggregator) getMissingRollingDayHeadEpochs(intendedHeadEpoch uint64) ([]uint64, error) { - week, err := d.getMissingRollingXDaysHeadEpochs(7, intendedHeadEpoch-1, edb.RollingWeeklyWriterTable) - if err != nil { - return nil, errors.Wrap(err, "failed to get missing 7d head epochs") - } - month, err := d.getMissingRollingXDaysHeadEpochs(30, intendedHeadEpoch-1, edb.RollingMonthlyWriterTable) - if err != nil { - return nil, errors.Wrap(err, "failed to get missing 30d head epochs") - } - ninety, err := d.getMissingRollingXDaysHeadEpochs(90, intendedHeadEpoch-1, edb.RollingNinetyDaysWriterTable) - if err != nil { - return nil, errors.Wrap(err, "failed to get missing 90d head epochs") - } - - heads := utils.Deduplicate(append(append(week, month...), ninety...)) - d.log.Infof("missing head: %v", heads) - - return heads, nil -} - -func (d *dayUpAggregator) getMissingRollingXDaysTailEpochs(days int, intendedHeadEpoch uint64, tableName string) ([]uint64, error) { - return d.rollingAggregator.getMissingRollingTailEpochs(days, intendedHeadEpoch, tableName) -} - -func (d *dayUpAggregator) getMissingRollingXDaysHeadEpochs(days int, intendedHeadEpoch uint64, tableName string) ([]uint64, error) { - bounds, err := d.rollingAggregator.getCurrentRollingBounds(nil, tableName) - if err != nil { - if err != sql.ErrNoRows { - return nil, errors.Wrap(err, fmt.Sprintf("failed to get latest exported rolling %dd bounds", days)) - } - } - - if bounds.EpochEnd <= 0 || intendedHeadEpoch-bounds.EpochEnd < d.epochWriter.getRetentionEpochDuration() { - return nil, nil - } - - // don't fetch head if bootstrap is necessary anyway - if intendedHeadEpoch+1-bounds.EpochEnd >= d.rollingAggregator.getBootstrapOnEpochsBehind() { - return nil, nil - } - - return edb.GetMissingEpochsBetween(int64(bounds.EpochEnd), int64(intendedHeadEpoch)+1) -} - -func (d *dayUpAggregator) aggregateRollingXDays(days int, tableName string, currentEpochHead uint64) error { - d.setupMutex.Lock() - if _, ok := d.mutexes[tableName]; !ok { - d.mutexes[tableName] = &sync.Mutex{} - } - d.setupMutex.Unlock() - - d.mutexes[tableName].Lock() - defer d.mutexes[tableName].Unlock() - - return d.rollingAggregator.Aggregate(days, tableName, currentEpochHead) -} - -// -- rolling aggregate -- - -type MultipleDaysRollingAggregatorImpl struct { - log ModuleLog -} - -// returns both start_epochs -// the epoch_start from the the bootstrap tail -// and the epoch_start from the bootstrap head (epoch_start of epoch) -func (d *MultipleDaysRollingAggregatorImpl) getBootstrapBounds(epoch uint64, days uint64) (uint64, uint64) { - currentStartBounds, _ := getDayAggregateBounds(epoch) - xDayOldEpoch := int64(currentStartBounds - days*utils.EpochsPerDay()) - if xDayOldEpoch < 0 { - xDayOldEpoch = 0 - } - dayOldBoundsStart, _ := getDayAggregateBounds(uint64(xDayOldEpoch)) - return dayOldBoundsStart, currentStartBounds -} - -// how many epochs can the rolling table be behind without bootstrapping -func (d *MultipleDaysRollingAggregatorImpl) getBootstrapOnEpochsBehind() uint64 { - return utils.EpochsPerDay() -} - -// bootstrap rolling 7d, 30d, 90d table from utc day table -func (d *MultipleDaysRollingAggregatorImpl) bootstrap(tx *sqlx.Tx, days int, tableName string) error { - startTime := time.Now() - defer func() { - d.log.Infof("bootstrap rolling %vd took %v", days, time.Since(startTime)) - }() - - latestDayBounds, err := edb.GetLastExportedDay() - if err != nil && err != sql.ErrNoRows { - return errors.Wrap(err, "failed to get latest exported day") - } - latestDay := latestDayBounds.Day - - tailStart, _ := d.getBootstrapBounds(latestDayBounds.EpochStart, uint64(days)) - xDayOldDay := utils.EpochToTime(tailStart).Format("2006-01-02") - - d.log.Infof("agg %dd | latestDay: %v, oldDay: %v", days, latestDay, xDayOldDay) - - _, err = tx.Exec(fmt.Sprintf(`TRUNCATE %s`, tableName)) - if err != nil { - return errors.Wrap(err, "failed to delete old rolling aggregate") - } - - _, err = tx.Exec(fmt.Sprintf(` - WITH - epoch_starts as ( - SELECT min(epoch_start) as epoch_start FROM %[2]s WHERE day = $1 LIMIT 1 - ), - epoch_ends as ( - SELECT max(epoch_end) as epoch_end FROM %[2]s WHERE day = $2 LIMIT 1 - ), - balance_starts as ( - SELECT validator_index, balance_start, epoch_start FROM %[2]s WHERE day = $1 - ), - balance_ends as ( - SELECT validator_index, balance_end, epoch_end FROM %[2]s WHERE day = $2 - ), - aggregate as ( - SELECT - validator_index, - SUM(attestations_source_reward) as attestations_source_reward, - SUM(attestations_target_reward) as attestations_target_reward, - SUM(attestations_head_reward) as attestations_head_reward, - SUM(attestations_inactivity_reward) as attestations_inactivity_reward, - SUM(attestations_inclusion_reward) as attestations_inclusion_reward, - SUM(attestations_reward) as attestations_reward, - SUM(attestations_ideal_source_reward) as attestations_ideal_source_reward, - SUM(attestations_ideal_target_reward) as attestations_ideal_target_reward, - SUM(attestations_ideal_head_reward) as attestations_ideal_head_reward, - SUM(attestations_ideal_inactivity_reward) as attestations_ideal_inactivity_reward, - SUM(attestations_ideal_inclusion_reward) as attestations_ideal_inclusion_reward, - SUM(attestations_ideal_reward) as attestations_ideal_reward, - SUM(blocks_scheduled) as blocks_scheduled, - SUM(blocks_proposed) as blocks_proposed, - SUM(blocks_cl_reward) as blocks_cl_reward, - SUM(blocks_cl_attestations_reward) as blocks_cl_attestations_reward, - SUM(blocks_cl_sync_aggregate_reward) as blocks_cl_sync_aggregate_reward, - SUM(sync_scheduled) as sync_scheduled, - SUM(sync_executed) as sync_executed, - SUM(sync_reward) as sync_reward, - bool_or(slashed) as slashed, - SUM(deposits_count) as deposits_count, - SUM(deposits_amount) as deposits_amount, - SUM(withdrawals_count) as withdrawals_count, - SUM(withdrawals_amount) as withdrawals_amount, - SUM(inclusion_delay_sum) as inclusion_delay_sum, - SUM(blocks_expected) as blocks_expected, - SUM(sync_committees_expected) as sync_committees_expected, - SUM(attestations_scheduled) as attestations_scheduled, - SUM(attestations_observed) as attestations_observed, - SUM(attestations_head_executed) as attestations_head_executed, - SUM(attestations_source_executed) as attestations_source_executed, - SUM(attestations_target_executed) as attestations_target_executed, - SUM(optimal_inclusion_delay_sum) as optimal_inclusion_delay_sum, - SUM(slasher_reward) as slasher_reward, - MAX(slashed_by) as slashed_by, - MAX(slashed_violation) as slashed_violation, - MAX(last_executed_duty_epoch) as last_executed_duty_epoch - FROM %[2]s - WHERE day >= $1 AND day <= $2 - GROUP BY validator_index - ) - INSERT INTO %[1]s ( - validator_index, - epoch_start, - epoch_end, - attestations_source_reward, - attestations_target_reward, - attestations_head_reward, - attestations_inactivity_reward, - attestations_inclusion_reward, - attestations_reward, - attestations_ideal_source_reward, - attestations_ideal_target_reward, - attestations_ideal_head_reward, - attestations_ideal_inactivity_reward, - attestations_ideal_inclusion_reward, - attestations_ideal_reward, - blocks_scheduled, - blocks_proposed, - blocks_cl_reward, - blocks_cl_attestations_reward, - blocks_cl_sync_aggregate_reward, - sync_scheduled, - sync_executed, - sync_reward, - slashed, - balance_start, - balance_end, - deposits_count, - deposits_amount, - withdrawals_count, - withdrawals_amount, - inclusion_delay_sum, - blocks_expected, - sync_committees_expected, - attestations_scheduled, - attestations_observed, - attestations_head_executed, - attestations_source_executed, - attestations_target_executed, - optimal_inclusion_delay_sum, - slasher_reward, - slashed_by, - slashed_violation, - last_executed_duty_epoch - ) - SELECT - aggregate.validator_index, - (SELECT epoch_start FROM epoch_starts), - (SELECT epoch_end FROM epoch_ends), - attestations_source_reward, - attestations_target_reward, - attestations_head_reward, - attestations_inactivity_reward, - attestations_inclusion_reward, - attestations_reward, - attestations_ideal_source_reward, - attestations_ideal_target_reward, - attestations_ideal_head_reward, - attestations_ideal_inactivity_reward, - attestations_ideal_inclusion_reward, - attestations_ideal_reward, - blocks_scheduled, - blocks_proposed, - blocks_cl_reward, - blocks_cl_attestations_reward, - blocks_cl_sync_aggregate_reward, - sync_scheduled, - sync_executed, - sync_reward, - slashed, - balance_start, - balance_end, - deposits_count, - deposits_amount, - withdrawals_count, - withdrawals_amount, - inclusion_delay_sum, - blocks_expected, - sync_committees_expected, - attestations_scheduled, - attestations_observed, - attestations_head_executed, - attestations_source_executed, - attestations_target_executed, - optimal_inclusion_delay_sum, - slasher_reward, - slashed_by, - slashed_violation, - last_executed_duty_epoch - FROM aggregate - LEFT JOIN balance_starts ON aggregate.validator_index = balance_starts.validator_index - LEFT JOIN balance_ends ON aggregate.validator_index = balance_ends.validator_index - `, tableName, edb.DayWriterTableName), xDayOldDay, latestDay) - - if err != nil { - return errors.Wrap(err, "failed to insert rolling aggregate") - } - - return nil -} diff --git a/backend/pkg/exporter/modules/dashboard_data_w_rolling.go b/backend/pkg/exporter/modules/dashboard_data_w_rolling.go deleted file mode 100644 index 69d0d3392..000000000 --- a/backend/pkg/exporter/modules/dashboard_data_w_rolling.go +++ /dev/null @@ -1,794 +0,0 @@ -package modules - -import ( - "bytes" - "context" - "database/sql" - "fmt" - "text/template" - "time" - - "github.com/gobitfly/beaconchain/pkg/commons/db" - "github.com/gobitfly/beaconchain/pkg/commons/log" - "github.com/gobitfly/beaconchain/pkg/commons/metrics" - "github.com/gobitfly/beaconchain/pkg/commons/utils" - edb "github.com/gobitfly/beaconchain/pkg/exporter/db" - "github.com/jmoiron/sqlx" - "github.com/pkg/errors" -) - -/** -This file handles the logic for rolling aggregation for 24h, 7d, 31d and 90d. Total also relies on the AddRollingCustom method as well as the day and hour aggregate. -The way this works is by adding new epochs to the rolling table (adding to head) and removing the old epochs at the end (removing tail) so that the time duration of rolling stays constant. - -If the rolling tables fall out of sync due to long offline time or initial sync, the tables are bootstrapped. This bootstrap method must be provided, -7d, 31d, 90d use a bootstrap from the utc_days table to get started and 24h the hourly table. -*/ - -/* - In case exporter needs to be reset and re-export from an older epoch again, the rolling tables must be truncated. - Total must be bootstrapped by hand, for example via daily utc table. Other rolling tables will be bootstrapped automatically. -*/ - -type RollingAggregator struct { - RollingAggregatorInt - log ModuleLog -} - -type RollingAggregatorInt interface { - bootstrap(tx *sqlx.Tx, days int, tableName string) error - - // get the threshold on how many epochs you can be behind without bootstrap or at which distance there will be a bootstrap - getBootstrapOnEpochsBehind() uint64 - - // gets the aggegate bounds for a given epoch in the bootstrap table. Is useful if you want to know what aggregate an epoch is part of - getBootstrapBounds(epoch uint64, days uint64) (uint64, uint64) -} - -// Returns the epoch range of a current exported rolling table -// Ideally the epoch range has an exact with of 24h, 7d, 31d or 90d BUT it can be more after bootstrap or less if there are less epochs on the network than the rolling width -func (d *RollingAggregator) getCurrentRollingBounds(tx *sqlx.Tx, tableName string) (edb.EpochBounds, error) { - var bounds edb.EpochBounds - var err error - if tx == nil { - err = db.AlloyWriter.Get(&bounds, fmt.Sprintf(`SELECT max(epoch_start) as epoch_start, max(epoch_end) as epoch_end FROM %s`, tableName)) - } else { - err = tx.Get(&bounds, fmt.Sprintf(`SELECT max(epoch_start) as epoch_start, max(epoch_end) as epoch_end FROM %s`, tableName)) - } - return bounds, err -} - -// returns the tail epochs (those must be removed from rolling) for a given intendedHeadEpoch for a given rolling table -// fE a tail epoch for rolling 1 day aggregation (225 epochs) for boundsStart 0 (start epoch of last rolling export) and intendedHeadEpoch 227 on ethereum would correspond to a tail range of 0 - 1 -// meaning epoch [0,1] must be removed from the rolling table if you want to add epoch 227 -// first argument (start bound) is inclusive, second arg (end bound) is exclusive -func (d *RollingAggregator) getTailBoundsXDays(days int, boundsStart uint64, intendedHeadEpoch uint64) (int64, int64) { - aggTailEpochStart := int64(boundsStart) // current bounds start must be removed - aggTailEpochEnd := int64(intendedHeadEpoch-utils.EpochsPerDay()*uint64(days)) + 1 - d.log.Debugf("tail bounds for %dd: %d - %d | intendedHead: %v | boundsStart: %v", days, aggTailEpochStart, aggTailEpochEnd, intendedHeadEpoch, boundsStart) - - return aggTailEpochStart, aggTailEpochEnd -} - -// Note that currentEpochHead is the current exported epoch in the db -func (d *RollingAggregator) Aggregate(days int, tableName string, currentEpochHead uint64) error { - return d.aggregateInternal(days, tableName, currentEpochHead, debugForceBootstrapRollingTables) -} - -// Note that currentEpochHead is the current exported epoch in the db -func (d *RollingAggregator) aggregateInternal(days int, tableName string, currentEpochHead uint64, forceBootstrap bool) error { - tx, err := db.AlloyWriter.Beginx() - if err != nil { - return errors.Wrap(err, "failed to start transaction") - } - defer utils.Rollback(tx) - - bootstrap := false - if forceBootstrap { - bootstrap = true - d.log.Infof("force bootstrap rolling %dd", days) - } - - // get epoch boundaries for current stored rolling 24h - bounds, err := d.getCurrentRollingBounds(tx, tableName) - if err != nil { - if err == sql.ErrNoRows { - bootstrap = true - log.Infof("bootstraping rolling %dd due to empty table", days) - } else { - return errors.Wrap(err, "failed to get current rolling bounds") - } - } - - // if current stored rolling table is far behind, bootstrap again - // in this case far means more than what we aggregate in the hour table, meaning a bootstrap - // will get faster to head then re-exporting amount of getBootstrapOnEpochsBehind() old epochs - if currentEpochHead+1-bounds.EpochEnd >= d.getBootstrapOnEpochsBehind() { // EpochEnd is excl so +1 to get the inclusive epoch number - d.log.Infof("currentEpochHead: %d, bounds.EpochEnd: %d, getBootstrapOnEpochsBehind(): %d, leftsum: %d", currentEpochHead, bounds.EpochEnd, d.getBootstrapOnEpochsBehind(), currentEpochHead+1-bounds.EpochEnd) - bootstrap = true - } - - if bootstrap { - metrics.Tasks.WithLabelValues(fmt.Sprintf("exporter_v2dash_agg_bootstrap_%dd", days)).Inc() - d.log.Infof("rolling %dd bootstraping starting", days) - - err = d.bootstrap(tx, days, tableName) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("failed to bootstrap rolling %dd aggregate", days)) - } - - bounds, err = d.getCurrentRollingBounds(tx, tableName) - if err != nil { - return errors.Wrap(err, "failed to get current rolling bounds") - } - - d.log.Infof("rolling %dd bootstraping finished, currentHead: %v | bounds: %v | Epochs Per Day: %v", days, currentEpochHead, bounds, utils.EpochsPerDay()) - - // if rolling bounds are exactly what they should be, we are done here - if currentEpochHead == bounds.EpochEnd-1 && bounds.EpochEnd-uint64(days)*utils.EpochsPerDay() == bounds.EpochStart { - log.Infof("rolling %dd is up to date, nothing to do", days) // perfect bounds after bootstrap, lucky day, done here - err = tx.Commit() - if err != nil { - return errors.Wrap(err, "failed to commit transaction") - } - return nil - } - } - - if !bootstrap && bounds.EpochEnd-bounds.EpochStart != utils.EpochsPerDay()*uint64(days) { - log.Warnf("rolling %dd boundaries are out of bounds (%d-%d, %d), this is expected after bootstrap, but not after that. Keep an eye on it", days, bounds.EpochStart, bounds.EpochEnd, bounds.EpochEnd-bounds.EpochStart) - } - - // bounds for what to aggregate and add to the head of the rolling table - aggHeadEpochStart := bounds.EpochEnd - aggHeadEpochEnd := currentEpochHead + 1 - - // bounds for what to aggregate and remove from the tail of the rolling table - aggTailEpochStart, aggTailEpochEnd := d.getTailBoundsXDays(days, bounds.EpochStart, currentEpochHead) - d.log.Infof("rolling %dd epochs: %d - %d, %d - %d", days, aggHeadEpochStart, aggHeadEpochEnd, aggTailEpochStart, aggTailEpochEnd) - - // sanity check if all tail epochs are present in db - missing, err := edb.GetMissingEpochsBetween(aggTailEpochStart, aggTailEpochEnd) - if err != nil { - return errors.Wrap(err, "failed to get missing tail epochs") - } - if len(missing) > 0 { - // If exporter falls back on head around the bootstrap bound end, it will start backfill - // and will trigger a rolling aggregate on bound end. But it will not bootstrap since recent data is - // maybe just a few epochs old. So it will try to add to head and remove from tail but since we are in - // bootstrap mode tail epochs won't be there. - // Now when this occurs and epochs are missing AND the current head is a bootstrap end, we force a bootstrap to solve this. - - // a bool helper to indicate whether currentEpochHead is the end of a bootstrap bound - _, utcEndBound := getDayAggregateBounds(currentEpochHead) - isCurrentEpochHeadBootstrapBound := utcEndBound-1 == currentEpochHead - - if isCurrentEpochHeadBootstrapBound { - return d.aggregateInternal(days, tableName, currentEpochHead, true) - } - return errors.New(fmt.Sprintf("missing epochs in db for rolling %dd tail: %v", days, missing)) - } - - // sanity check if all head epochs are present in db - missingHead, err := edb.GetMissingEpochsBetween(int64(aggHeadEpochStart), int64(aggHeadEpochEnd)) - if err != nil { - return errors.Wrap(err, "failed to get missing head epochs") - } - if len(missingHead) > 0 { - return errors.New(fmt.Sprintf("missing epochs in db for rolling %dd head: %v", days, missingHead)) - } - - // add head and fix/remove from tail - err = d.aggregateRolling(tx, tableName, aggHeadEpochStart, aggHeadEpochEnd, aggTailEpochStart, aggTailEpochEnd) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("failed to aggregate rolling %dd", days)) - } - - // Sanity check, get bounds again after rolling aggregate - sanityBounds, err := d.getCurrentRollingBounds(tx, tableName) - if err != nil { - return errors.Wrap(err, "failed to get current rolling bounds for sanity check") - } - - // skip sanity check - if sanityBounds.EpochEnd-sanityBounds.EpochStart != utils.EpochsPerDay()*uint64(days) { - // only do sanity check if tail bounds are not negative (we store them as uint in the db so it will never be utils.EpochsPerDay()*uint64(days) in this case - if aggTailEpochStart >= 0 && aggTailEpochEnd >= 0 { - return errors.New(fmt.Sprintf("sanity check failed, rolling boundaries are out of bounds for %vd agg (%d-%d, %d)", days, sanityBounds.EpochStart, sanityBounds.EpochEnd, sanityBounds.EpochEnd-sanityBounds.EpochStart)) - } - } - - metrics.State.WithLabelValues(fmt.Sprintf("exporter_v2dash_rolling_%dd_bounds_end", days)).Set(float64(sanityBounds.EpochEnd)) - - err = tx.Commit() - if err != nil { - return errors.Wrap(err, "failed to commit transaction") - } - - return nil -} - -func (d *RollingAggregator) getMissingRollingTailEpochs(days int, intendedHeadEpoch uint64, tableName string) ([]uint64, error) { - bounds, err := d.getCurrentRollingBounds(nil, tableName) - if err != nil { - if err != sql.ErrNoRows { - return nil, errors.Wrap(err, fmt.Sprintf("failed to get latest exported rolling %dd bounds", days)) - } - } - - needsBootstrap := debugForceBootstrapRollingTables || int64(intendedHeadEpoch-bounds.EpochEnd) >= int64(d.getBootstrapOnEpochsBehind()) - - d.log.Infof("%dd needs bootstrap: %v", days, needsBootstrap) - // if rolling table is empty / not bootstrapped yet or needs a bootstrap assume bounds of what the would be after a bootstrap - if (bounds.EpochEnd == 0 && bounds.EpochStart == 0) || needsBootstrap { - // assume bounds after bootstrap - startBound, endBounds := d.getBootstrapBounds(intendedHeadEpoch, uint64(days)) - bounds.EpochStart = startBound - bounds.EpochEnd = endBounds - d.log.Infof("bootstrap bounds for rolling %dd: %d - %d | current Head (excl): %v", days, bounds.EpochStart, bounds.EpochEnd, intendedHeadEpoch+1) - } - - aggTailEpochStart, aggTailEpochEnd := d.getTailBoundsXDays(days, bounds.EpochStart, intendedHeadEpoch) - - return edb.GetMissingEpochsBetween(aggTailEpochStart, aggTailEpochEnd) -} - -// Adds the new epochs (headEpochStart to headEpochEnd) to the rolling table and removes the old ones (tailEpochStart to tailEpochEnd) -// start params are inclusive, end params are exclusive -func (d *RollingAggregator) aggregateRolling(tx *sqlx.Tx, tableName string, headEpochStart, headEpochEnd uint64, tailEpochStart, tailEpochEnd int64) error { - d.log.Infof("aggregating rolling %s epochs: %d - %d, %d - %d", tableName, headEpochStart, headEpochEnd, tailEpochStart, tailEpochEnd) - defer d.log.Infof("aggregated rolling %s epochs: %d - %d, %d - %d", tableName, headEpochStart, headEpochEnd, tailEpochStart, tailEpochEnd) - - // Important to remove first since head could contain validators that were not present in tail, would interfere with boundaries - if tailEpochEnd >= tailEpochStart { - err := d.removeFromRolling(tx, tableName, tailEpochStart, tailEpochEnd) - if err != nil { - return errors.Wrap(err, "failed to remove epochs from rolling") - } - } - if headEpochEnd >= headEpochStart { - err := d.addToRolling(tx, tableName, headEpochStart, headEpochEnd, tailEpochEnd) - if err != nil { - return errors.Wrap(err, "failed to add epochs to rolling") - } - } - - return nil -} - -// Inserts new validators or updated existing ones into the rolling table -// startEpoch incl | endEpoch, tailEnd excl -func (d *RollingAggregator) addToRolling(tx *sqlx.Tx, tableName string, startEpoch, endEpoch uint64, tailEnd int64) error { - startTime := time.Now() - d.log.Infof("add to rolling %s epochs: %d - %d", tableName, startEpoch, endEpoch) - defer func() { - d.log.Infof("added to rolling %s took %v", tableName, time.Since(startTime)) - }() - - if tailEnd < 0 { - tailEnd = 0 - } - - return AddToRollingCustom(tx, CustomRolling{ - StartEpoch: startEpoch, - EndEpoch: endEpoch, - StartBoundEpoch: tailEnd, - TableFrom: edb.EpochWriterTableName, - TableTo: tableName, - TableFromEpochColumn: "epoch", - Log: d.log, - TableConflict: "(validator_index)", - }) -} - -type CustomRolling struct { - Log ModuleLog // for logging, must provide - StartEpoch uint64 // incl, must be provided - EndEpoch uint64 // excl, must be provided - StartBoundEpoch int64 // incl, must be provided - TableFrom string // must provide - TableTo string // must provide - TableFromEpochColumn string // must provide - TableConflict string // must provide - - TailBalancesQuery string // optional - TailBalancesJoinQuery string // optional - TailBalancesInsertColumnQuery string // optional - TableDayColum string // optional - TableDayValue string // optional - - Agg TableAGG // do not provide, will be overwritten -} - -type TableAGG struct { - SUM string - BOOL_OR string - MAX string - AGG_END string - WHERE_AND_GROUP string - BALANCE_END_Q string - BALANCE_END_COLUMN string - HEAD_BALANCE_QUERY string - HEAD_BALANCE_JOIN string - HEAD_BALANCE_SINGLE_EPOCH_COLUMN string -} - -// This method is the bread and butter of all aggregation. It is used by rolling window aggregation to add to head, -// it is used by total to add to head, it is used by utc day and hour aggregation to add to head -func AddToRollingCustom(tx *sqlx.Tx, custom CustomRolling) error { - if custom.TailBalancesInsertColumnQuery == "" { - custom.TailBalancesInsertColumnQuery = "null," - } - - if custom.StartEpoch > custom.EndEpoch-1 { - custom.Log.Infof("nothing to do, start epoch is greater than end epoch (%d > %d)", custom.StartEpoch, custom.EndEpoch-1) - return nil - } - - // When aggregating multiple epochs (aggregate mode) - custom.Agg = TableAGG{ - SUM: "SUM(", - BOOL_OR: "bool_or(", - MAX: "MAX(", - AGG_END: ")", - - // What epochs to select for aggregate - WHERE_AND_GROUP: fmt.Sprintf(` - WHERE %[1]s >= $1 AND %[1]s < $2 - GROUP BY validator_index - `, custom.TableFromEpochColumn), - - // Head balance from the most recent epoch so select and join the agg set - HEAD_BALANCE_QUERY: fmt.Sprintf(` - head_balance_ends as ( - SELECT validator_index, balance_end FROM %[1]s WHERE %[2]s = $2 -1 - ), - `, custom.TableFrom, custom.TableFromEpochColumn), - HEAD_BALANCE_JOIN: ` - LEFT JOIN head_balance_ends ON aggregate_head.validator_index = head_balance_ends.validator_index - `, - HEAD_BALANCE_SINGLE_EPOCH_COLUMN: "", // since data is provided by join we dont need to select it - } - - // I know how this looks but the query planner of postgres has difficulty with the agg group by for just one epoch - // So here some optimization for when we only aggregate one epoch - // just select, no aggregate, grouping or join needed - if custom.StartEpoch == custom.EndEpoch-1 { - custom.Agg = TableAGG{ - SUM: "", - BOOL_OR: "", - MAX: "", - AGG_END: "", - WHERE_AND_GROUP: fmt.Sprintf(`WHERE %s = $1`, custom.TableFromEpochColumn), - - // For head balance with just one epoch just use the column, forget about the join - HEAD_BALANCE_SINGLE_EPOCH_COLUMN: "balance_end,", - } - - // If start bound is the same as start and end, we can ommit that join altogether - // postgres really doesnt like joining on the same epoch - if custom.StartBoundEpoch == int64(custom.StartEpoch) && custom.TailBalancesQuery != "" { - custom.TailBalancesQuery = "" - custom.TailBalancesJoinQuery = "" - custom.Agg.HEAD_BALANCE_SINGLE_EPOCH_COLUMN += custom.TailBalancesInsertColumnQuery - } - } - - // Use NULLIF(x,0) to save storage space - tmpl := ` - -- Agg Add, From: {{ .TableFrom }}, To: {{ .TableTo }} - WITH - {{ .Agg.HEAD_BALANCE_QUERY }} - {{ .TailBalancesQuery }} -- balance start query - aggregate_head as ( - SELECT - validator_index, - {{ .Agg.HEAD_BALANCE_SINGLE_EPOCH_COLUMN }} - {{ .Agg.SUM }}attestations_source_reward{{ .Agg.AGG_END }} as attestations_source_reward, - {{ .Agg.SUM }}attestations_target_reward{{ .Agg.AGG_END }} as attestations_target_reward, - {{ .Agg.SUM }}attestations_head_reward{{ .Agg.AGG_END }} as attestations_head_reward, - {{ .Agg.SUM }}attestations_inactivity_reward{{ .Agg.AGG_END }} as attestations_inactivity_reward, - {{ .Agg.SUM }}attestations_inclusion_reward{{ .Agg.AGG_END }} as attestations_inclusion_reward, - {{ .Agg.SUM }}attestations_reward{{ .Agg.AGG_END }} as attestations_reward, - {{ .Agg.SUM }}attestations_ideal_source_reward{{ .Agg.AGG_END }} as attestations_ideal_source_reward, - {{ .Agg.SUM }}attestations_ideal_target_reward{{ .Agg.AGG_END }} as attestations_ideal_target_reward, - {{ .Agg.SUM }}attestations_ideal_head_reward{{ .Agg.AGG_END }} as attestations_ideal_head_reward, - {{ .Agg.SUM }}attestations_ideal_inactivity_reward{{ .Agg.AGG_END }} as attestations_ideal_inactivity_reward, - {{ .Agg.SUM }}attestations_ideal_inclusion_reward{{ .Agg.AGG_END }} as attestations_ideal_inclusion_reward, - {{ .Agg.SUM }}attestations_ideal_reward{{ .Agg.AGG_END }} as attestations_ideal_reward, - {{ .Agg.SUM }}blocks_scheduled{{ .Agg.AGG_END }} as blocks_scheduled, - {{ .Agg.SUM }}blocks_proposed{{ .Agg.AGG_END }} as blocks_proposed, - {{ .Agg.SUM }}blocks_cl_reward{{ .Agg.AGG_END }} as blocks_cl_reward, - {{ .Agg.SUM }}blocks_cl_attestations_reward{{ .Agg.AGG_END }} as blocks_cl_attestations_reward, - {{ .Agg.SUM }}blocks_cl_sync_aggregate_reward{{ .Agg.AGG_END }} as blocks_cl_sync_aggregate_reward, - {{ .Agg.SUM }}sync_scheduled{{ .Agg.AGG_END }} as sync_scheduled, - {{ .Agg.SUM }}sync_executed{{ .Agg.AGG_END }} as sync_executed, - {{ .Agg.SUM }}sync_reward{{ .Agg.AGG_END }} as sync_reward, - {{ .Agg.BOOL_OR }}slashed{{ .Agg.AGG_END }} as slashed, - {{ .Agg.SUM }}deposits_count{{ .Agg.AGG_END }} as deposits_count, - {{ .Agg.SUM }}deposits_amount{{ .Agg.AGG_END }} as deposits_amount, - {{ .Agg.SUM }}withdrawals_count{{ .Agg.AGG_END }} as withdrawals_count, - {{ .Agg.SUM }}withdrawals_amount{{ .Agg.AGG_END }} as withdrawals_amount, - {{ .Agg.SUM }}inclusion_delay_sum{{ .Agg.AGG_END }} as inclusion_delay_sum, - {{ .Agg.SUM }}blocks_expected{{ .Agg.AGG_END }} as blocks_expected, - {{ .Agg.SUM }}sync_committees_expected{{ .Agg.AGG_END }} as sync_committees_expected, - {{ .Agg.SUM }}attestations_scheduled{{ .Agg.AGG_END }} as attestations_scheduled, - {{ .Agg.SUM }}attestations_observed{{ .Agg.AGG_END }} as attestations_observed, - {{ .Agg.SUM }}attestations_head_executed{{ .Agg.AGG_END }} as attestations_head_executed, - {{ .Agg.SUM }}attestations_source_executed{{ .Agg.AGG_END }} as attestations_source_executed, - {{ .Agg.SUM }}attestations_target_executed{{ .Agg.AGG_END }} as attestations_target_executed, - {{ .Agg.SUM }}optimal_inclusion_delay_sum{{ .Agg.AGG_END }} as optimal_inclusion_delay_sum, - {{ .Agg.SUM }}slasher_reward{{ .Agg.AGG_END }} as slasher_reward, - {{ .Agg.MAX }}slashed_by{{ .Agg.AGG_END }} as slashed_by, - {{ .Agg.MAX }}slashed_violation{{ .Agg.AGG_END }} as slashed_violation, - {{ .Agg.MAX }}last_executed_duty_epoch{{ .Agg.AGG_END }} as last_executed_duty_epoch - FROM {{ .TableFrom }} - {{ .Agg.WHERE_AND_GROUP }} - ) - INSERT INTO {{ .TableTo }} ( - {{ .TableDayColum }} - epoch_end, - epoch_start, - validator_index, - attestations_source_reward, - attestations_target_reward, - attestations_head_reward, - attestations_inactivity_reward, - attestations_inclusion_reward, - attestations_reward, - attestations_ideal_source_reward, - attestations_ideal_target_reward, - attestations_ideal_head_reward, - attestations_ideal_inactivity_reward, - attestations_ideal_inclusion_reward, - attestations_ideal_reward, - blocks_scheduled, - blocks_proposed, - blocks_cl_reward, - blocks_cl_attestations_reward, - blocks_cl_sync_aggregate_reward, - sync_scheduled, - sync_executed, - sync_reward, - slashed, - balance_end, - balance_start, - deposits_count, - deposits_amount, - withdrawals_count, - withdrawals_amount, - inclusion_delay_sum, - blocks_expected, - sync_committees_expected, - attestations_scheduled, - attestations_observed, - attestations_head_executed, - attestations_source_executed, - attestations_target_executed, - optimal_inclusion_delay_sum, - slasher_reward, - slashed_by, - slashed_violation, - last_executed_duty_epoch - ) - SELECT - {{ .TableDayValue }} - $2 as epoch_end, -- exclusive - $3 as epoch_start, -- inclusive, only write on insert - do not update in UPDATE part! Use tail start epoch - aggregate_head.validator_index as validator_index, - COALESCE(aggregate_head.attestations_source_reward, 0) as attestations_source_reward, - COALESCE(aggregate_head.attestations_target_reward, 0) as attestations_target_reward, - COALESCE(aggregate_head.attestations_head_reward, 0) as attestations_head_reward, - COALESCE(aggregate_head.attestations_inactivity_reward, 0) as attestations_inactivity_reward, - COALESCE(aggregate_head.attestations_inclusion_reward, 0) as attestations_inclusion_reward, - COALESCE(aggregate_head.attestations_reward, 0) as attestations_reward, - COALESCE(aggregate_head.attestations_ideal_source_reward, 0) as attestations_ideal_source_reward, - COALESCE(aggregate_head.attestations_ideal_target_reward, 0) as attestations_ideal_target_reward, - COALESCE(aggregate_head.attestations_ideal_head_reward, 0) as attestations_ideal_head_reward, - COALESCE(aggregate_head.attestations_ideal_inactivity_reward, 0) as attestations_ideal_inactivity_reward, - COALESCE(aggregate_head.attestations_ideal_inclusion_reward, 0) as attestations_ideal_inclusion_reward, - COALESCE(aggregate_head.attestations_ideal_reward, 0) as attestations_ideal_reward, - COALESCE(aggregate_head.blocks_scheduled, 0) as blocks_scheduled, - COALESCE(aggregate_head.blocks_proposed, 0) as blocks_proposed, - COALESCE(aggregate_head.blocks_cl_reward, 0) as blocks_cl_reward, - COALESCE(aggregate_head.blocks_cl_attestations_reward, 0) as blocks_cl_attestations_reward, - COALESCE(aggregate_head.blocks_cl_sync_aggregate_reward, 0) as blocks_cl_sync_aggregate_reward, - COALESCE(aggregate_head.sync_scheduled, 0) as sync_scheduled, - COALESCE(aggregate_head.sync_executed, 0) as sync_executed, - COALESCE(aggregate_head.sync_reward, 0) as , - aggregate_head.slashed, - balance_end, - {{ .TailBalancesInsertColumnQuery }} -- balance_start - COALESCE(aggregate_head.deposits_count, 0) as deposits_count, - COALESCE(aggregate_head.deposits_amount, 0) as deposits_amount, - COALESCE(aggregate_head.withdrawals_count, 0) as withdrawals_count, - COALESCE(aggregate_head.withdrawals_amount, 0) as withdrawals_amount, - COALESCE(aggregate_head.inclusion_delay_sum, 0) as inclusion_delay_sum, - COALESCE(aggregate_head.blocks_expected, 0) as blocks_expected, - COALESCE(aggregate_head.sync_committees_expected, 0) as sync_committees_expected, - COALESCE(aggregate_head.attestations_scheduled, 0) as attestations_scheduled, - COALESCE(aggregate_head.attestations_observed, 0) as attestations_observed, - COALESCE(aggregate_head.attestations_head_executed, 0) as attestations_head_executed, - COALESCE(aggregate_head.attestations_source_executed, 0) as attestations_source_executed, - COALESCE(aggregate_head.attestations_target_executed, 0) as attestations_target_executed, - COALESCE(aggregate_head.optimal_inclusion_delay_sum, 0) as optimal_inclusion_delay_sum, - COALESCE(aggregate_head.slasher_reward, 0) as slasher_reward, - aggregate_head.slashed_by, - aggregate_head.slashed_violation, - aggregate_head.last_executed_duty_epoch - FROM aggregate_head - {{ .TailBalancesJoinQuery }} -- balance start join - {{ .Agg.HEAD_BALANCE_JOIN }} - ON CONFLICT {{ .TableConflict }} DO UPDATE SET - attestations_source_reward = NULLIF(COALESCE({{ .TableTo }}.attestations_source_reward, 0) + EXCLUDED.attestations_source_reward, 0), - attestations_target_reward = NULLIF(COALESCE({{ .TableTo }}.attestations_target_reward, 0) + EXCLUDED.attestations_target_reward, 0), - attestations_head_reward = NULLIF(COALESCE({{ .TableTo }}.attestations_head_reward, 0) + EXCLUDED.attestations_head_reward, 0), - attestations_inactivity_reward = NULLIF(COALESCE({{ .TableTo }}.attestations_inactivity_reward, 0) + EXCLUDED.attestations_inactivity_reward, 0), - attestations_inclusion_reward = NULLIF(COALESCE({{ .TableTo }}.attestations_inclusion_reward, 0) + EXCLUDED.attestations_inclusion_reward, 0), - attestations_reward = NULLIF(COALESCE({{ .TableTo }}.attestations_reward, 0) + EXCLUDED.attestations_reward, 0), - attestations_ideal_source_reward = NULLIF(COALESCE({{ .TableTo }}.attestations_ideal_source_reward, 0) + EXCLUDED.attestations_ideal_source_reward, 0), - attestations_ideal_target_reward = NULLIF(COALESCE({{ .TableTo }}.attestations_ideal_target_reward, 0) + EXCLUDED.attestations_ideal_target_reward, 0), - attestations_ideal_head_reward = NULLIF(COALESCE({{ .TableTo }}.attestations_ideal_head_reward, 0) + EXCLUDED.attestations_ideal_head_reward, 0), - attestations_ideal_inactivity_reward = NULLIF(COALESCE({{ .TableTo }}.attestations_ideal_inactivity_reward, 0) + EXCLUDED.attestations_ideal_inactivity_reward, 0), - attestations_ideal_inclusion_reward = NULLIF(COALESCE({{ .TableTo }}.attestations_ideal_inclusion_reward, 0) + EXCLUDED.attestations_ideal_inclusion_reward, 0), - attestations_ideal_reward = NULLIF(COALESCE({{ .TableTo }}.attestations_ideal_reward, 0) + EXCLUDED.attestations_ideal_reward, 0), - blocks_scheduled = NULLIF(COALESCE({{ .TableTo }}.blocks_scheduled, 0) + EXCLUDED.blocks_scheduled, 0), - blocks_proposed = NULLIF(COALESCE({{ .TableTo }}.blocks_proposed, 0) + EXCLUDED.blocks_proposed, 0), - blocks_cl_reward = NULLIF(COALESCE({{ .TableTo }}.blocks_cl_reward, 0) + EXCLUDED.blocks_cl_reward, 0), - blocks_cl_attestations_reward = NULLIF(COALESCE({{ .TableTo }}.blocks_cl_attestations_reward, 0) + EXCLUDED.blocks_cl_attestations_reward, 0), - blocks_cl_sync_aggregate_reward = NULLIF(COALESCE({{ .TableTo }}.blocks_cl_sync_aggregate_reward, 0) + EXCLUDED.blocks_cl_sync_aggregate_reward, 0), - sync_scheduled = NULLIF(COALESCE({{ .TableTo }}.sync_scheduled, 0) + EXCLUDED.sync_scheduled, 0), - sync_executed = NULLIF(COALESCE({{ .TableTo }}.sync_executed, 0) + EXCLUDED.sync_executed, 0), - sync_reward = NULLIF(COALESCE({{ .TableTo }}.sync_reward, 0) + EXCLUDED.sync_reward, 0), - slashed = EXCLUDED.slashed OR {{ .TableTo }}.slashed, - balance_end = COALESCE(EXCLUDED.balance_end, {{ .TableTo }}.balance_end), - deposits_count = NULLIF(COALESCE({{ .TableTo }}.deposits_count, 0) + EXCLUDED.deposits_count, 0), - deposits_amount = NULLIF(COALESCE({{ .TableTo }}.deposits_amount, 0) + EXCLUDED.deposits_amount, 0), - withdrawals_count = NULLIF(COALESCE({{ .TableTo }}.withdrawals_count, 0) + EXCLUDED.withdrawals_count, 0), - withdrawals_amount = NULLIF(COALESCE({{ .TableTo }}.withdrawals_amount, 0) + EXCLUDED.withdrawals_amount, 0), - inclusion_delay_sum = NULLIF(COALESCE({{ .TableTo }}.inclusion_delay_sum, 0) + EXCLUDED.inclusion_delay_sum, 0), - blocks_expected = NULLIF(COALESCE({{ .TableTo }}.blocks_expected, 0) + EXCLUDED.blocks_expected, 0), - sync_committees_expected = NULLIF(COALESCE({{ .TableTo }}.sync_committees_expected, 0) + EXCLUDED.sync_committees_expected, 0), - attestations_scheduled = NULLIF(COALESCE({{ .TableTo }}.attestations_scheduled, 0) + EXCLUDED.attestations_scheduled, 0), - attestations_observed = NULLIF(COALESCE({{ .TableTo }}.attestations_observed, 0) + EXCLUDED.attestations_observed, 0), - attestations_head_executed = NULLIF(COALESCE({{ .TableTo }}.attestations_head_executed, 0) + EXCLUDED.attestations_head_executed, 0), - attestations_source_executed = NULLIF(COALESCE({{ .TableTo }}.attestations_source_executed, 0) + EXCLUDED.attestations_source_executed, 0), - attestations_target_executed = NULLIF(COALESCE({{ .TableTo }}.attestation_target_executed, 0) + EXCLUDED.attestations_target_executed, 0), - optimal_inclusion_delay_sum = NULLIF(COALESCE({{ .TableTo }}.optimal_inclusion_delay_sum, 0) + EXCLUDED.optimal_inclusion_delay_sum, 0), - epoch_end = EXCLUDED.epoch_end, - slasher_reward = NULLIF(COALESCE({{ .TableTo }}.slasher_reward, 0) + EXCLUDED.slasher_reward, 0), - slashed_by = COALESCE(EXCLUDED.slashed_by, {{ .TableTo }}.slashed_by), - slashed_violation = COALESCE(EXCLUDED.slashed_violation, {{ .TableTo }}.slashed_violation), - last_executed_duty_epoch = COALESCE(EXCLUDED.last_executed_duty_epoch, {{ .TableTo }}.last_executed_duty_epoch)` - - t := template.Must(template.New("tmpl").Parse(tmpl)) - var queryBuffer bytes.Buffer - if err := t.Execute(&queryBuffer, custom); err != nil { - return errors.Wrap(err, "failed to execute template") - } - - custom.Log.Debugf("TableTo: %v | TableFrom: %v | StartEpoch: %v | EndEpoch: %v | StartBoundEpoch: %v", custom.TableTo, custom.TableFrom, custom.StartEpoch, custom.EndEpoch, custom.StartBoundEpoch) - - timeout := 60 * time.Minute - if debugDeadlockBandaid { - timeout = 35 * time.Minute - } - - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - resultChan := make(chan sql.Result, 1) - errChan := make(chan error, 1) - - go func() { - result, err := tx.ExecContext(ctx, queryBuffer.String(), - custom.StartEpoch, custom.EndEpoch, custom.StartBoundEpoch, - ) - if err != nil { - errChan <- err - return - } - resultChan <- result - }() - - select { - case result := <-resultChan: - rowsAffected, err := result.RowsAffected() - if err != nil { - return errors.Wrap(err, "failed to get rows affected") - } - - custom.Log.Infof("updated %s, affected %d rows", custom.TableTo, rowsAffected) - if rowsAffected == 0 { - custom.Log.Infof("no rows affected, nothing to update for %s", custom.TableTo) - } - - return nil - case err := <-errChan: - if err != nil { - return errors.Wrap(err, "failed to update rolling table") - } - case <-ctx.Done(): - // Query took longer than x minutes, cancel and return error - cancel() - metrics.Errors.WithLabelValues("exporter_v2dash_bandaid").Inc() - if debugDeadlockBandaid { - _, err := db.AlloyWriter.Exec(`SELECT pg_cancel_backend(pid) - FROM pg_stat_activity - WHERE pid = ANY(pg_blocking_pids(pg_backend_pid())) - AND state = 'active' - AND query_start < now() - interval '10 minutes' - AND query LIKE '%validator_dashboard%';`) - if err != nil { - return errors.Wrap(err, "failed to kill backend process") - } - } - return errors.New(fmt.Sprintf("query took longer than %d minutes, canceled and backend process canceled", int(timeout.Minutes()))) - } - - return nil -} - -// startEpoch incl, endEpoch excl -func (d *RollingAggregator) removeFromRolling(tx *sqlx.Tx, tableName string, startEpoch, endEpoch int64) error { - startTime := time.Now() - d.log.Infof("remove from rolling %s epochs: %d - %d ", tableName, startEpoch, endEpoch) - defer func() { - d.log.Infof("removed from rolling %s took %v", tableName, time.Since(startTime)) - }() - - if endEpoch-1 < 0 { - // if selected time frame is more than epochs exists we log an info - d.log.Infof("rolling %sd tail epoch is negative, no end cutting", tableName) - endEpoch = 0 // since its inclusive make it -1 so it stored 0 in table - } - - result, err := tx.Exec(fmt.Sprintf(` - WITH - footer_balance_starts as ( - SELECT validator_index, balance_end as balance_start FROM %[2]s WHERE epoch = $2 -1 -- end balance of epoch we want to remove = start epoch of epoch we start from - ), - aggregate_tail as ( - SELECT - validator_index, - SUM(attestations_source_reward) as attestations_source_reward, - SUM(attestations_target_reward) as attestations_target_reward, - SUM(attestations_head_reward) as attestations_head_reward, - SUM(attestations_inactivity_reward) as attestations_inactivity_reward, - SUM(attestations_inclusion_reward) as attestations_inclusion_reward, - SUM(attestations_reward) as attestations_reward, - SUM(attestations_ideal_source_reward) as attestations_ideal_source_reward, - SUM(attestations_ideal_target_reward) as attestations_ideal_target_reward, - SUM(attestations_ideal_head_reward) as attestations_ideal_head_reward, - SUM(attestations_ideal_inactivity_reward) as attestations_ideal_inactivity_reward, - SUM(attestations_ideal_inclusion_reward) as attestations_ideal_inclusion_reward, - SUM(attestations_ideal_reward) as attestations_ideal_reward, - SUM(blocks_scheduled) as blocks_scheduled, - SUM(blocks_proposed) as blocks_proposed, - SUM(blocks_cl_reward) as blocks_cl_reward, - SUM(blocks_cl_attestations_reward) as blocks_cl_attestations_reward, - SUM(blocks_cl_sync_aggregate_reward) as blocks_cl_sync_aggregate_reward, - SUM(sync_scheduled) as sync_scheduled, - SUM(sync_executed) as sync_executed, - SUM(sync_reward) as sync_reward, - SUM(deposits_count) as deposits_count, - SUM(deposits_amount) as deposits_amount, - SUM(withdrawals_count) as withdrawals_count, - SUM(withdrawals_amount) as withdrawals_amount, - SUM(inclusion_delay_sum) as inclusion_delay_sum, - SUM(blocks_expected) as blocks_expected, - SUM(sync_committees_expected) as sync_committees_expected, - SUM(attestations_scheduled) as attestations_scheduled, - SUM(attestations_observed) as attestations_observed, - SUM(attestations_head_executed) as attestations_head_executed, - SUM(attestations_source_executed) as attestations_source_executed, - SUM(attestations_target_executed) as attestations_target_executed, - SUM(optimal_inclusion_delay_sum) as optimal_inclusion_delay_sum, - SUM(slasher_reward) as slasher_reward, - MAX(last_executed_duty_epoch) as last_executed_duty_epoch - FROM %[2]s - WHERE epoch >= $1 AND epoch < $2 - GROUP BY validator_index - ), - result as ( - SELECT - $2 as epoch_start, --since its inclusive in the func $2 will be deducted hence +1 - aggregate_tail.validator_index as validator_index, - COALESCE(aggregate_tail.attestations_source_reward, 0) as attestations_source_reward, - COALESCE(aggregate_tail.attestations_target_reward, 0) as attestations_target_reward, - COALESCE(aggregate_tail.attestations_head_reward, 0) as attestations_head_reward, - COALESCE(aggregate_tail.attestations_inactivity_reward, 0) as attestations_inactivity_reward, - COALESCE(aggregate_tail.attestations_inclusion_reward, 0) as attestations_inclusion_reward, - COALESCE(aggregate_tail.attestations_reward, 0) as attestations_reward, - COALESCE(aggregate_tail.attestations_ideal_source_reward, 0) as attestations_ideal_source_reward, - COALESCE(aggregate_tail.attestations_ideal_target_reward, 0) as attestations_ideal_target_reward, - COALESCE(aggregate_tail.attestations_ideal_head_reward, 0) as attestations_ideal_head_reward, - COALESCE(aggregate_tail.attestations_ideal_inactivity_reward, 0) as attestations_ideal_inactivity_reward, - COALESCE(aggregate_tail.attestations_ideal_inclusion_reward, 0) as attestations_ideal_inclusion_reward, - COALESCE(aggregate_tail.attestations_ideal_reward, 0) as attestations_ideal_reward, - COALESCE(aggregate_tail.blocks_scheduled, 0) as blocks_scheduled, - COALESCE(aggregate_tail.blocks_proposed, 0) as blocks_proposed, - COALESCE(aggregate_tail.blocks_cl_reward, 0) as blocks_cl_reward, - COALESCE(aggregate_tail.blocks_cl_attestations_reward, 0) as blocks_cl_attestations_reward, - COALESCE(aggregate_tail.blocks_cl_sync_aggregate_reward, 0) as blocks_cl_sync_aggregate_reward, - COALESCE(aggregate_tail.sync_scheduled, 0) as sync_scheduled, - COALESCE(aggregate_tail.sync_executed, 0) as sync_executed, - COALESCE(aggregate_tail.sync_reward, 0) as sync_reward, - balance_start, - COALESCE(aggregate_tail.deposits_count, 0) as deposits_count, - COALESCE(aggregate_tail.deposits_amount, 0) as deposits_amount, - COALESCE(aggregate_tail.withdrawals_count, 0) as withdrawals_count, - COALESCE(aggregate_tail.withdrawals_amount, 0) as withdrawals_amount, - COALESCE(aggregate_tail.inclusion_delay_sum, 0) as inclusion_delay_sum, - COALESCE(aggregate_tail.blocks_expected, 0) as blocks_expected, - COALESCE(aggregate_tail.sync_committees_expected, 0) as sync_committees_expected, - COALESCE(aggregate_tail.attestations_scheduled, 0) as attestations_scheduled, - COALESCE(aggregate_tail.attestations_observed, 0) as attestations_observed, - COALESCE(aggregate_tail.attestations_head_executed, 0) as attestations_head_executed, - COALESCE(aggregate_tail.attestations_source_executed, 0) as attestations_source_executed, - COALESCE(aggregate_tail.attestations_target_executed, 0) as attestations_target_executed, - COALESCE(aggregate_tail.optimal_inclusion_delay_sum, 0) as optimal_inclusion_delay_sum, - COALESCE(aggregate_tail.slasher_reward, 0) as slasher_reward, - aggregate_tail.last_executed_duty_epoch - FROM aggregate_tail - LEFT JOIN footer_balance_starts ON aggregate_tail.validator_index = footer_balance_starts.validator_index - ) - UPDATE %[1]s AS v SET - attestations_source_reward = COALESCE(v.attestations_source_reward, 0) - result.attestations_source_reward, - attestations_target_reward = COALESCE(v.attestations_target_reward, 0) - result.attestations_target_reward, - attestations_head_reward = COALESCE(v.attestations_head_reward, 0) - result.attestations_head_reward, - attestations_inactivity_reward = COALESCE(v.attestations_inactivity_reward, 0) - result.attestations_inactivity_reward, - attestations_inclusion_reward = COALESCE(v.attestations_inclusion_reward, 0) - result.attestations_inclusion_reward, - attestations_reward = COALESCE(v.attestations_reward, 0) - result.attestations_reward, - attestations_ideal_source_reward = COALESCE(v.attestations_ideal_source_reward, 0) - result.attestations_ideal_source_reward, - attestations_ideal_target_reward = COALESCE(v.attestations_ideal_target_reward, 0) - result.attestations_ideal_target_reward, - attestations_ideal_head_reward = COALESCE(v.attestations_ideal_head_reward, 0) - result.attestations_ideal_head_reward, - attestations_ideal_inactivity_reward = COALESCE(v.attestations_ideal_inactivity_reward, 0) - result.attestations_ideal_inactivity_reward, - attestations_ideal_inclusion_reward = COALESCE(v.attestations_ideal_inclusion_reward, 0) - result.attestations_ideal_inclusion_reward, - attestations_ideal_reward = COALESCE(v.attestations_ideal_reward, 0) - result.attestations_ideal_reward, - blocks_scheduled = COALESCE(v.blocks_scheduled, 0) - result.blocks_scheduled, - blocks_proposed = COALESCE(v.blocks_proposed, 0) - result.blocks_proposed, - blocks_cl_reward = COALESCE(v.blocks_cl_reward, 0) - result.blocks_cl_reward, - blocks_cl_attestations_reward = COALESCE(v.blocks_cl_attestations_reward, 0) - result.blocks_cl_attestations_reward, - blocks_cl_sync_aggregate_reward = COALESCE(v.blocks_cl_sync_aggregate_reward, 0) - result.blocks_cl_sync_aggregate_reward, - sync_scheduled = COALESCE(v.sync_scheduled, 0) - result.sync_scheduled, - sync_executed = COALESCE(v.sync_executed, 0) - result.sync_executed, - sync_reward = COALESCE(v.sync_reward, 0) - result.sync_reward, - balance_start = COALESCE(result.balance_start, v.balance_start), - deposits_count = COALESCE(v.deposits_count, 0) - result.deposits_count, - deposits_amount = COALESCE(v.deposits_amount, 0) - result.deposits_amount, - withdrawals_count = COALESCE(v.withdrawals_count, 0) - result.withdrawals_count, - withdrawals_amount = COALESCE(v.withdrawals_amount, 0) - result.withdrawals_amount, - inclusion_delay_sum = COALESCE(v.inclusion_delay_sum, 0) - result.inclusion_delay_sum, - blocks_expected = COALESCE(v.blocks_expected, 0) - result.blocks_expected, - sync_committees_expected = COALESCE(v.sync_committees_expected, 0) - result.sync_committees_expected, - attestations_scheduled = COALESCE(v.attestations_scheduled, 0) - result.attestations_scheduled, - attestations_observed = COALESCE(v.attestations_observed, 0) - result.attestations_observed, - attestations_head_executed = COALESCE(v.attestations_head_executed, 0) - result.attestations_head_executed, - attestations_source_executed = COALESCE(v.attestations_source_executed, 0) - result.attestations_source_executed, - attestations_target_executed = COALESCE(v.attestations_target_executed, 0) - result.attestations_target_executed, - optimal_inclusion_delay_sum = COALESCE(v.optimal_inclusion_delay_sum, 0) - result.optimal_inclusion_delay_sum, - epoch_start = result.epoch_start, - slasher_reward = COALESCE(v.slasher_reward, 0) - result.slasher_reward, - last_executed_duty_epoch = COALESCE(result.last_executed_duty_epoch, v.last_executed_duty_epoch) - FROM result - WHERE v.validator_index = result.validator_index; - - `, tableName, edb.EpochWriterTableName), startEpoch, endEpoch) - - if err != nil { - return errors.Wrap(err, "failed to update rolling table") - } - - rowsAffected, err := result.RowsAffected() - if err != nil { - return errors.Wrap(err, "failed to get rows affected") - } - - d.log.Infof("updated %s, affected %d rows", tableName, rowsAffected) - if rowsAffected == 0 { - d.log.Infof("no rows affected, nothing to update for %s", tableName) - } - - return err -} diff --git a/backend/pkg/exporter/modules/slot_exporter.go b/backend/pkg/exporter/modules/slot_exporter.go index 5b8ed2b22..bf0221265 100644 --- a/backend/pkg/exporter/modules/slot_exporter.go +++ b/backend/pkg/exporter/modules/slot_exporter.go @@ -45,7 +45,7 @@ func NewSlotExporter(moduleContext ModuleContext) ModuleInterface { } } -var latestEpoch, latestSlot, finalizedEpoch, latestProposed uint64 +var latestEpoch, latestSlot, finalizedEpoch, latestProposed uint64 // holy shit these should really be in the slotExporterData struct or some other module might accidentally corrupt them var processSlotMutex = &sync.Mutex{} diff --git a/backend/pkg/exporter/types/dashboard_data.go b/backend/pkg/exporter/types/dashboard_data.go new file mode 100644 index 000000000..fc5c6e92f --- /dev/null +++ b/backend/pkg/exporter/types/dashboard_data.go @@ -0,0 +1,247 @@ +package types + +import ( + "fmt" + "reflect" + "strconv" + "time" + + "github.com/gobitfly/beaconchain/pkg/commons/db" + "github.com/gobitfly/beaconchain/pkg/commons/log" + "github.com/google/uuid" + "golang.org/x/sync/errgroup" +) + +type VDBDataEpochColumns struct { + // this should be the same as Epoch but only once, basically + EpochsContained []uint64 `custom_size:"1"` + InsertBatchID []uuid.UUID `custom_size:"1"` + ValidatorIndex []uint64 + Epoch []int64 + EpochTimestamp []*time.Time + BalanceEffectiveStart []int64 + BalanceEffectiveEnd []int64 + BalanceStart []int64 + BalanceEnd []int64 + DepositsCount []int64 + DepositsAmount []int64 + WithdrawalsCount []int64 + WithdrawalsAmount []int64 + AttestationsScheduled []int64 + AttestationsObserved []int64 + AttestationsHeadMatched []int64 + AttestationsSourceMatched []int64 + AttestationsTargetMatched []int64 + AttestationsHeadExecuted []int64 + AttestationsSourceExecuted []int64 + AttestationsTargetExecuted []int64 + AttestationsHeadReward []int64 + AttestationsSourceReward []int64 + AttestationsTargetReward []int64 + AttestationsInactivityReward []int64 + AttestationsInclusionReward []int64 + AttestationsIdealHeadReward []int64 + AttestationsIdealSourceReward []int64 + AttestationsIdealTargetReward []int64 + AttestationsIdealInactivityReward []int64 + AttestationsIdealInclusionReward []int64 + AttestationsLocalizedMaxReward []int64 + AttestationsHyperLocalizedMaxReward []int64 + InclusionDelaySum []int64 + OptimalInclusionDelaySum []int64 + BlocksStatusSlot [][]int64 + BlocksStatusProposed [][]bool + BlockRewardsSlot [][]int64 + BlockRewardsAttestationsReward [][]int64 + BlockRewardsSyncAggregateReward [][]int64 + BlockRewardsSlasherReward [][]int64 + BlocksClMissedMedianReward []int64 + BlocksSlashingCount []int64 + BlocksExpected []float64 + SyncScheduled []int64 + SyncStatusSlot [][]int64 + SyncStatusExecuted [][]bool + SyncRewardsSlot [][]int64 + SyncRewardsReward [][]int64 + SyncLocalizedMaxReward []int64 + SyncCommitteesExpected []float64 + Slashed []bool + AttestationAssignmentsSlot [][]int64 + AttestationAssignmentsCommittee [][]int64 + AttestationAssignmentsIndex [][]int64 + SyncCommitteeAssignmentsPeriod [][]int64 + SyncCommitteeAssignmentsIndex [][]int64 +} + +// get by string +func (c *VDBDataEpochColumns) Get(str string) any { + // test type assertion + switch str { + case "validator_index": + return c.ValidatorIndex + case "epoch": + return c.Epoch + case "epoch_timestamp": + return c.EpochTimestamp + case "balance_effective_start": + return c.BalanceEffectiveStart + case "balance_effective_end": + return c.BalanceEffectiveEnd + case "balance_start": + return c.BalanceStart + case "balance_end": + return c.BalanceEnd + case "deposits_count": + return c.DepositsCount + case "deposits_amount": + return c.DepositsAmount + case "withdrawals_count": + return c.WithdrawalsCount + case "withdrawals_amount": + return c.WithdrawalsAmount + case "attestations_scheduled": + return c.AttestationsScheduled + case "attestations_observed": + return c.AttestationsObserved + case "attestations_head_executed": + return c.AttestationsHeadExecuted + case "attestations_source_executed": + return c.AttestationsSourceExecuted + case "attestations_target_executed": + return c.AttestationsTargetExecuted + case "attestations_head_matched": + return c.AttestationsHeadMatched + case "attestations_source_matched": + return c.AttestationsSourceMatched + case "attestations_target_matched": + return c.AttestationsTargetMatched + case "attestations_head_reward": + return c.AttestationsHeadReward + case "attestations_source_reward": + return c.AttestationsSourceReward + case "attestations_target_reward": + return c.AttestationsTargetReward + case "attestations_inactivity_reward": + return c.AttestationsInactivityReward + case "attestations_inclusion_reward": + return c.AttestationsInclusionReward + case "attestations_ideal_head_reward": + return c.AttestationsIdealHeadReward + case "attestations_ideal_source_reward": + return c.AttestationsIdealSourceReward + case "attestations_ideal_target_reward": + return c.AttestationsIdealTargetReward + case "attestations_ideal_inactivity_reward": + return c.AttestationsIdealInactivityReward + case "attestations_ideal_inclusion_reward": + return c.AttestationsIdealInclusionReward + case "attestations_localized_max_reward": + return c.AttestationsLocalizedMaxReward + case "attestations_hyperlocalized_max_reward": + return c.AttestationsHyperLocalizedMaxReward + case "inclusion_delay_sum": + return c.InclusionDelaySum + case "optimal_inclusion_delay_sum": + return c.OptimalInclusionDelaySum + case "blocks_status.slot": + return c.BlocksStatusSlot + case "blocks_status.proposed": + return c.BlocksStatusProposed + case "block_rewards.slot": + return c.BlockRewardsSlot + case "block_rewards.attestations_reward": + return c.BlockRewardsAttestationsReward + case "block_rewards.sync_aggregate_reward": + return c.BlockRewardsSyncAggregateReward + case "block_rewards.slasher_reward": + return c.BlockRewardsSlasherReward + case "blocks_cl_missed_median_reward": + return c.BlocksClMissedMedianReward + case "blocks_slashing_count": + return c.BlocksSlashingCount + case "blocks_expected": + return c.BlocksExpected + case "sync_scheduled": + return c.SyncScheduled + case "sync_status.slot": + return c.SyncStatusSlot + case "sync_status.executed": + return c.SyncStatusExecuted + case "sync_rewards.slot": + return c.SyncRewardsSlot + case "sync_rewards.reward": + return c.SyncRewardsReward + case "sync_localized_max_reward": + return c.SyncLocalizedMaxReward + case "sync_committees_expected": + return c.SyncCommitteesExpected + case "slashed": + return c.Slashed + case "attestation_assignments.slot": + return c.AttestationAssignmentsSlot + case "attestation_assignments.committee": + return c.AttestationAssignmentsCommittee + case "attestation_assignments.index": + return c.AttestationAssignmentsIndex + case "sync_committee_assignments.period": + return c.SyncCommitteeAssignmentsPeriod + case "sync_committee_assignments.index": + return c.SyncCommitteeAssignmentsIndex + default: + return nil + } +} + +// factory that pre allocates slices with a given capacity +func NewVDBDataEpochColumns(capacity int) (VDBDataEpochColumns, error) { + start := time.Now() + c := VDBDataEpochColumns{} + ct := reflect.TypeOf(c) + cv := reflect.ValueOf(&c).Elem() + var g errgroup.Group + + for i := 0; i < ct.NumField(); i++ { + i := i // capture loop variable + g.Go(func() error { + f := cv.Field(i) + // check for custom_size tag + if tag := ct.Field(i).Tag.Get("custom_size"); tag != "" { + cap, err := strconv.Atoi(tag) + if err != nil { + return fmt.Errorf("failed to parse custom_size tag: %w", err) + } + f.Set(reflect.MakeSlice(f.Type(), cap, cap)) + } else { + f.Set(reflect.MakeSlice(f.Type(), capacity, capacity)) + } + return nil + }) + } + + if err := g.Wait(); err != nil { + return c, err + } + + log.Debugf("allocated in %s", time.Since(start)) + return c, nil +} + +// util function to combine two VDBDataEpochColumns +func (c *VDBDataEpochColumns) Extend(cOther db.UltraFastClickhouseStruct) error { + c2, ok := (cOther).(*VDBDataEpochColumns) + if !ok { + return fmt.Errorf("type assertion failed") + } + // use reflection baby + start := time.Now() + ct := reflect.TypeOf(*c) + cv := reflect.ValueOf(c).Elem() + for i := 0; i < ct.NumField(); i++ { + f := cv.Field(i) + f2 := reflect.ValueOf(c2).Elem().Field(i) + // append + cv.Field(i).Set(reflect.AppendSlice(f, f2)) + } + log.Debugf("extended in %s", time.Since(start)) + return nil +} diff --git a/backend/pkg/nodejobs/node_jobs.go b/backend/pkg/nodejobs/node_jobs.go index aff2ba511..ec43f2cbf 100644 --- a/backend/pkg/nodejobs/node_jobs.go +++ b/backend/pkg/nodejobs/node_jobs.go @@ -11,6 +11,7 @@ import ( "time" "github.com/attestantio/go-eth2-client/spec/capella" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/gobitfly/beaconchain/pkg/commons/db" "github.com/gobitfly/beaconchain/pkg/commons/log" "github.com/gobitfly/beaconchain/pkg/commons/types" @@ -368,7 +369,11 @@ func CreateVoluntaryExitNodeJob(nj *types.NodeJob) (*types.NodeJob, error) { } forkVersion := utils.ForkVersionAtEpoch(uint64(njd.Message.Epoch)) - err = utils.VerifyVoluntaryExitSignature(njd, forkVersion.CurrentVersion, vali.Pubkey) + currentFork, err := hexutil.Decode(forkVersion.CurrentVersion) + if err != nil { + return nil, fmt.Errorf("internal: failed to parse fork version: %w", err) + } + err = utils.VerifyVoluntaryExitSignature(njd, currentFork, vali.Pubkey) if err != nil { return nil, err } From 77efa30e678b5376c16864407b88be11cbb7f390 Mon Sep 17 00:00:00 2001 From: invis-bitfly <162128378+invis-bitfly@users.noreply.github.com> Date: Wed, 29 Jan 2025 15:54:33 +0100 Subject: [PATCH 2/2] fix(monitoring): failing status reports --- backend/pkg/monitoring/services/base.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/backend/pkg/monitoring/services/base.go b/backend/pkg/monitoring/services/base.go index 9858d63e4..ccf016d28 100644 --- a/backend/pkg/monitoring/services/base.go +++ b/backend/pkg/monitoring/services/base.go @@ -9,6 +9,7 @@ import ( "sync/atomic" "time" + "github.com/ClickHouse/clickhouse-go/v2" "github.com/gobitfly/beaconchain/pkg/commons/db" "github.com/gobitfly/beaconchain/pkg/commons/log" "github.com/gobitfly/beaconchain/pkg/commons/utils" @@ -66,8 +67,15 @@ func NewStatusReport(id constants.Event, timeout time.Duration, check_interval t } // report status to monitoring - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + timeoutContext, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() + // wrap in clickhouse context so we can set the setting throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert to 0 + // we have no materialized views on the status_reports table, but it triggers as we need the deduplication setting for the other tables + ctx := clickhouse.Context(timeoutContext, clickhouse.WithSettings( + clickhouse.Settings{ + "throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert": 0, + }, + )) timeouts_at := now.Add(1 * time.Minute) if timeout != constants.Default {