Skip to content

Commit

Permalink
BEDS-140: feat(exporter): clickhouse cl dashboard data exporter
Browse files Browse the repository at this point in the history
  • Loading branch information
@invis-bitfly
invis-bitfly committed Jan 20, 2025
1 parent 0ed1e97 commit 7021fb1
Show file tree
Hide file tree
Showing 57 changed files with 7,935 additions and 4,777 deletions.
66 changes: 39 additions & 27 deletions backend/cmd/exporter/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,14 +60,44 @@ func Run() {
defer wg.Done()
db.WriterDb, db.ReaderDb = db.MustInitDB(&cfg.WriterDatabase, &cfg.ReaderDatabase, "pgx", "postgres")
}()
wg.Add(1)
go func() {
defer wg.Done()
db.AlloyWriter, db.AlloyReader = db.MustInitDB(&cfg.AlloyWriter, &cfg.AlloyReader, "pgx", "postgres")
}()
wg.Add(1)
go func() {
defer wg.Done()
bt, err := db.InitBigtable(utils.Config.Bigtable.Project, utils.Config.Bigtable.Instance, fmt.Sprintf("%d", utils.Config.Chain.ClConfig.DepositChainID), utils.Config.RedisCacheEndpoint)
if err != nil {
log.Fatal(err, "error connecting to bigtable", 0)
}
db.BigtableClient = bt
}()
if utils.Config.TieredCacheProvider != "redis" {
log.Fatal(fmt.Errorf("no cache provider set, please set TierdCacheProvider (example redis)"), "", 0)
}
if utils.Config.TieredCacheProvider == "redis" || len(utils.Config.RedisCacheEndpoint) != 0 {
wg.Add(1)
go func() {
defer wg.Done()
cache.MustInitTieredCache(utils.Config.RedisCacheEndpoint)
log.Infof("tiered Cache initialized, latest finalized epoch: %v", cache.LatestFinalizedEpoch.Get())
}()
}
} else {
log.Warnf("------- EXPORTER RUNNING IN V2 ONLY MODE ------")
}

wg.Add(1)
go func() {
defer wg.Done()
db.AlloyWriter, db.AlloyReader = db.MustInitDB(&cfg.AlloyWriter, &cfg.AlloyReader, "pgx", "postgres")
db.ClickHouseWriter, db.ClickHouseReader = db.MustInitDB(&cfg.ClickHouse.WriterDatabase, &cfg.ClickHouse.ReaderDatabase, "clickhouse", "clickhouse")
}()
wg.Add(1)
go func() {
defer wg.Done()
db.ClickHouseNativeWriter = db.MustInitClickhouseNative(&cfg.ClickHouse.WriterDatabase)
}()

wg.Add(1)
Expand Down Expand Up @@ -101,25 +131,6 @@ func Run() {
}
}()

wg.Add(1)
go func() {
defer wg.Done()
bt, err := db.InitBigtable(utils.Config.Bigtable.Project, utils.Config.Bigtable.Instance, fmt.Sprintf("%d", utils.Config.Chain.ClConfig.DepositChainID), utils.Config.RedisCacheEndpoint)
if err != nil {
log.Fatal(err, "error connecting to bigtable", 0)
}
db.BigtableClient = bt
}()

if utils.Config.TieredCacheProvider == "redis" || len(utils.Config.RedisCacheEndpoint) != 0 {
wg.Add(1)
go func() {
defer wg.Done()
cache.MustInitTieredCache(utils.Config.RedisCacheEndpoint)
log.Infof("tiered Cache initialized, latest finalized epoch: %v", cache.LatestFinalizedEpoch.Get())
}()
}

wg.Add(1)
go func() {
defer wg.Done()
Expand All @@ -141,17 +152,18 @@ func Run() {
monitoring.Init(false)
monitoring.Start()

if utils.Config.TieredCacheProvider != "redis" {
log.Fatal(fmt.Errorf("no cache provider set, please set TierdCacheProvider (example redis)"), "", 0)
}

if !cfg.JustV2 {
defer db.ReaderDb.Close()
defer db.WriterDb.Close()
defer db.AlloyReader.Close()
defer db.AlloyWriter.Close()
defer db.BigtableClient.Close()
}
defer db.AlloyReader.Close()
defer db.AlloyWriter.Close()
defer db.BigtableClient.Close()
defer db.ClickHouseReader.Close()
defer db.ClickHouseWriter.Close()
defer db.ClickHouseNativeWriter.Close()

wg.Wait()

context, err := modules.GetModuleContext()
if err != nil {
Expand Down
25 changes: 13 additions & 12 deletions backend/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ require (
cloud.google.com/go/bigtable v1.21.0
cloud.google.com/go/secretmanager v1.11.5
firebase.google.com/go/v4 v4.14.1
github.com/ClickHouse/clickhouse-go/v2 v2.17.1
github.com/ClickHouse/clickhouse-go/v2 v2.30.0
github.com/Gurpartap/storekit-go v0.0.0-20201205024111-36b6cd5c6a21
github.com/alexedwards/scs/redisstore v0.0.0-20240316134038-7e11d57e8885
github.com/alexedwards/scs/v2 v2.8.0
Expand Down Expand Up @@ -52,7 +52,7 @@ require (
github.com/juliangruber/go-intersect v1.1.0
github.com/jung-kurt/gofpdf v1.16.2
github.com/k3a/html2text v1.2.1
github.com/klauspost/compress v1.17.6
github.com/klauspost/compress v1.17.7
github.com/klauspost/pgzip v1.2.6
github.com/lib/pq v1.10.9
github.com/mailgun/mailgun-go/v4 v4.12.0
Expand All @@ -65,6 +65,7 @@ require (
github.com/prysmaticlabs/go-ssz v0.0.0-20210121151755-f6208871c388
github.com/rocket-pool/rocketpool-go v1.8.4-0.20241009143357-7b6894d57365
github.com/rocket-pool/smartnode v1.14.1
github.com/segmentio/encoding v0.4.0
github.com/sethvargo/go-envconfig v1.1.0
github.com/shopspring/decimal v1.4.0
github.com/sirupsen/logrus v1.9.3
Expand Down Expand Up @@ -94,14 +95,14 @@ require (
cloud.google.com/go/iam v1.1.7 // indirect
cloud.google.com/go/longrunning v0.5.5 // indirect
cloud.google.com/go/storage v1.40.0 // indirect
github.com/ClickHouse/ch-go v0.58.2 // indirect
github.com/ClickHouse/ch-go v0.61.5 // indirect
github.com/MicahParks/keyfunc v1.9.0 // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 // indirect
github.com/ajg/form v1.5.1 // indirect
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
github.com/alessio/shellescape v1.4.1 // indirect
github.com/andybalholm/brotli v1.0.6 // indirect
github.com/andybalholm/brotli v1.1.1 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.0 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.0 // indirect
Expand Down Expand Up @@ -148,7 +149,7 @@ require (
github.com/glendc/go-external-ip v0.1.0 // indirect
github.com/go-chi/chi/v5 v5.0.8 // indirect
github.com/go-faster/city v1.0.1 // indirect
github.com/go-faster/errors v0.6.1 // indirect
github.com/go-faster/errors v0.7.1 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
Expand Down Expand Up @@ -217,9 +218,9 @@ require (
github.com/multiformats/go-multihash v0.2.3 // indirect
github.com/multiformats/go-varint v0.0.7 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/paulmach/orb v0.10.0 // indirect
github.com/paulmach/orb v0.11.1 // indirect
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
github.com/pierrec/lz4/v4 v4.1.18 // indirect
github.com/pierrec/lz4/v4 v4.1.21 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/polydawn/refmt v0.89.0 // indirect
github.com/prometheus/client_model v0.6.0 // indirect
Expand Down Expand Up @@ -259,9 +260,9 @@ require (
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
go.opentelemetry.io/otel v1.24.0 // indirect
go.opentelemetry.io/otel/metric v1.24.0 // indirect
go.opentelemetry.io/otel/trace v1.24.0 // indirect
go.opentelemetry.io/otel v1.26.0 // indirect
go.opentelemetry.io/otel/metric v1.26.0 // indirect
go.opentelemetry.io/otel/trace v1.26.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.0 // indirect
golang.org/x/mod v0.17.0 // indirect
Expand All @@ -272,8 +273,8 @@ require (
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/appengine/v2 v2.0.2 // indirect
google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect
google.golang.org/grpc v1.62.1 // indirect
gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
Expand Down
Loading

0 comments on commit 7021fb1

Please sign in to comment.