Beds 1179/pectra api

backend/cmd/exporter/main.go

@@ -19,7 +19,6 @@ import (
 	"github.com/gobitfly/beaconchain/pkg/commons/version"
 	"github.com/gobitfly/beaconchain/pkg/exporter/modules"
 	"github.com/gobitfly/beaconchain/pkg/exporter/services"
-	"github.com/gobitfly/beaconchain/pkg/monitoring"
 )
 
 func Run() {
@@ -138,8 +137,8 @@ func Run() {
 	wg.Wait()
 
 	// enable light-weight db connection monitoring
-	monitoring.Init(false)
-	monitoring.Start()
+	// monitoring.Init(false)
+	// monitoring.Start()
 
 	if utils.Config.TieredCacheProvider != "redis" {
 		log.Fatal(fmt.Errorf("no cache provider set, please set TierdCacheProvider (example redis)"), "", 0)

backend/cmd/misc/main.go

@@ -138,12 +138,13 @@ func Run() {
 	requires, ok := REQUIRES_LIST[opts.Command]
 	if !ok {
 		requires = misctypes.Requires{
-			Bigtable:   true,
-			Redis:      true,
-			ClNode:     true,
-			ElNode:     true,
-			UserDBs:    true,
-			NetworkDBs: true,
+			Bigtable:      true,
+			Redis:         true,
+			ClNode:        true,
+			ElNode:        true,
+			UserDBs:       true,
+			NetworkDBs:    true,
+			ClickhouseDBs: false,
 		}
 	}
 
@@ -195,9 +196,11 @@ func Run() {
 	}
 
 	// clickhouse
-	db.ClickHouseWriter, db.ClickHouseReader = db.MustInitDB(&cfg.ClickHouse.WriterDatabase, &cfg.ClickHouse.ReaderDatabase, "clickhouse", "clickhouse")
-	defer db.ClickHouseReader.Close()
-	defer db.ClickHouseWriter.Close()
+	if requires.ClickhouseDBs {
+		db.ClickHouseWriter, db.ClickHouseReader = db.MustInitDB(&cfg.ClickHouse.WriterDatabase, &cfg.ClickHouse.ReaderDatabase, "clickhouse", "clickhouse")
+		defer db.ClickHouseReader.Close()
+		defer db.ClickHouseWriter.Close()
+	}
 
 	// Initialize the persistent redis client
 	if requires.Redis {
@@ -1142,7 +1145,7 @@ func debugBlocks(clClient *rpc.LighthouseClient) error {
 		} else if clBlock.ExecutionPayload.BlockNumber != i {
 			log.Warnf("clBlock.ExecutionPayload.BlockNumber != i: %v != %v", clBlock.ExecutionPayload.BlockNumber, i)
 		} else {
-			logFields["cl.txs"] = len(clBlock.ExecutionPayload.Transactions)
+			logFields["cl.txs"] = clBlock.ExecutionPayload.TransactionsCount
 		}
 
 		log.InfoWithFields(logFields, "debug block")

backend/cmd/misc/misctypes/requires.go

@@ -1,10 +1,11 @@
 package misctypes
 
 type Requires struct {
-	Bigtable   bool
-	Redis      bool
-	ClNode     bool
-	ElNode     bool
-	NetworkDBs bool
-	UserDBs    bool
+	Bigtable      bool
+	Redis         bool
+	ClNode        bool
+	ElNode        bool
+	NetworkDBs    bool
+	UserDBs       bool
+	ClickhouseDBs bool
 }

backend/pkg/api/data_access/dummy.go

@@ -214,6 +214,10 @@ func (d *DummyService) GetValidatorsFromSlices(ctx context.Context, indices []ui
 	return getDummyData[[]t.VDBValidator](ctx)
 }
 
+func (d *DummyService) GetValidatorsEffectiveBalanceTotal(ctx context.Context, indices []uint64) (uint64, error) {
+	return getDummyData[uint64](ctx)
+}
+
 func (d *DummyService) GetUserDashboards(ctx context.Context, userId uint64) (*t.UserDashboardsData, error) {
 	return getDummyStruct[t.UserDashboardsData](ctx)
 }
@@ -266,7 +270,7 @@ func (d *DummyService) GetValidatorDashboardGroupExists(ctx context.Context, das
 	return true, nil
 }
 
-func (d *DummyService) AddValidatorDashboardValidators(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, validators []t.VDBValidator) ([]t.VDBPostValidatorsData, error) {
+func (d *DummyService) AddValidatorDashboardValidators(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, validators []t.VDBValidator, limit uint64) ([]t.VDBPostValidatorsData, error) {
 	return getDummyData[[]t.VDBPostValidatorsData](ctx)
 }
 
@@ -542,7 +546,7 @@ func (d *DummyService) GetValidatorDashboardGroupCount(ctx context.Context, dash
 	return getDummyData[uint64](ctx)
 }
 
-func (d *DummyService) GetValidatorDashboardValidatorsCount(ctx context.Context, dashboardId t.VDBIdPrimary) (uint64, error) {
+func (d *DummyService) GetValidatorDashboardEffectiveBalanceTotal(ctx context.Context, dashboardId t.VDBId) (uint64, error) {
 	return getDummyData[uint64](ctx)
 }
 

backend/pkg/api/data_access/general.go

@@ -2,13 +2,17 @@ package dataaccess
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
 
 	"github.com/doug-martin/goqu/v9"
 	"github.com/doug-martin/goqu/v9/exp"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/gobitfly/beaconchain/pkg/api/types"
+	t "github.com/gobitfly/beaconchain/pkg/api/types"
 	"github.com/gobitfly/beaconchain/pkg/commons/db"
+	"github.com/gobitfly/beaconchain/pkg/commons/utils"
+	constypes "github.com/gobitfly/beaconchain/pkg/consapi/types"
 )
 
 // retrieve (primary) ens name and optional name (=label) maintained by beaconcha.in, if present
@@ -131,3 +135,83 @@ func applySortAndPagination(defaultColumns []types.SortColumn, primary types.Sor
 
 	return queryOrder, queryWhere, nil
 }
+
+// returns the effective balances of the provided validators
+// executed from the vdb premium limits pov, i.e. exited validators account for the EB at exit time
+func (d *DataAccessService) GetValidatorsEffectiveBalances(ctx context.Context, validators []t.VDBValidator, onlyActive bool) (map[t.VDBValidator]uint64, error) {
+	validatorMapping, err := d.services.GetCurrentValidatorMapping()
+	if err != nil {
+		return nil, err
+	}
+
+	// active
+	effectiveBalances := make(map[t.VDBValidator]uint64)
+	var validatorExitEpochs []exp.Expression
+	for _, validator := range validators {
+		if len(validatorMapping.ValidatorMetadata) <= int(validator) {
+			return nil, fmt.Errorf("validator index %d not found in validator mapping", validator)
+		}
+		status := constypes.ValidatorDbStatus(validatorMapping.ValidatorMetadata[validator].Status)
+		if !onlyActive &&
+			(status == constypes.DbSlashed || status == constypes.DbExited) &&
+			validatorMapping.ValidatorMetadata[validator].EffectiveBalance == 0 {
+			// exited & balance withdrawn, need to query latest EB before exit
+			if !validatorMapping.ValidatorMetadata[validator].ExitEpoch.Valid {
+				return nil, fmt.Errorf("validator %d has no exit epoch", validator)
+			}
+			// goqu can't pass tuples directly
+			epochTs := utils.EpochToTime(uint64(validatorMapping.ValidatorMetadata[validator].ExitEpoch.Int64)).Unix()
+			validatorExitEpochs = append(validatorExitEpochs, goqu.L("(fromUnixTimestamp(?), ?)", epochTs, validator))
+		} else {
+			effectiveBalances[validator] = validatorMapping.ValidatorMetadata[validator].EffectiveBalance
+		}
+	}
+
+	// exited
+	if len(validatorExitEpochs) > 0 {
+		ds := goqu.Dialect("postgres").
+			Select(
+				// goqu.SUM(goqu.I("balance_effective_end")).As("balance_effective_end"),
+				goqu.I("validator_index"),
+				goqu.I("balance_effective_end"),
+			).
+			From("validator_dashboard_data_epoch").
+			Where(
+				goqu.L("(epoch_timestamp, validator_index)").In(validatorExitEpochs),
+			)
+		query, args, err := ds.Prepared(true).ToSQL()
+		if err != nil {
+			return nil, err
+		}
+
+		ebsBeforeExit := []struct {
+			ValidatorIndex   uint64 `db:"validator_index"`
+			EffectiveBalance uint64 `db:"balance_effective_end"`
+		}{}
+		err = d.clickhouseReader.SelectContext(ctx, &ebsBeforeExit, query, args...)
+		if err != nil && err != sql.ErrNoRows {
+			return nil, err
+		}
+		for _, eb := range ebsBeforeExit {
+			effectiveBalances[eb.ValidatorIndex] = eb.EffectiveBalance
+		}
+	}
+	return effectiveBalances, nil
+}
+
+func (d *DataAccessService) GetValidatorsEffectiveBalanceTotal(ctx context.Context, validators []t.VDBValidator) (uint64, error) {
+	validatorEbs, err := d.GetValidatorsEffectiveBalances(ctx, validators, false)
+	if err != nil {
+		return 0, err
+	}
+
+	var totalEb uint64
+	for _, validator := range validators {
+		if eb, ok := validatorEbs[validator]; !ok {
+			return 0, fmt.Errorf("effective balance for validator %d not found", validator)
+		} else {
+			totalEb += eb
+		}
+	}
+	return totalEb, nil
+}

backend/pkg/api/data_access/networks.go

@@ -31,5 +31,15 @@ func (d *DataAccessService) GetAllNetworks() ([]types.NetworkInfo, error) {
 			Name:              "sepolia",
 			NotificationsName: "sepolia",
 		},
+		{
+			ChainId:           7088110746,
+			Name:              "pectra-devnet-5",
+			NotificationsName: "pectra-devnet-5",
+		},
+		{
+			ChainId:           7072151312,
+			Name:              "pectra-devnet-6",
+			NotificationsName: "pectra-devnet-6",
+		},
 	}, nil
 }

backend/pkg/api/data_access/user.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/doug-martin/goqu/v9"
 	t "github.com/gobitfly/beaconchain/pkg/api/types"
 	"github.com/gobitfly/beaconchain/pkg/commons/db"
 	"github.com/gobitfly/beaconchain/pkg/commons/utils"
@@ -268,7 +269,11 @@ func (d *DataAccessService) GetProductSummary(ctx context.Context) (*t.ProductSu
 }
 
 func (d *DataAccessService) GetFreeTierPerks(ctx context.Context) (*t.PremiumPerks, error) {
-	return db.GetFreeTierPerks(ctx)
+	perks, err := db.GetFreeTierProduct(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return &perks.PremiumPerks, nil
 }
 
 func (d *DataAccessService) GetUserDashboards(ctx context.Context, userId uint64) (*t.UserDashboardsData, error) {
@@ -361,6 +366,61 @@ func (d *DataAccessService) GetUserDashboards(ctx context.Context, userId uint64
 		return nil
 	})
 
+	// TODO could merge with above query
+	validatorDashboardEBs := make(map[uint64]uint64, 0)
+	wg.Go(func() error {
+		validatorsDs := goqu.Dialect("postgres").
+			From(goqu.T("users_val_dashboards").As("uvd")).
+			LeftJoin(
+				goqu.T("users_val_dashboards_validators").As("uvdv"),
+				goqu.On(goqu.I("uvd.id").Eq(goqu.I("uvdv.dashboard_id"))),
+			).
+			Select(
+				goqu.I("uvd.id").As("dashboard_id"),
+				goqu.I("uvdv.validator_index").As("validator_index"),
+			).
+			Where(goqu.I("uvd.user_id").Eq(userId))
+
+		validatorsQuery, args, err := validatorsDs.Prepared(true).ToSQL()
+		if err != nil {
+			return err
+		}
+
+		// could also scan into array
+		dashboardValidators := []struct {
+			Id             uint64 `db:"dashboard_id"`
+			ValidatorIndex uint64 `db:"validator_index"`
+		}{}
+		err = d.alloyReader.SelectContext(ctx, &dashboardValidators, validatorsQuery, args...)
+		if err != nil {
+			return err
+		}
+		dashboardValidatorsMap := make(map[uint64][]t.VDBValidator, 0)
+		validators := make([]t.VDBValidator, 0, len(dashboardValidators))
+		for _, row := range dashboardValidators {
+			dashboardValidatorsMap[row.Id] = append(dashboardValidatorsMap[row.Id], row.ValidatorIndex)
+			validators = append(validators, row.ValidatorIndex)
+		}
+		validatorEbs, err := d.GetValidatorsEffectiveBalances(ctx, validators, false)
+		if err != nil {
+			return err
+		}
+
+		for dashboard_id, validators := range dashboardValidatorsMap {
+			var dashboardTotalEb uint64
+			for _, validator := range validators {
+				if eb, ok := validatorEbs[validator]; !ok {
+					return fmt.Errorf("effective balance for validator %d not found", validator)
+				} else {
+					dashboardTotalEb += eb
+				}
+			}
+			validatorDashboardEBs[dashboard_id] = dashboardTotalEb
+		}
+
+		return nil
+	})
+
 	err := wg.Wait()
 	if err != nil {
 		return nil, fmt.Errorf("error retrieving user dashboards data: %w", err)
@@ -370,6 +430,7 @@ func (d *DataAccessService) GetUserDashboards(ctx context.Context, userId uint64
 	for _, validatorDashboard := range validatorDashboardMap {
 		validatorDashboard.GroupCount = validatorDashboardCountMap[validatorDashboard.Id].GroupCount
 		validatorDashboard.ValidatorCount = validatorDashboardCountMap[validatorDashboard.Id].ValidatorCount
+		validatorDashboard.EffectiveBalance = validatorDashboardEBs[validatorDashboard.Id]
 
 		result.ValidatorDashboards = append(result.ValidatorDashboards, *validatorDashboard)
 	}

backend/pkg/api/data_access/vdb.go

@@ -29,14 +29,14 @@ type ValidatorDashboardRepository interface {
 	GetValidatorDashboardGroupCount(ctx context.Context, dashboardId t.VDBIdPrimary) (uint64, error)
 	GetValidatorDashboardGroupExists(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64) (bool, error)
 
-	AddValidatorDashboardValidators(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, validators []t.VDBValidator) ([]t.VDBPostValidatorsData, error)
+	AddValidatorDashboardValidators(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, validators []t.VDBValidator, limit uint64) ([]t.VDBPostValidatorsData, error)
 	AddValidatorDashboardValidatorsByDepositAddress(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, address string, limit uint64) ([]t.VDBPostValidatorsData, error)
 	AddValidatorDashboardValidatorsByWithdrawalCredential(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, credential string, limit uint64) ([]t.VDBPostValidatorsData, error)
 	AddValidatorDashboardValidatorsByGraffiti(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, graffiti string, limit uint64) ([]t.VDBPostValidatorsData, error)
 
 	RemoveValidatorDashboardValidators(ctx context.Context, dashboardId t.VDBIdPrimary, validators []t.VDBValidator) error
 	GetValidatorDashboardValidators(ctx context.Context, dashboardId t.VDBId, groupId int64, cursor string, colSort t.Sort[enums.VDBManageValidatorsColumn], search string, limit uint64) ([]t.VDBManageValidatorsTableRow, *t.Paging, error)
-	GetValidatorDashboardValidatorsCount(ctx context.Context, dashboardId t.VDBIdPrimary) (uint64, error)
+	GetValidatorDashboardEffectiveBalanceTotal(ctx context.Context, dashboardId t.VDBId) (uint64, error)
 
 	CreateValidatorDashboardPublicId(ctx context.Context, dashboardId t.VDBIdPrimary, name string, shareGroups bool) (*t.VDBPublicId, error)
 	GetValidatorDashboardPublicId(ctx context.Context, publicDashboardId t.VDBIdPublic) (*t.VDBPublicId, error)