From 41dd0e2c69660e0932d23012080d74672b98de93 Mon Sep 17 00:00:00 2001 From: Lucca Dukic <109136188+LuccaBitfly@users.noreply.github.com> Date: Wed, 8 Jan 2025 09:07:58 +0100 Subject: [PATCH 1/6] refactor: remove `internal_` prefix from unexported func name See: BEDS-1065 --- backend/pkg/api/data_access/mobile.go | 4 +- backend/pkg/api/data_access/vdb_helpers.go | 156 +++++++++++++++++ backend/pkg/api/data_access/vdb_management.go | 2 +- backend/pkg/api/data_access/vdb_summary.go | 157 +----------------- 4 files changed, 160 insertions(+), 159 deletions(-) diff --git a/backend/pkg/api/data_access/mobile.go b/backend/pkg/api/data_access/mobile.go index 8eaef10a1..7d7125436 100644 --- a/backend/pkg/api/data_access/mobile.go +++ b/backend/pkg/api/data_access/mobile.go @@ -263,7 +263,7 @@ func (d *DataAccessService) GetValidatorDashboardMobileWidget(ctx context.Contex retrieveApr := func(hours int, apr *float64) { eg.Go(func() error { - _, elApr, _, clApr, err := d.internal_getElClAPR(ctx, wrappedDashboardId, -1, hours) + _, elApr, _, clApr, err := d.getElClAPR(ctx, wrappedDashboardId, -1, hours) if err != nil { return err } @@ -274,7 +274,7 @@ func (d *DataAccessService) GetValidatorDashboardMobileWidget(ctx context.Contex retrieveRewards := func(hours int, rewards *t.ClElValue[decimal.Decimal]) { eg.Go(func() error { - elRewards, _, clRewards, _, err := d.internal_getElClAPR(ctx, wrappedDashboardId, -1, hours) + elRewards, _, clRewards, _, err := d.getElClAPR(ctx, wrappedDashboardId, -1, hours) if err != nil { return err } diff --git a/backend/pkg/api/data_access/vdb_helpers.go b/backend/pkg/api/data_access/vdb_helpers.go index 30fd72f61..c3945af24 100644 --- a/backend/pkg/api/data_access/vdb_helpers.go +++ b/backend/pkg/api/data_access/vdb_helpers.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "math" "time" "github.com/doug-martin/goqu/v9" @@ -13,6 +14,7 @@ import ( "github.com/gobitfly/beaconchain/pkg/commons/utils" "github.com/lib/pq" "github.com/pkg/errors" + "github.com/shopspring/decimal" ) //////////////////// Helper functions (must be used by more than one VDB endpoint!) @@ -130,3 +132,157 @@ func (d *DataAccessService) getTimeToNextWithdrawal(distance uint64) time.Time { return timeToWithdrawal } + +func (d *DataAccessService) getElClAPR(ctx context.Context, dashboardId t.VDBId, groupId int64, hours int) (elIncome decimal.Decimal, elAPR float64, clIncome decimal.Decimal, clAPR float64, err error) { + table := "" + + switch hours { + case 1: + table = "validator_dashboard_data_rolling_1h" + case 24: + table = "validator_dashboard_data_rolling_24h" + case 7 * 24: + table = "validator_dashboard_data_rolling_7d" + case 30 * 24: + table = "validator_dashboard_data_rolling_30d" + case -1: + table = "validator_dashboard_data_rolling_90d" + default: + return decimal.Zero, 0, decimal.Zero, 0, fmt.Errorf("invalid hours value: %v", hours) + } + + type RewardsResult struct { + EpochStart uint64 `db:"epoch_start"` + EpochEnd uint64 `db:"epoch_end"` + ValidatorCount uint64 `db:"validator_count"` + Reward sql.NullInt64 `db:"reward"` + } + + var rewardsResultTable RewardsResult + var rewardsResultTotal RewardsResult + + rewardsDs := goqu.Dialect("postgres"). + From(goqu.L(fmt.Sprintf("%s AS r FINAL", table))). + With("validators", goqu.L("(SELECT group_id, validator_index FROM users_val_dashboards_validators WHERE dashboard_id = ?)", dashboardId.Id)). + Select( + goqu.L("MIN(epoch_start) AS epoch_start"), + goqu.L("MAX(epoch_end) AS epoch_end"), + goqu.L("COUNT(*) AS validator_count"), + goqu.L(` + ( + SUM(COALESCE(finalizeAggregation(r.balance_end), 0)) + + SUM(COALESCE(r.withdrawals_amount, 0)) - + SUM(COALESCE(r.deposits_amount, 0)) - + SUM(COALESCE(finalizeAggregation(r.balance_start), 0)) + ) AS reward + `)) + if len(dashboardId.Validators) > 0 { + rewardsDs = rewardsDs. + Where(goqu.L("validator_index IN ?", dashboardId.Validators)) + } else { + rewardsDs = rewardsDs. + InnerJoin(goqu.L("validators v"), goqu.On(goqu.L("r.validator_index = v.validator_index"))). + Where(goqu.L("r.validator_index IN (SELECT validator_index FROM validators)")) + + if groupId != -1 { + rewardsDs = rewardsDs. + Where(goqu.L("v.group_id = ?", groupId)) + } + } + + query, args, err := rewardsDs.Prepared(true).ToSQL() + if err != nil { + return decimal.Zero, 0, decimal.Zero, 0, fmt.Errorf("error preparing query: %w", err) + } + + err = d.clickhouseReader.GetContext(ctx, &rewardsResultTable, query, args...) + if err != nil || !rewardsResultTable.Reward.Valid { + return decimal.Zero, 0, decimal.Zero, 0, err + } + + if rewardsResultTable.ValidatorCount == 0 { + return decimal.Zero, 0, decimal.Zero, 0, nil + } + + aprDivisor := hours + if hours == -1 { // for all time APR + aprDivisor = 90 * 24 + } + clAPR = ((float64(rewardsResultTable.Reward.Int64) / float64(aprDivisor)) / (float64(32e9) * float64(rewardsResultTable.ValidatorCount))) * 24.0 * 365.0 * 100.0 + if math.IsNaN(clAPR) { + clAPR = 0 + } + + clIncome = decimal.NewFromInt(rewardsResultTable.Reward.Int64).Mul(decimal.NewFromInt(1e9)) + + if hours == -1 { + rewardsDs = rewardsDs. + From(goqu.L("validator_dashboard_data_rolling_total AS r FINAL")) + + query, args, err = rewardsDs.Prepared(true).ToSQL() + if err != nil { + return decimal.Zero, 0, decimal.Zero, 0, fmt.Errorf("error preparing query: %w", err) + } + + err = d.clickhouseReader.GetContext(ctx, &rewardsResultTotal, query, args...) + if err != nil || !rewardsResultTotal.Reward.Valid { + return decimal.Zero, 0, decimal.Zero, 0, err + } + + clIncome = decimal.NewFromInt(rewardsResultTotal.Reward.Int64).Mul(decimal.NewFromInt(1e9)) + } + + elDs := goqu.Dialect("postgres"). + Select(goqu.COALESCE(goqu.SUM(goqu.L("value / 1e18")), 0)). + From(goqu.I("execution_rewards_finalized").As("b")) + + if len(dashboardId.Validators) > 0 { + elDs = elDs. + Where(goqu.L("b.proposer = ANY(?)", pq.Array(dashboardId.Validators))) + } else { + elDs = elDs. + InnerJoin(goqu.L("users_val_dashboards_validators v"), goqu.On(goqu.L("b.proposer = v.validator_index"))). + Where(goqu.L("v.dashboard_id = ?", dashboardId.Id)) + + if groupId != -1 { + elDs = elDs. + Where(goqu.L("v.group_id = ?", groupId)) + } + } + + elTableDs := elDs. + Where(goqu.L("b.epoch >= ? AND b.epoch <= ?", rewardsResultTable.EpochStart, rewardsResultTable.EpochEnd)) + + query, args, err = elTableDs.Prepared(true).ToSQL() + if err != nil { + return decimal.Zero, 0, decimal.Zero, 0, fmt.Errorf("error preparing query: %w", err) + } + + err = d.alloyReader.GetContext(ctx, &elIncome, query, args...) + if err != nil { + return decimal.Zero, 0, decimal.Zero, 0, err + } + elIncomeFloat, _ := elIncome.Float64() // EL income is in ETH + elAPR = ((elIncomeFloat / float64(aprDivisor)) / (float64(32) * float64(rewardsResultTable.ValidatorCount))) * 24.0 * 365.0 * 100.0 + if math.IsNaN(elAPR) { + elAPR = 0 + } + + if hours == -1 { + elTotalDs := elDs. + Where(goqu.L("b.epoch >= ? AND b.epoch <= ?", rewardsResultTotal.EpochStart, rewardsResultTotal.EpochEnd)) + + query, args, err = elTotalDs.Prepared(true).ToSQL() + if err != nil { + return decimal.Zero, 0, decimal.Zero, 0, fmt.Errorf("error preparing query: %w", err) + } + + err = d.alloyReader.GetContext(ctx, &elIncome, query, args...) + if err != nil { + return decimal.Zero, 0, decimal.Zero, 0, err + } + } + elIncome = elIncome.Mul(decimal.NewFromInt(1e18)) + + return elIncome, elAPR, clIncome, clAPR, nil +} diff --git a/backend/pkg/api/data_access/vdb_management.go b/backend/pkg/api/data_access/vdb_management.go index 0127d5cc9..a9e955096 100644 --- a/backend/pkg/api/data_access/vdb_management.go +++ b/backend/pkg/api/data_access/vdb_management.go @@ -457,7 +457,7 @@ func (d *DataAccessService) GetValidatorDashboardOverview(ctx context.Context, d retrieveRewardsAndEfficiency := func(table string, hours int, rewards *t.ClElValue[decimal.Decimal], apr *t.ClElValue[float64], efficiency *float64) { // Rewards + APR eg.Go(func() error { - (*rewards).El, (*apr).El, (*rewards).Cl, (*apr).Cl, err = d.internal_getElClAPR(ctx, dashboardId, -1, hours) + (*rewards).El, (*apr).El, (*rewards).Cl, (*apr).Cl, err = d.getElClAPR(ctx, dashboardId, -1, hours) if err != nil { return err } diff --git a/backend/pkg/api/data_access/vdb_summary.go b/backend/pkg/api/data_access/vdb_summary.go index d806caba6..8fa5c6cf5 100644 --- a/backend/pkg/api/data_access/vdb_summary.go +++ b/backend/pkg/api/data_access/vdb_summary.go @@ -748,12 +748,11 @@ func (d *DataAccessService) GetValidatorDashboardGroupSummary(ctx context.Contex totalInclusionDelayDivisor += row.AttestationsObserved } } - ret.MissedRewards.Attestations = utils.GWeiToWei(big.NewInt(totalMissedRewardsAttestations)) ret.MissedRewards.Sync = utils.GWeiToWei(big.NewInt(totalMissedRewardsSync)) ret.MissedRewards.ProposerRewards.Cl = utils.GWeiToWei(big.NewInt(totalMissedRewardsCl)) - _, ret.Apr.El, _, ret.Apr.Cl, err = d.internal_getElClAPR(ctx, dashboardId, groupId, hours) + _, ret.Apr.El, _, ret.Apr.Cl, err = d.getElClAPR(ctx, dashboardId, groupId, hours) if err != nil { return nil, err } @@ -839,160 +838,6 @@ func (d *DataAccessService) GetValidatorDashboardGroupSummary(ctx context.Contex return ret, nil } -func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId t.VDBId, groupId int64, hours int) (elIncome decimal.Decimal, elAPR float64, clIncome decimal.Decimal, clAPR float64, err error) { - table := "" - - switch hours { - case 1: - table = "validator_dashboard_data_rolling_1h" - case 24: - table = "validator_dashboard_data_rolling_24h" - case 7 * 24: - table = "validator_dashboard_data_rolling_7d" - case 30 * 24: - table = "validator_dashboard_data_rolling_30d" - case -1: - table = "validator_dashboard_data_rolling_90d" - default: - return decimal.Zero, 0, decimal.Zero, 0, fmt.Errorf("invalid hours value: %v", hours) - } - - type RewardsResult struct { - EpochStart uint64 `db:"epoch_start"` - EpochEnd uint64 `db:"epoch_end"` - ValidatorCount uint64 `db:"validator_count"` - Reward sql.NullInt64 `db:"reward"` - } - - var rewardsResultTable RewardsResult - var rewardsResultTotal RewardsResult - - rewardsDs := goqu.Dialect("postgres"). - From(goqu.L(fmt.Sprintf("%s AS r FINAL", table))). - With("validators", goqu.L("(SELECT group_id, validator_index FROM users_val_dashboards_validators WHERE dashboard_id = ?)", dashboardId.Id)). - Select( - goqu.L("MIN(epoch_start) AS epoch_start"), - goqu.L("MAX(epoch_end) AS epoch_end"), - goqu.L("COUNT(*) AS validator_count"), - goqu.L(` - ( - SUM(COALESCE(finalizeAggregation(r.balance_end), 0)) + - SUM(COALESCE(r.withdrawals_amount, 0)) - - SUM(COALESCE(r.deposits_amount, 0)) - - SUM(COALESCE(finalizeAggregation(r.balance_start), 0)) - ) AS reward - `)) - if len(dashboardId.Validators) > 0 { - rewardsDs = rewardsDs. - Where(goqu.L("validator_index IN ?", dashboardId.Validators)) - } else { - rewardsDs = rewardsDs. - InnerJoin(goqu.L("validators v"), goqu.On(goqu.L("r.validator_index = v.validator_index"))). - Where(goqu.L("r.validator_index IN (SELECT validator_index FROM validators)")) - - if groupId != -1 { - rewardsDs = rewardsDs. - Where(goqu.L("v.group_id = ?", groupId)) - } - } - - query, args, err := rewardsDs.Prepared(true).ToSQL() - if err != nil { - return decimal.Zero, 0, decimal.Zero, 0, fmt.Errorf("error preparing query: %w", err) - } - - err = d.clickhouseReader.GetContext(ctx, &rewardsResultTable, query, args...) - if err != nil || !rewardsResultTable.Reward.Valid { - return decimal.Zero, 0, decimal.Zero, 0, err - } - - if rewardsResultTable.ValidatorCount == 0 { - return decimal.Zero, 0, decimal.Zero, 0, nil - } - - aprDivisor := hours - if hours == -1 { // for all time APR - aprDivisor = 90 * 24 - } - clAPR = ((float64(rewardsResultTable.Reward.Int64) / float64(aprDivisor)) / (float64(32e9) * float64(rewardsResultTable.ValidatorCount))) * 24.0 * 365.0 * 100.0 - if math.IsNaN(clAPR) { - clAPR = 0 - } - - clIncome = decimal.NewFromInt(rewardsResultTable.Reward.Int64).Mul(decimal.NewFromInt(1e9)) - - if hours == -1 { - rewardsDs = rewardsDs. - From(goqu.L("validator_dashboard_data_rolling_total AS r FINAL")) - - query, args, err = rewardsDs.Prepared(true).ToSQL() - if err != nil { - return decimal.Zero, 0, decimal.Zero, 0, fmt.Errorf("error preparing query: %w", err) - } - - err = d.clickhouseReader.GetContext(ctx, &rewardsResultTotal, query, args...) - if err != nil || !rewardsResultTotal.Reward.Valid { - return decimal.Zero, 0, decimal.Zero, 0, err - } - - clIncome = decimal.NewFromInt(rewardsResultTotal.Reward.Int64).Mul(decimal.NewFromInt(1e9)) - } - - elDs := goqu.Dialect("postgres"). - Select(goqu.COALESCE(goqu.SUM(goqu.L("value / 1e18")), 0)). - From(goqu.I("execution_rewards_finalized").As("b")) - - if len(dashboardId.Validators) > 0 { - elDs = elDs. - Where(goqu.L("b.proposer = ANY(?)", pq.Array(dashboardId.Validators))) - } else { - elDs = elDs. - InnerJoin(goqu.L("users_val_dashboards_validators v"), goqu.On(goqu.L("b.proposer = v.validator_index"))). - Where(goqu.L("v.dashboard_id = ?", dashboardId.Id)) - - if groupId != -1 { - elDs = elDs. - Where(goqu.L("v.group_id = ?", groupId)) - } - } - - elTableDs := elDs. - Where(goqu.L("b.epoch >= ? AND b.epoch <= ?", rewardsResultTable.EpochStart, rewardsResultTable.EpochEnd)) - - query, args, err = elTableDs.Prepared(true).ToSQL() - if err != nil { - return decimal.Zero, 0, decimal.Zero, 0, fmt.Errorf("error preparing query: %w", err) - } - - err = d.alloyReader.GetContext(ctx, &elIncome, query, args...) - if err != nil { - return decimal.Zero, 0, decimal.Zero, 0, err - } - elIncomeFloat, _ := elIncome.Float64() // EL income is in ETH - elAPR = ((elIncomeFloat / float64(aprDivisor)) / (float64(32) * float64(rewardsResultTable.ValidatorCount))) * 24.0 * 365.0 * 100.0 - if math.IsNaN(elAPR) { - elAPR = 0 - } - - if hours == -1 { - elTotalDs := elDs. - Where(goqu.L("b.epoch >= ? AND b.epoch <= ?", rewardsResultTotal.EpochStart, rewardsResultTotal.EpochEnd)) - - query, args, err = elTotalDs.Prepared(true).ToSQL() - if err != nil { - return decimal.Zero, 0, decimal.Zero, 0, fmt.Errorf("error preparing query: %w", err) - } - - err = d.alloyReader.GetContext(ctx, &elIncome, query, args...) - if err != nil { - return decimal.Zero, 0, decimal.Zero, 0, err - } - } - elIncome = elIncome.Mul(decimal.NewFromInt(1e18)) - - return elIncome, elAPR, clIncome, clAPR, nil -} - // for summary charts: series id is group id, no stack func (d *DataAccessService) GetValidatorDashboardSummaryChart(ctx context.Context, dashboardId t.VDBId, groupIds []int64, efficiency enums.VDBSummaryChartEfficiencyType, aggregation enums.ChartAggregation, afterTs uint64, beforeTs uint64) (*t.ChartData[int, float64], error) { From 7866cb6cd3b821a86f0d39788a3c044eb4a2cb2a Mon Sep 17 00:00:00 2001 From: Lucca Dukic <109136188+LuccaBitfly@users.noreply.github.com> Date: Wed, 8 Jan 2025 10:41:09 +0100 Subject: [PATCH 2/6] refactor: add fields to validator dashboard group summary - adds efficiency, balances and rewards fields See: BEDS-1065 --- backend/pkg/api/types/validator_dashboard.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/pkg/api/types/validator_dashboard.go b/backend/pkg/api/types/validator_dashboard.go index 663aa745a..043fa0dce 100644 --- a/backend/pkg/api/types/validator_dashboard.go +++ b/backend/pkg/api/types/validator_dashboard.go @@ -13,7 +13,7 @@ type VDBOverviewGroup struct { Count uint64 `json:"count"` } -type VDBOverviewBalances struct { +type ValidatorBalances struct { Total decimal.Decimal `json:"total"` Effective decimal.Decimal `json:"effective"` StakedEth decimal.Decimal `json:"staked_eth"` @@ -28,7 +28,7 @@ type VDBOverviewData struct { Rewards PeriodicValues[ClElValue[decimal.Decimal]] `json:"rewards"` Apr PeriodicValues[ClElValue[float64]] `json:"apr"` ChartHistorySeconds ChartHistorySeconds `json:"chart_history_seconds"` - Balances VDBOverviewBalances `json:"balances"` + Balances ValidatorBalances `json:"balances"` } type GetValidatorDashboardResponse ApiDataResponse[VDBOverviewData] @@ -82,6 +82,10 @@ type VDBGroupSummaryMissedRewards struct { Sync decimal.Decimal `json:"sync"` } type VDBGroupSummaryData struct { + Efficiency float64 `json:"efficiency"` + Balances ValidatorBalances `json:"balances"` + Rewards ClElValue[decimal.Decimal] `json:"rewards"` + AttestationsHead StatusCount `json:"attestations_head"` AttestationsSource StatusCount `json:"attestations_source"` AttestationsTarget StatusCount `json:"attestations_target"` From ee8ff56b9d5ddc518e6f894cae4f8504555d5b4b Mon Sep 17 00:00:00 2001 From: Lucca Dukic <109136188+LuccaBitfly@users.noreply.github.com> Date: Wed, 8 Jan 2025 10:52:09 +0100 Subject: [PATCH 3/6] feat: fill efficiency field in dashboard group summary See: BEDS-1065 --- backend/pkg/api/data_access/vdb_summary.go | 36 ++++++++++++++++------ 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/backend/pkg/api/data_access/vdb_summary.go b/backend/pkg/api/data_access/vdb_summary.go index 8fa5c6cf5..8fc0ca9fc 100644 --- a/backend/pkg/api/data_access/vdb_summary.go +++ b/backend/pkg/api/data_access/vdb_summary.go @@ -676,8 +676,13 @@ func (d *DataAccessService) GetValidatorDashboardGroupSummary(ctx context.Contex totalBlockChance := float64(0) totalInclusionDelaySum := int64(0) totalInclusionDelayDivisor := int64(0) + totalSyncExpected := float64(0) - totalProposals := uint32(0) + totalSyncScheduled := uint32(0) + totalSyncExecuted := uint32(0) + + totalBlocksScheduled := uint32(0) + totalBlocksProposed := uint32(0) totalMissedRewardsCl := int64(0) totalMissedRewardsAttestations := int64(0) @@ -705,8 +710,8 @@ func (d *DataAccessService) GetValidatorDashboardGroupSummary(ctx context.Contex if row.ValidatorIndex == 0 && row.BlocksProposed > 0 && row.BlocksProposed != row.BlocksScheduled { row.BlocksProposed-- // subtract the genesis block from validator 0 (TODO: remove when fixed in the dashoard data exporter) } - - totalProposals += row.BlocksScheduled + totalBlocksProposed += row.BlocksProposed + totalBlocksScheduled += row.BlocksScheduled if row.BlocksScheduled > 0 { if ret.ProposalValidators == nil { ret.ProposalValidators = make([]t.VDBValidator, 0, 10) @@ -714,6 +719,9 @@ func (d *DataAccessService) GetValidatorDashboardGroupSummary(ctx context.Contex ret.ProposalValidators = append(ret.ProposalValidators, t.VDBValidator(row.ValidatorIndex)) } + totalSyncScheduled += row.SyncScheduled + totalSyncExecuted += row.SyncExecuted + ret.SyncCommittee.StatusCount.Success += uint64(row.SyncExecuted) ret.SyncCommittee.StatusCount.Failed += uint64(row.SyncScheduled) - uint64(row.SyncExecuted) @@ -768,11 +776,6 @@ func (d *DataAccessService) GetValidatorDashboardGroupSummary(ctx context.Contex return nil, fmt.Errorf("error retrieving past sync committee count: %w", err) } - ret.AttestationEfficiency = float64(totalAttestationRewards) / float64(totalIdealAttestationRewards) * 100 - if ret.AttestationEfficiency < 0 || math.IsNaN(ret.AttestationEfficiency) { - ret.AttestationEfficiency = 0 - } - luckHours := float64(hours) if hours == -1 { luckHours = time.Since(time.Unix(int64(utils.Config.Chain.GenesisTimestamp), 0)).Hours() @@ -782,7 +785,7 @@ func (d *DataAccessService) GetValidatorDashboardGroupSummary(ctx context.Contex } if totalBlockChance > 0 { - ret.Luck.Proposal.Percent = (float64(totalProposals)) / totalBlockChance * 100 + ret.Luck.Proposal.Percent = (float64(totalBlocksScheduled)) / totalBlockChance * 100 // calculate the average time it takes for the set of validators to propose a single block on average ret.Luck.Proposal.AverageIntervalSeconds = uint64(time.Duration((luckHours / totalBlockChance) * float64(time.Hour)).Seconds()) @@ -834,6 +837,21 @@ func (d *DataAccessService) GetValidatorDashboardGroupSummary(ctx context.Contex ret.SyncCommittee.Validators = ret.SyncCommittee.Validators[:3] } } + var attestationEfficiency, proposerEfficiency, syncEfficiency sql.NullFloat64 + if totalIdealAttestationRewards > 0 { + attestationEfficiency.Float64 = decimal.NewFromInt(totalAttestationRewards).Div(decimal.NewFromInt(totalIdealAttestationRewards)).InexactFloat64() + attestationEfficiency.Valid = true + ret.AttestationEfficiency = max(attestationEfficiency.Float64, 0) + } + if totalBlocksScheduled > 0 { + proposerEfficiency.Float64 = float64(totalBlocksProposed) / float64(totalBlocksScheduled) + proposerEfficiency.Valid = true + } + if totalSyncScheduled > 0 { + syncEfficiency.Float64 = float64(totalSyncExecuted) / float64(totalSyncScheduled) + syncEfficiency.Valid = true + } + ret.Efficiency = utils.CalculateTotalEfficiency(attestationEfficiency, proposerEfficiency, syncEfficiency) return ret, nil } From 93a1b31e475c4ac6369adc81f4652397da24e8ba Mon Sep 17 00:00:00 2001 From: Lucca Dukic <109136188+LuccaBitfly@users.noreply.github.com> Date: Thu, 9 Jan 2025 12:50:56 +0100 Subject: [PATCH 4/6] feat: fill balances field in dashboard group summary See: BEDS-1065 --- backend/pkg/api/data_access/vdb_helpers.go | 97 +++++++++++++++++++ backend/pkg/api/data_access/vdb_management.go | 86 +--------------- backend/pkg/api/data_access/vdb_summary.go | 33 ++++--- 3 files changed, 121 insertions(+), 95 deletions(-) diff --git a/backend/pkg/api/data_access/vdb_helpers.go b/backend/pkg/api/data_access/vdb_helpers.go index c3945af24..c7b5ed9fb 100644 --- a/backend/pkg/api/data_access/vdb_helpers.go +++ b/backend/pkg/api/data_access/vdb_helpers.go @@ -5,10 +5,12 @@ import ( "database/sql" "fmt" "math" + "math/big" "time" "github.com/doug-martin/goqu/v9" "github.com/gobitfly/beaconchain/pkg/api/enums" + "github.com/gobitfly/beaconchain/pkg/api/services" t "github.com/gobitfly/beaconchain/pkg/api/types" "github.com/gobitfly/beaconchain/pkg/commons/cache" "github.com/gobitfly/beaconchain/pkg/commons/utils" @@ -286,3 +288,98 @@ func (d *DataAccessService) getElClAPR(ctx context.Context, dashboardId t.VDBId, return elIncome, elAPR, clIncome, clAPR, nil } + +type RpOperatorInfo struct { + ValidatorIndex uint64 `db:"validatorindex"` + NodeFee float64 `db:"node_fee"` + NodeDepositBalance decimal.Decimal `db:"node_deposit_balance"` + UserDepositBalance decimal.Decimal `db:"user_deposit_balance"` +} + +func (d *DataAccessService) getValidatorDashboardRpOperatorInfo(ctx context.Context, dashboardId t.VDBId) ([]RpOperatorInfo, error) { + var rpOperatorInfo []RpOperatorInfo + + ds := goqu.Dialect("postgres"). + Select( + goqu.L("v.validatorindex"), + goqu.L("rplm.node_fee"), + goqu.L("rplm.node_deposit_balance"), + goqu.L("rplm.user_deposit_balance")). + From(goqu.L("rocketpool_minipools AS rplm")). + LeftJoin(goqu.L("validators AS v"), goqu.On(goqu.L("rplm.pubkey = v.pubkey"))). + Where(goqu.L("node_deposit_balance IS NOT NULL")). + Where(goqu.L("user_deposit_balance IS NOT NULL")) + + if len(dashboardId.Validators) == 0 { + ds = ds. + LeftJoin(goqu.L("users_val_dashboards_validators uvdv"), goqu.On(goqu.L("uvdv.validator_index = v.validatorindex"))). + Where(goqu.L("uvdv.dashboard_id = ?", dashboardId.Id)) + } else { + ds = ds. + Where(goqu.L("v.validatorindex = ANY(?)", pq.Array(dashboardId.Validators))) + } + + query, args, err := ds.Prepared(true).ToSQL() + if err != nil { + return nil, fmt.Errorf("error preparing query: %w", err) + } + + err = d.alloyReader.SelectContext(ctx, &rpOperatorInfo, query, args...) + if err != nil { + return nil, fmt.Errorf("error retrieving rocketpool validators data: %w", err) + } + return rpOperatorInfo, nil +} + +func (d *DataAccessService) calculateValidatorDashboardBalance(ctx context.Context, rpOperatorInfo []RpOperatorInfo, validators []t.VDBValidator, validatorMapping *services.ValidatorMapping, protocolModes t.VDBProtocolModes) (t.ValidatorBalances, error) { + balances := t.ValidatorBalances{} + + rpValidators := make(map[uint64]RpOperatorInfo) + for _, res := range rpOperatorInfo { + rpValidators[res.ValidatorIndex] = res + } + + // Create a new sub-dashboard to get the total cl deposits for non-rocketpool validators + var nonRpDashboardId t.VDBId + + for _, validator := range validators { + metadata := validatorMapping.ValidatorMetadata[validator] + validatorBalance := utils.GWeiToWei(big.NewInt(int64(metadata.Balance))) + effectiveBalance := utils.GWeiToWei(big.NewInt(int64(metadata.EffectiveBalance))) + + if rpValidator, ok := rpValidators[validator]; ok { + if protocolModes.RocketPool { + // Calculate the balance of the operator + fullDeposit := rpValidator.UserDepositBalance.Add(rpValidator.NodeDepositBalance) + operatorShare := rpValidator.NodeDepositBalance.Div(fullDeposit) + invOperatorShare := decimal.NewFromInt(1).Sub(operatorShare) + + base := decimal.Min(decimal.Max(decimal.Zero, validatorBalance.Sub(rpValidator.UserDepositBalance)), rpValidator.NodeDepositBalance) + commission := decimal.Max(decimal.Zero, validatorBalance.Sub(fullDeposit).Mul(invOperatorShare).Mul(decimal.NewFromFloat(rpValidator.NodeFee))) + reward := decimal.Max(decimal.Zero, validatorBalance.Sub(fullDeposit).Mul(operatorShare).Add(commission)) + + operatorBalance := base.Add(reward) + + balances.Total = balances.Total.Add(operatorBalance) + } else { + balances.Total = balances.Total.Add(validatorBalance) + } + balances.StakedEth = balances.StakedEth.Add(rpValidator.NodeDepositBalance) + } else { + balances.Total = balances.Total.Add(validatorBalance) + + nonRpDashboardId.Validators = append(nonRpDashboardId.Validators, validator) + } + balances.Effective = balances.Effective.Add(effectiveBalance) + } + + // Get the total cl deposits for non-rocketpool validators + if len(nonRpDashboardId.Validators) > 0 { + totalNonRpDeposits, err := d.GetValidatorDashboardTotalClDeposits(ctx, nonRpDashboardId) + if err != nil { + return balances, fmt.Errorf("error retrieving total cl deposits for non-rocketpool validators: %w", err) + } + balances.StakedEth = balances.StakedEth.Add(totalNonRpDeposits.TotalAmount) + } + return balances, nil +} diff --git a/backend/pkg/api/data_access/vdb_management.go b/backend/pkg/api/data_access/vdb_management.go index a9e955096..92772772b 100644 --- a/backend/pkg/api/data_access/vdb_management.go +++ b/backend/pkg/api/data_access/vdb_management.go @@ -364,92 +364,16 @@ func (d *DataAccessService) GetValidatorDashboardOverview(ctx context.Context, d } } - // Find rocketpool validators - type RpOperatorInfo struct { - ValidatorIndex uint64 `db:"validatorindex"` - NodeFee float64 `db:"node_fee"` - NodeDepositBalance decimal.Decimal `db:"node_deposit_balance"` - UserDepositBalance decimal.Decimal `db:"user_deposit_balance"` - } - var queryResult []RpOperatorInfo - - ds := goqu.Dialect("postgres"). - Select( - goqu.L("v.validatorindex"), - goqu.L("rplm.node_fee"), - goqu.L("rplm.node_deposit_balance"), - goqu.L("rplm.user_deposit_balance")). - From(goqu.L("rocketpool_minipools AS rplm")). - LeftJoin(goqu.L("validators AS v"), goqu.On(goqu.L("rplm.pubkey = v.pubkey"))). - Where(goqu.L("node_deposit_balance IS NOT NULL")). - Where(goqu.L("user_deposit_balance IS NOT NULL")) - - if len(dashboardId.Validators) == 0 { - ds = ds. - LeftJoin(goqu.L("users_val_dashboards_validators uvdv"), goqu.On(goqu.L("uvdv.validator_index = v.validatorindex"))). - Where(goqu.L("uvdv.dashboard_id = ?", dashboardId.Id)) - } else { - ds = ds. - Where(goqu.L("v.validatorindex = ANY(?)", pq.Array(dashboardId.Validators))) - } - - query, args, err := ds.Prepared(true).ToSQL() + rpOperatorInfo, err := d.getValidatorDashboardRpOperatorInfo(ctx, dashboardId) if err != nil { - return fmt.Errorf("error preparing query: %w", err) + return err } - err = d.alloyReader.SelectContext(ctx, &queryResult, query, args...) + balances, err := d.calculateValidatorDashboardBalance(ctx, rpOperatorInfo, validators, validatorMapping, protocolModes) if err != nil { - return fmt.Errorf("error retrieving rocketpool validators data: %w", err) - } - - rpValidators := make(map[uint64]RpOperatorInfo) - for _, res := range queryResult { - rpValidators[res.ValidatorIndex] = res - } - - // Create a new sub-dashboard to get the total cl deposits for non-rocketpool validators - var nonRpDashboardId t.VDBId - - for _, validator := range validators { - metadata := validatorMapping.ValidatorMetadata[validator] - validatorBalance := utils.GWeiToWei(big.NewInt(int64(metadata.Balance))) - effectiveBalance := utils.GWeiToWei(big.NewInt(int64(metadata.EffectiveBalance))) - - if rpValidator, ok := rpValidators[validator]; ok { - if protocolModes.RocketPool { - // Calculate the balance of the operator - fullDeposit := rpValidator.UserDepositBalance.Add(rpValidator.NodeDepositBalance) - operatorShare := rpValidator.NodeDepositBalance.Div(fullDeposit) - invOperatorShare := decimal.NewFromInt(1).Sub(operatorShare) - - base := decimal.Min(decimal.Max(decimal.Zero, validatorBalance.Sub(rpValidator.UserDepositBalance)), rpValidator.NodeDepositBalance) - commission := decimal.Max(decimal.Zero, validatorBalance.Sub(fullDeposit).Mul(invOperatorShare).Mul(decimal.NewFromFloat(rpValidator.NodeFee))) - reward := decimal.Max(decimal.Zero, validatorBalance.Sub(fullDeposit).Mul(operatorShare).Add(commission)) - - operatorBalance := base.Add(reward) - - data.Balances.Total = data.Balances.Total.Add(operatorBalance) - } else { - data.Balances.Total = data.Balances.Total.Add(validatorBalance) - } - data.Balances.StakedEth = data.Balances.StakedEth.Add(rpValidator.NodeDepositBalance) - } else { - data.Balances.Total = data.Balances.Total.Add(validatorBalance) - - nonRpDashboardId.Validators = append(nonRpDashboardId.Validators, validator) - } - data.Balances.Effective = data.Balances.Effective.Add(effectiveBalance) - } - - // Get the total cl deposits for non-rocketpool validators - if len(nonRpDashboardId.Validators) > 0 { - totalNonRpDeposits, err := d.GetValidatorDashboardTotalClDeposits(ctx, nonRpDashboardId) - if err != nil { - return fmt.Errorf("error retrieving total cl deposits for non-rocketpool validators: %w", err) - } - data.Balances.StakedEth = data.Balances.StakedEth.Add(totalNonRpDeposits.TotalAmount) + return err } + data.Balances = balances return nil }) diff --git a/backend/pkg/api/data_access/vdb_summary.go b/backend/pkg/api/data_access/vdb_summary.go index 8fc0ca9fc..f293e314a 100644 --- a/backend/pkg/api/data_access/vdb_summary.go +++ b/backend/pkg/api/data_access/vdb_summary.go @@ -521,11 +521,6 @@ func (d *DataAccessService) GetValidatorDashboardGroupSummary(ctx context.Contex return nil, err } - validators := make([]t.VDBValidator, 0) - if dashboardId.Validators != nil { - validators = dashboardId.Validators - } - getLastScheduledBlockAndSyncDate := func() (time.Time, time.Time, error) { // we need to go to the all time table for last scheduled block/sync committee epoch clickhouseTotalTable, _, err := d.getTablesForPeriod(enums.AllTime) @@ -546,7 +541,7 @@ func (d *DataAccessService) GetValidatorDashboardGroupSummary(ctx context.Contex Where(goqu.L("validator_index IN (SELECT validator_index FROM validators)")) } else { ds = ds. - Where(goqu.L("validator_index IN ?", validators)) + Where(goqu.L("validator_index IN ?", dashboardId.Validators)) } query, args, err := ds.Prepared(true).ToSQL() @@ -603,7 +598,7 @@ func (d *DataAccessService) GetValidatorDashboardGroupSummary(ctx context.Contex Where(goqu.L("validator_index IN (SELECT validator_index FROM validators)")) } else { ds = ds. - Where(goqu.L("validator_index IN ?", validators)) + Where(goqu.L("validator_index IN ?", dashboardId.Validators)) } type QueryResult struct { @@ -688,9 +683,9 @@ func (d *DataAccessService) GetValidatorDashboardGroupSummary(ctx context.Contex totalMissedRewardsAttestations := int64(0) totalMissedRewardsSync := int64(0) - validatorArr := make([]t.VDBValidator, 0) + validators := make([]t.VDBValidator, 0) for _, row := range rows { - validatorArr = append(validatorArr, t.VDBValidator(row.ValidatorIndex)) + validators = append(validators, t.VDBValidator(row.ValidatorIndex)) totalAttestationRewards += row.AttestationReward totalIdealAttestationRewards += row.AttestationsIdealReward @@ -765,13 +760,9 @@ func (d *DataAccessService) GetValidatorDashboardGroupSummary(ctx context.Contex return nil, err } - if len(validators) > 0 { - validatorArr = validators - } - pastSyncPeriodCutoff := utils.SyncPeriodOfEpoch(rows[0].EpochStart) currentSyncPeriod := utils.SyncPeriodOfEpoch(latestEpoch) - err = d.readerDb.GetContext(ctx, &ret.SyncCommitteeCount.PastPeriods, `SELECT COUNT(*) FROM sync_committees WHERE period >= $1 AND period < $2 AND validatorindex = ANY($3)`, pastSyncPeriodCutoff, currentSyncPeriod, validatorArr) + err = d.readerDb.GetContext(ctx, &ret.SyncCommitteeCount.PastPeriods, `SELECT COUNT(*) FROM sync_committees WHERE period >= $1 AND period < $2 AND validatorindex = ANY($3)`, pastSyncPeriodCutoff, currentSyncPeriod, validators) if err != nil { return nil, fmt.Errorf("error retrieving past sync committee count: %w", err) } @@ -853,6 +844,20 @@ func (d *DataAccessService) GetValidatorDashboardGroupSummary(ctx context.Contex } ret.Efficiency = utils.CalculateTotalEfficiency(attestationEfficiency, proposerEfficiency, syncEfficiency) + rpOperatorInfo, err := d.getValidatorDashboardRpOperatorInfo(ctx, dashboardId) + if err != nil { + return nil, err + } + validatorMapping, err := d.services.GetCurrentValidatorMapping() + if err != nil { + return nil, err + } + balances, err := d.calculateValidatorDashboardBalance(ctx, rpOperatorInfo, validators, validatorMapping, protocolModes) + if err != nil { + return nil, err + } + ret.Balances = balances + return ret, nil } From 9fc153bc2df6e91f8f9e2ab3321459a5046fb53a Mon Sep 17 00:00:00 2001 From: Lucca Dukic <109136188+LuccaBitfly@users.noreply.github.com> Date: Thu, 9 Jan 2025 14:09:30 +0100 Subject: [PATCH 5/6] feat: fill rewards field in dashboard group summary See: BEDS-1065 --- backend/pkg/api/data_access/vdb_summary.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/pkg/api/data_access/vdb_summary.go b/backend/pkg/api/data_access/vdb_summary.go index f293e314a..4e911f018 100644 --- a/backend/pkg/api/data_access/vdb_summary.go +++ b/backend/pkg/api/data_access/vdb_summary.go @@ -755,7 +755,7 @@ func (d *DataAccessService) GetValidatorDashboardGroupSummary(ctx context.Contex ret.MissedRewards.Sync = utils.GWeiToWei(big.NewInt(totalMissedRewardsSync)) ret.MissedRewards.ProposerRewards.Cl = utils.GWeiToWei(big.NewInt(totalMissedRewardsCl)) - _, ret.Apr.El, _, ret.Apr.Cl, err = d.getElClAPR(ctx, dashboardId, groupId, hours) + ret.Rewards.El, ret.Apr.El, ret.Rewards.Cl, ret.Apr.Cl, err = d.getElClAPR(ctx, dashboardId, groupId, hours) if err != nil { return nil, err } From 02a0a6c8bc07b53ec4c2e369aceef6cf716390e5 Mon Sep 17 00:00:00 2001 From: Lucca Dukic <109136188+LuccaBitfly@users.noreply.github.com> Date: Thu, 9 Jan 2025 14:10:35 +0100 Subject: [PATCH 6/6] chore: convert typescript See: BEDS-1065 --- frontend/types/api/validator_dashboard.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/frontend/types/api/validator_dashboard.ts b/frontend/types/api/validator_dashboard.ts index 9d557abef..5128ee519 100644 --- a/frontend/types/api/validator_dashboard.ts +++ b/frontend/types/api/validator_dashboard.ts @@ -10,7 +10,7 @@ export interface VDBOverviewGroup { name: string; count: number /* uint64 */; } -export interface VDBOverviewBalances { +export interface ValidatorBalances { total: string /* decimal.Decimal */; effective: string /* decimal.Decimal */; staked_eth: string /* decimal.Decimal */; @@ -24,7 +24,7 @@ export interface VDBOverviewData { rewards: PeriodicValues>; apr: PeriodicValues>; chart_history_seconds: ChartHistorySeconds; - balances: VDBOverviewBalances; + balances: ValidatorBalances; } export type GetValidatorDashboardResponse = ApiDataResponse; export interface VDBPostArchivingReturnData { @@ -68,6 +68,9 @@ export interface VDBGroupSummaryMissedRewards { sync: string /* decimal.Decimal */; } export interface VDBGroupSummaryData { + efficiency: number /* float64 */; + balances: ValidatorBalances; + rewards: ClElValue; attestations_head: StatusCount; attestations_source: StatusCount; attestations_target: StatusCount;