Beds 1056/postman ci cd

.github/workflows/backend-integration-test.yml

@@ -33,6 +33,9 @@ jobs:
           cache-dependency-path: 'backend/go.sum'
       - name: Test with the Go CLI
         working-directory: backend
+        env:
+          CLICKHOUSE_WRITER_DB_NAME: holesky_re_export_2024_11_09
+          CLICKHOUSE_READER_DB_NAME: holesky_re_export_2024_11_09
         run:
           go install github.com/swaggo/swag/cmd/swag@latest && swag init --ot json -o ./pkg/api/docs -d ./pkg/api/ -g ./handlers/public.go
           go test -failfast ./pkg/api/... -config "${{ secrets.CI_CONFIG_PATH }}"

.github/workflows/backend-postman.yml

@@ -0,0 +1,45 @@
+name: Backend and Postman Tests
+
+on: [push, pull_request]
+
+jobs:
+  backend-and-postman-tests:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: '1.20'
+
+    - name: Set up Node.js
+      uses: actions/setup-node@v3
+      with:
+        node-version: '16'
+
+    - name: Install Newman
+      run: npm install -g newman
+
+    - name: Start Backend in Background
+      working-directory: backend
+      env:
+        CLICKHOUSE_WRITER_DB_NAME: holesky_re_export_2024_11_09
+        CLICKHOUSE_READER_DB_NAME: holesky_re_export_2024_11_09
+        CONFIG_PATH: ${{ secrets.CI_CONFIG_PATH }}
+      run: |
+        nohup go run cmd/main.go api -config="${CONFIG_PATH}" > backend.log 2>&1 &
+        sleep 60 # Wait for the backend to initialize
+        newman run postman/api.json \
+          --reporters cli,junit \
+          --reporter-junit-export results.xml
+        pkill -f "go run cmd/main.go"
+
+    - name: Upload Newman JUnit Report
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: newman-junit-report
+        path: results.xml

.github/workflows/backend-publish-docker.yml

@@ -11,6 +11,7 @@ on:
       - main
       - staging
       - staging-2
+      - publish-docker
 
 # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
 env:

.github/workflows/frontend-publish-docker.yml

@@ -11,6 +11,7 @@ on:
       - main
       - staging
       - staging-2
+      - publish-docker
 
 # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
 env:

backend/pkg/api/data_access/vdb_blocks.go

@@ -361,37 +361,22 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das
 		Slot     uint64              `db:"slot"`
 		ClReward decimal.NullDecimal `db:"cl_reward"`
 	}{}
-	if utils.Config.Chain.ClConfig.DepositChainID == 17000 {
-		clRewardsQuery := goqu.Dialect("postgres").
-			From(goqu.T("consensus_payloads")).
-			Select(
-				goqu.C("slot"),
-				goqu.L("cl_attestations_reward / 1e9 + cl_sync_aggregate_reward / 1e9 + cl_slashing_inclusion_reward / 1e9 AS cl_reward"),
-			).Where(goqu.C("slot").In(slots))
-		clRewardsQuerySql, args, err := clRewardsQuery.Prepared(true).ToSQL()
-		if err != nil {
-			return nil, nil, err
-		}
-		err = d.alloyReader.SelectContext(ctx, &clRewardsData, clRewardsQuerySql, args...)
-		if err != nil {
-			return nil, nil, err
-		}
-	} else {
-		clRewardsQuery := goqu.Dialect("postgres").
-			From(goqu.L("mainnet.validator_proposal_rewards_slot")).
-			Select(
-				goqu.C("slot"),
-				goqu.L("attestations_reward / 1e9 + sync_aggregate_reward / 1e9 + slasher_reward / 1e9 AS cl_reward"),
-			).Where(goqu.C("slot").In(slots))
-		clRewardsQuerySql, args, err := clRewardsQuery.Prepared(true).ToSQL()
-		if err != nil {
-			return nil, nil, err
-		}
-		err = d.clickhouseReader.SelectContext(ctx, &clRewardsData, clRewardsQuerySql, args...)
-		if err != nil {
-			return nil, nil, err
-		}
+
+	clRewardsQuery := goqu.Dialect("postgres").
+		From(goqu.L("validator_proposal_rewards_slot")).
+		Select(
+			goqu.C("slot"),
+			goqu.L("attestations_reward / 1e9 + sync_aggregate_reward / 1e9 + slasher_reward / 1e9 AS cl_reward"),
+		).Where(goqu.C("slot").In(slots))
+	clRewardsQuerySql, args, err := clRewardsQuery.Prepared(true).ToSQL()
+	if err != nil {
+		return nil, nil, err
+	}
+	err = d.clickhouseReader.SelectContext(ctx, &clRewardsData, clRewardsQuerySql, args...)
+	if err != nil {
+		return nil, nil, err
 	}
+
 	clRewards := make(map[uint64]decimal.NullDecimal)
 	for _, reward := range clRewardsData {
 		clRewards[reward.Slot] = reward.ClReward

backend/pkg/api/data_access/vdb_rewards.go

@@ -86,9 +86,9 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da
 		With("validators", goqu.L("(SELECT validator_index as validator_index, group_id FROM users_val_dashboards_validators WHERE dashboard_id = ?)", dashboardId.Id)).
 		Select(
 			goqu.L("e.epoch"),
-			goqu.L(`SUM(COALESCE(e.attestations_reward, 0) + COALESCE(e.blocks_cl_reward, 0) + COALESCE(e.sync_rewards, 0)) AS cl_rewards`),
+			goqu.L(`SUM(COALESCE(e.attestations_reward, 0) + COALESCE(e.blocks_cl_reward, 0) + COALESCE(e.sync_reward, 0)) AS cl_rewards`),
 			goqu.L("SUM(COALESCE(e.attestations_scheduled, 0)) AS attestations_scheduled"),
-			goqu.L("SUM(COALESCE(e.attestations_executed, 0)) AS attestations_executed"),
+			goqu.L("SUM(COALESCE(e.attestations_observed, 0)) AS attestations_observed"),
 			goqu.L("SUM(COALESCE(e.blocks_scheduled, 0)) AS blocks_scheduled"),
 			goqu.L("SUM(COALESCE(e.blocks_proposed, 0)) AS blocks_proposed"),
 			goqu.L("SUM(COALESCE(e.sync_scheduled, 0)) AS sync_scheduled"),
@@ -300,7 +300,7 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da
 		GroupId               int64  `db:"result_group_id"`
 		ClRewards             int64  `db:"cl_rewards"`
 		AttestationsScheduled uint64 `db:"attestations_scheduled"`
-		AttestationsExecuted  uint64 `db:"attestations_executed"`
+		AttestationsExecuted  uint64 `db:"attestations_observed"`
 		BlocksScheduled       uint64 `db:"blocks_scheduled"`
 		BlocksProposed        uint64 `db:"blocks_proposed"`
 		SyncScheduled         uint64 `db:"sync_scheduled"`
@@ -539,15 +539,15 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex
 			goqu.L("COALESCE(e.attestations_inactivity_reward, 0) AS attestations_inactivity_reward"),
 			goqu.L("COALESCE(e.attestations_inclusion_reward, 0) AS attestations_inclusion_reward"),
 			goqu.L("COALESCE(e.attestations_scheduled, 0) AS attestations_scheduled"),
-			goqu.L("COALESCE(e.attestation_head_executed, 0) AS attestation_head_executed"),
-			goqu.L("COALESCE(e.attestation_source_executed, 0) AS attestation_source_executed"),
-			goqu.L("COALESCE(e.attestation_target_executed, 0) AS attestation_target_executed"),
+			goqu.L("COALESCE(e.attestations_head_executed, 0) AS attestations_head_executed"),
+			goqu.L("COALESCE(e.attestations_source_executed, 0) AS attestations_source_executed"),
+			goqu.L("COALESCE(e.attestations_target_executed, 0) AS attestations_target_executed"),
 			goqu.L("COALESCE(e.blocks_scheduled, 0) AS blocks_scheduled"),
 			goqu.L("COALESCE(e.blocks_proposed, 0) AS blocks_proposed"),
 			goqu.L("COALESCE(e.blocks_cl_reward, 0) AS blocks_cl_reward"),
 			goqu.L("COALESCE(e.sync_scheduled, 0) AS sync_scheduled"),
 			goqu.L("COALESCE(e.sync_executed, 0) AS sync_executed"),
-			goqu.L("COALESCE(e.sync_rewards, 0) AS sync_rewards"),
+			goqu.L("COALESCE(e.sync_reward, 0) AS sync_reward"),
 			goqu.L("(CASE WHEN e.slashed THEN 1 ELSE 0 END) AS slashed_in_epoch"),
 			goqu.L("COALESCE(e.blocks_slashing_count, 0) AS slashed_amount"),
 			goqu.L("COALESCE(e.blocks_cl_slasher_reward, 0) AS slasher_reward"),
@@ -601,18 +601,18 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex
 		AttestationInactivitytReward decimal.Decimal `db:"attestations_inactivity_reward"`
 		AttestationInclusionReward   decimal.Decimal `db:"attestations_inclusion_reward"`
 
-		AttestationsScheduled     int64 `db:"attestations_scheduled"`
-		AttestationHeadExecuted   int64 `db:"attestation_head_executed"`
-		AttestationSourceExecuted int64 `db:"attestation_source_executed"`
-		AttestationTargetExecuted int64 `db:"attestation_target_executed"`
+		AttestationsScheduled      int64 `db:"attestations_scheduled"`
+		AttestationsHeadExecuted   int64 `db:"attestations_head_executed"`
+		AttestationsSourceExecuted int64 `db:"attestations_source_executed"`
+		AttestationsTargetExecuted int64 `db:"attestations_target_executed"`
 
 		BlocksScheduled uint32          `db:"blocks_scheduled"`
 		BlocksProposed  uint32          `db:"blocks_proposed"`
 		BlocksClReward  decimal.Decimal `db:"blocks_cl_reward"`
 
 		SyncScheduled uint32          `db:"sync_scheduled"`
 		SyncExecuted  uint32          `db:"sync_executed"`
-		SyncRewards   decimal.Decimal `db:"sync_rewards"`
+		SyncReward    decimal.Decimal `db:"sync_reward"`
 
 		SlashedInEpoch bool            `db:"slashed_in_epoch"`
 		SlashedAmount  uint32          `db:"slashed_amount"`
@@ -662,16 +662,16 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex
 
 	for _, entry := range queryResult {
 		ret.AttestationsHead.Income = ret.AttestationsHead.Income.Add(entry.AttestationHeadReward.Mul(gWei))
-		ret.AttestationsHead.StatusCount.Success += uint64(entry.AttestationHeadExecuted)
-		ret.AttestationsHead.StatusCount.Failed += uint64(entry.AttestationsScheduled) - uint64(entry.AttestationHeadExecuted)
+		ret.AttestationsHead.StatusCount.Success += uint64(entry.AttestationsHeadExecuted)
+		ret.AttestationsHead.StatusCount.Failed += uint64(entry.AttestationsScheduled) - uint64(entry.AttestationsHeadExecuted)
 
 		ret.AttestationsSource.Income = ret.AttestationsSource.Income.Add(entry.AttestationSourceReward.Mul(gWei))
-		ret.AttestationsSource.StatusCount.Success += uint64(entry.AttestationSourceExecuted)
-		ret.AttestationsSource.StatusCount.Failed += uint64(entry.AttestationsScheduled) - uint64(entry.AttestationSourceExecuted)
+		ret.AttestationsSource.StatusCount.Success += uint64(entry.AttestationsSourceExecuted)
+		ret.AttestationsSource.StatusCount.Failed += uint64(entry.AttestationsScheduled) - uint64(entry.AttestationsSourceExecuted)
 
 		ret.AttestationsTarget.Income = ret.AttestationsTarget.Income.Add(entry.AttestationTargetReward.Mul(gWei))
-		ret.AttestationsTarget.StatusCount.Success += uint64(entry.AttestationTargetExecuted)
-		ret.AttestationsTarget.StatusCount.Failed += uint64(entry.AttestationsScheduled) - uint64(entry.AttestationTargetExecuted)
+		ret.AttestationsTarget.StatusCount.Success += uint64(entry.AttestationsTargetExecuted)
+		ret.AttestationsTarget.StatusCount.Failed += uint64(entry.AttestationsScheduled) - uint64(entry.AttestationsTargetExecuted)
 
 		ret.Inactivity.Income = ret.Inactivity.Income.Add(entry.AttestationInactivitytReward.Mul(gWei))
 		if entry.AttestationInactivitytReward.LessThan(decimal.Zero) {
@@ -684,7 +684,7 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex
 		ret.Proposal.StatusCount.Success += uint64(entry.BlocksProposed)
 		ret.Proposal.StatusCount.Failed += uint64(entry.BlocksScheduled) - uint64(entry.BlocksProposed)
 
-		ret.Sync.Income = ret.Sync.Income.Add(entry.SyncRewards.Mul(gWei))
+		ret.Sync.Income = ret.Sync.Income.Add(entry.SyncReward.Mul(gWei))
 		ret.Sync.StatusCount.Success += uint64(entry.SyncExecuted)
 		ret.Sync.StatusCount.Failed += uint64(entry.SyncScheduled) - uint64(entry.SyncExecuted)
 
@@ -725,7 +725,7 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex
 	rewardsDs := goqu.Dialect("postgres").
 		Select(
 			goqu.L("e.epoch"),
-			goqu.L(`SUM(COALESCE(e.attestations_reward, 0) + COALESCE(e.blocks_cl_reward, 0) + COALESCE(e.sync_rewards, 0)) AS cl_rewards`)).
+			goqu.L(`SUM(COALESCE(e.attestations_reward, 0) + COALESCE(e.blocks_cl_reward, 0) + COALESCE(e.sync_reward, 0)) AS cl_rewards`)).
 		From(goqu.L("validator_dashboard_data_epoch e")).
 		With("validators", goqu.L("(SELECT validator_index as validator_index, group_id FROM users_val_dashboards_validators WHERE dashboard_id = ?)", dashboardId.Id)).
 		Where(goqu.L("e.epoch_timestamp >= fromUnixTimestamp(?)", utils.EpochToTime(startEpoch).Unix()))
@@ -958,15 +958,15 @@ func (d *DataAccessService) GetValidatorDashboardDuties(ctx context.Context, das
 		Select(
 			goqu.L("e.validator_index"),
 			goqu.L("COALESCE(e.attestations_scheduled, 0) AS attestations_scheduled"),
-			goqu.L("COALESCE(e.attestation_source_executed, 0) AS attestation_source_executed"),
+			goqu.L("COALESCE(e.attestations_source_executed, 0) AS attestations_source_executed"),
 			goqu.L("COALESCE(e.attestations_source_reward, 0) AS attestations_source_reward"),
-			goqu.L("COALESCE(e.attestation_target_executed, 0) AS attestation_target_executed"),
+			goqu.L("COALESCE(e.attestations_target_executed, 0) AS attestations_target_executed"),
 			goqu.L("COALESCE(e.attestations_target_reward, 0) AS attestations_target_reward"),
-			goqu.L("COALESCE(e.attestation_head_executed, 0) AS attestation_head_executed"),
+			goqu.L("COALESCE(e.attestations_head_executed, 0) AS attestations_head_executed"),
 			goqu.L("COALESCE(e.attestations_head_reward, 0) AS attestations_head_reward"),
 			goqu.L("COALESCE(e.sync_scheduled, 0) AS sync_scheduled"),
 			goqu.L("COALESCE(e.sync_executed, 0) AS sync_executed"),
-			goqu.L("COALESCE(e.sync_rewards, 0) AS sync_rewards"),
+			goqu.L("COALESCE(e.sync_reward, 0) AS sync_reward"),
 			goqu.L("e.slashed AS slashed_in_epoch"),
 			goqu.L("COALESCE(e.blocks_slashing_count, 0) AS slashed_amount"),
 			goqu.L("COALESCE(e.blocks_cl_slasher_reward, 0) AS slasher_reward"),
@@ -1044,15 +1044,15 @@ func (d *DataAccessService) GetValidatorDashboardDuties(ctx context.Context, das
 	queryResult := []struct {
 		ValidatorIndex              uint64 `db:"validator_index"`
 		AttestationsScheduled       uint64 `db:"attestations_scheduled"`
-		AttestationsSourceExecuted  uint64 `db:"attestation_source_executed"`
+		AttestationsSourceExecuted  uint64 `db:"attestations_source_executed"`
 		AttestationsSourceReward    int64  `db:"attestations_source_reward"`
-		AttestationsTargetExecuted  uint64 `db:"attestation_target_executed"`
+		AttestationsTargetExecuted  uint64 `db:"attestations_target_executed"`
 		AttestationsTargetReward    int64  `db:"attestations_target_reward"`
-		AttestationsHeadExecuted    uint64 `db:"attestation_head_executed"`
+		AttestationsHeadExecuted    uint64 `db:"attestations_head_executed"`
 		AttestationsHeadReward      int64  `db:"attestations_head_reward"`
 		SyncScheduled               uint64 `db:"sync_scheduled"`
 		SyncExecuted                uint64 `db:"sync_executed"`
-		SyncRewards                 int64  `db:"sync_rewards"`
+		SyncRewards                 int64  `db:"sync_reward"`
 		SlashedInEpoch              bool   `db:"slashed_in_epoch"`
 		SlashedAmount               uint64 `db:"slashed_amount"`
 		SlasherReward               int64  `db:"slasher_reward"`