From 359e9983c6bee925517f981a2dc0b7dc11689472 Mon Sep 17 00:00:00 2001 From: Gianmarco Fraccaroli Date: Mon, 20 May 2024 12:02:43 +0200 Subject: [PATCH 1/8] search proprosals by text --- swagger.yml | 48 ++++++++++++++++++++++++-- webserver/src/error/governance.rs | 3 ++ webserver/src/handler/governance.rs | 19 ++++++++++ webserver/src/repository/governance.rs | 32 +++++++++++++++-- webserver/src/service/governance.rs | 21 +++++++++++ 5 files changed, 119 insertions(+), 4 deletions(-) diff --git a/swagger.yml b/swagger.yml index 234d9f915..9c33e1f6a 100644 --- a/swagger.yml +++ b/swagger.yml @@ -139,7 +139,7 @@ paths: name: status schema: type: string - enum: [pending, voting, ended] + enum: [pending, voting, passed, rejected] description: The status of the proposal responses: '200': @@ -168,6 +168,50 @@ paths: total_items: type: integer minimum: 0 + /gov/search/{text}: + get: + summary: Get a list of governance proposals matching a text in the title + parameters: + - in: query + name: page + schema: + type: integer + minimum: 0 + description: Pagination parameter + - in: path + name: text + schema: + type: string + minimum: 3 + required: true + description: The text to match against the proposal title + responses: + '200': + description: A list of governance proposal matching the text. + content: + application/json: + schema: + type: object + properties: + data: + type: array + items: + $ref: '#/components/schemas/Proposal' + pagination: + type: object + properties: + page: + type: integer + minimum: 0 + per_page: + type: integer + minimum: 0 + total_pages: + type: integer + minimum: 0 + total_items: + type: integer + minimum: 0 /gov/proposal/{id}: get: summary: Get a governance proposal by proposal id @@ -316,7 +360,7 @@ components: type: integer status: type: string - enum: [pending, voting, ended] + enum: [pending, voting, passed, rejected] yayVotes: type: integer nayVotes: diff --git a/webserver/src/error/governance.rs b/webserver/src/error/governance.rs index ff48c5599..5d6252477 100644 --- a/webserver/src/error/governance.rs +++ b/webserver/src/error/governance.rs @@ -6,6 +6,8 @@ use crate::response::api::ApiErrorResponse; #[derive(Error, Debug)] pub enum GovernanceError { + #[error("Too Short pattern, minimum character 3, got {0}")] + TooShortPattern(usize), #[error("Proposal {0} not found")] NotFound(u64), #[error("Database error: {0}")] @@ -17,6 +19,7 @@ pub enum GovernanceError { impl IntoResponse for GovernanceError { fn into_response(self) -> Response { let status_code = match self { + GovernanceError::TooShortPattern(_) => StatusCode::BAD_REQUEST, GovernanceError::NotFound(_) => StatusCode::NOT_FOUND, GovernanceError::Unknown(_) | GovernanceError::Database(_) => { StatusCode::INTERNAL_SERVER_ERROR diff --git a/webserver/src/handler/governance.rs b/webserver/src/handler/governance.rs index 021647d80..013f4f8f5 100644 --- a/webserver/src/handler/governance.rs +++ b/webserver/src/handler/governance.rs @@ -47,6 +47,25 @@ pub async fn get_governance_proposal_by_id( } } +#[debug_handler] +pub async fn search_governance_proposals_by_pattern( + _trace_id: TraceId, + _headers: HeaderMap, + Path(pattern): Path, + State(state): State, + Query(page): Query>, +) -> Result>>, ApiError> { + let page = page.unwrap_or(0); + + let (proposals, total_proposals) = state + .gov_service + .search_governance_proposals_by_pattern(pattern, page) + .await?; + + let response = PaginatedResponse::new(proposals, page, total_proposals); + Ok(Json(response)) +} + #[debug_handler] pub async fn get_governance_proposal_votes( _trace_id: TraceId, diff --git a/webserver/src/repository/governance.rs b/webserver/src/repository/governance.rs index 19e49240b..3528437a9 100644 --- a/webserver/src/repository/governance.rs +++ b/webserver/src/repository/governance.rs @@ -1,7 +1,7 @@ use axum::async_trait; use diesel::{ - BoolExpressionMethods, ExpressionMethods, QueryDsl, RunQueryDsl, - SelectableHelper, + BoolExpressionMethods, ExpressionMethods, PgTextExpressionMethods, + QueryDsl, RunQueryDsl, SelectableHelper, }; use orm::governance_proposal::{ GovernanceProposalDb, GovernanceProposalResultDb, @@ -32,6 +32,12 @@ pub trait GovernanceRepoTrait { proposal_id: i32, ) -> Result, String>; + async fn search_governance_proposals_by_pattern( + &self, + pattern: String, + page: i64, + ) -> Result<(Vec, i64), String>; + async fn find_governance_proposal_votes( &self, proposal_id: i32, @@ -87,6 +93,28 @@ impl GovernanceRepoTrait for GovernanceRepo { .map_err(|e| e.to_string()) } + async fn search_governance_proposals_by_pattern( + &self, + pattern: String, + page: i64, + ) -> Result<(Vec, i64), String> { + let conn = self.app_state.get_db_connection().await; + + conn.interact(move |conn| { + governance_proposals::table + .filter( + governance_proposals::dsl::content + .ilike(format!("%{}%", pattern)), + ) + .select(GovernanceProposalDb::as_select()) + .paginate(page) + .load_and_count_pages(conn) + .unwrap() + }) + .await + .map_err(|e| e.to_string()) + } + async fn find_governance_proposal_votes( &self, proposal_id: i32, diff --git a/webserver/src/service/governance.rs b/webserver/src/service/governance.rs index b67d6c740..e060b3577 100644 --- a/webserver/src/service/governance.rs +++ b/webserver/src/service/governance.rs @@ -70,6 +70,27 @@ impl GovernanceService { Ok(db_proposal.map(Proposal::from)) } + pub async fn search_governance_proposals_by_pattern( + &self, + pattern: String, + page: u64, + ) -> Result<(Vec, u64), GovernanceError> { + if pattern.len() < 3 { + return Err(GovernanceError::TooShortPattern(pattern.len())); + } + + let (db_proposals, total_items) = self + .governance_repo + .search_governance_proposals_by_pattern(pattern, page as i64) + .await + .map_err(GovernanceError::Database)?; + + Ok(( + db_proposals.into_iter().map(Proposal::from).collect(), + total_items as u64, + )) + } + pub async fn find_governance_proposal_votes( &self, proposal_id: u64, From 0f947596b6ea273993a27daffcbd94bd6743053d Mon Sep 17 00:00:00 2001 From: Gianmarco Fraccaroli Date: Mon, 20 May 2024 12:24:24 +0200 Subject: [PATCH 2/8] minor swagger fix --- swagger.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swagger.yml b/swagger.yml index 9c33e1f6a..259169afa 100644 --- a/swagger.yml +++ b/swagger.yml @@ -220,7 +220,7 @@ paths: name: id schema: type: integer - minimum: 1 + minimum: 0 required: true description: Proposal id responses: From 9a5adf71b6f51e8236e46db476f96a89a49b6eb3 Mon Sep 17 00:00:00 2001 From: Gianmarco Fraccaroli Date: Mon, 20 May 2024 15:54:19 +0200 Subject: [PATCH 3/8] wip --- chain/run.sh | 2 +- chain/src/main.rs | 13 ++++++++-- docker-compose.yml | 3 +-- governance/run.sh | 2 +- governance/src/main.rs | 10 +++++++- .../2024-05-09-042930_init_unbonds/up.sql | 2 +- pos/run.sh | 2 +- rewards/run.sh | 2 ++ rewards/src/main.rs | 10 +++++++- shared/src/transaction.rs | 2 ++ swagger.yml | 24 +++++++++---------- webserver/run.sh | 2 +- webserver/src/app.rs | 21 +++++++++------- webserver/src/dto/governance.rs | 2 ++ webserver/src/dto/mod.rs | 1 + webserver/src/dto/pos.rs | 10 ++++++++ webserver/src/handler/governance.rs | 11 ++++----- webserver/src/handler/pos.rs | 4 +++- 18 files changed, 85 insertions(+), 38 deletions(-) create mode 100755 rewards/run.sh diff --git a/chain/run.sh b/chain/run.sh index 2bc4409ac..9afc34d16 100755 --- a/chain/run.sh +++ b/chain/run.sh @@ -1,2 +1,2 @@ -cargo run -- --tendermint-url http://127.0.0.1:27657 --checksums-filepath ../artifacts/checksums.json --chain-id localnet.a38cf62f63db8c1a1f3c9 --database-url postgres://postgres:password@0.0.0.0:5435/indexer_local +cargo run -- --tendermint-url http://127.0.0.1:27657 --chain-id local.988b6b47aadd23b12b8a9a03 --database-url postgres://postgres:password@0.0.0.0:5435/namada-indexer diff --git a/chain/src/main.rs b/chain/src/main.rs index 8c13c8301..f8f592978 100644 --- a/chain/src/main.rs +++ b/chain/src/main.rs @@ -18,6 +18,7 @@ use clap::Parser; use clap_verbosity_flag::LevelFilter; use deadpool_diesel::postgres::Object; use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl}; +use orm::migrations::run_migrations; use orm::schema::pos_rewards; use shared::block::Block; use shared::block_result::BlockResult; @@ -44,7 +45,7 @@ async fn main() -> Result<(), MainError> { .unwrap_or_else(|| { panic!("{} must be defined in namada storage.", code_path) }); - checksums.add(code_path, code); + checksums.add(code_path, code.to_lowercase()); } let log_level = match config.verbosity.log_level_filter() { @@ -65,6 +66,12 @@ async fn main() -> Result<(), MainError> { let app_state = AppState::new(config.database_url).into_db_error()?; let conn = Arc::new(app_state.get_db_connection().await.into_db_error()?); + + // Run migrations + run_migrations(&conn) + .await + .context_db_interact_error() + .into_db_error()?; initial_query(&client, &conn).await?; @@ -227,6 +234,8 @@ async fn initial_query( let pos_crawler_epoch = get_pos_crawler_state(conn).await.into_db_error(); + println!("{:?}, {:?}", epoch, pos_crawler_epoch); + match pos_crawler_epoch { Ok(pos_crawler_epoch) if pos_crawler_epoch.epoch == epoch => { break; @@ -236,7 +245,7 @@ async fn initial_query( tracing::info!("Waiting for PoS service update..."); - sleep(Duration::from_secs(5)).await; + sleep(Duration::from_secs(1)).await; } let balances = query_all_balances(client).await.into_rpc_error()?; diff --git a/docker-compose.yml b/docker-compose.yml index f27be8534..0b7e1f0fe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,8 +13,7 @@ services: environment: DATABASE_URL: postgres://postgres:password@postgres:5435/namada-indexer TENDERMINT_URL: http://localhost:27657 - CHAIN_ID: local.1321312131 - CHECKSUMS_FILE: /app/checksums.json + CHAIN_ID: local.988b6b47aadd23b12b8a9a03 governance: restart: on-failure diff --git a/governance/run.sh b/governance/run.sh index 1c272e878..d4bc7ea96 100755 --- a/governance/run.sh +++ b/governance/run.sh @@ -1,2 +1,2 @@ -cargo run -- --cargo-env development --tendermint-url http://127.0.0.1:27657 --database-url postgres://postgres:password@0.0.0.0:5435/indexer_local +cargo run -- --tendermint-url http://127.0.0.1:27657 --database-url postgres://postgres:password@0.0.0.0:5435/namada-indexer --sleep-for 15 diff --git a/governance/src/main.rs b/governance/src/main.rs index ca34d016c..73c8ce0df 100644 --- a/governance/src/main.rs +++ b/governance/src/main.rs @@ -9,8 +9,9 @@ use governance::config::AppConfig; use governance::services::{db as db_service, namada as namada_service}; use governance::state::AppState; use orm::governance_proposal::GovernanceProposalUpdateStatusDb; +use orm::migrations::run_migrations; use orm::schema::governance_proposals; -use shared::error::{AsDbError, AsRpcError, MainError}; +use shared::error::{AsDbError, AsRpcError, ContextDbInteractError, MainError}; use tendermint_rpc::HttpClient; use tokio::signal; use tokio::time::sleep; @@ -45,6 +46,13 @@ async fn main() -> anyhow::Result<()> { let app_state = AppState::new(config.database_url).into_db_error()?; + let conn = Arc::new(app_state.get_db_connection().await.into_db_error()?); + // Run migrations + run_migrations(&conn) + .await + .context_db_interact_error() + .into_db_error()?; + let retry_strategy = FixedInterval::from_millis(5000).map(jitter); let exit_handle = must_exit_handle(); diff --git a/orm/migrations/2024-05-09-042930_init_unbonds/up.sql b/orm/migrations/2024-05-09-042930_init_unbonds/up.sql index 9326b9495..316f99a01 100644 --- a/orm/migrations/2024-05-09-042930_init_unbonds/up.sql +++ b/orm/migrations/2024-05-09-042930_init_unbonds/up.sql @@ -9,6 +9,6 @@ CREATE TABLE unbonds ( CONSTRAINT fk_validator_id FOREIGN KEY(validator_id) REFERENCES validators(id) ON DELETE CASCADE ); -ALTER TABLE unbonds ADD UNIQUE (address, validator_id, withdraw_epoch, epoch); +ALTER TABLE unbonds ADD UNIQUE (address, validator_id, withdraw_epoch); CREATE INDEX index_unbonds_owner_withdraw_epoch ON unbonds (address, withdraw_epoch); diff --git a/pos/run.sh b/pos/run.sh index 671fe11cd..1b5ea3fb8 100755 --- a/pos/run.sh +++ b/pos/run.sh @@ -1,2 +1,2 @@ -cargo run -- --tendermint-url http://127.0.0.1:27657 --database-url postgres://postgres:password@0.0.0.0:5435/indexer_local +cargo run -- --tendermint-url http://127.0.0.1:27657 --database-url postgres://postgres:password@0.0.0.0:5435/namada-indexer diff --git a/rewards/run.sh b/rewards/run.sh new file mode 100755 index 000000000..d4bc7ea96 --- /dev/null +++ b/rewards/run.sh @@ -0,0 +1,2 @@ + +cargo run -- --tendermint-url http://127.0.0.1:27657 --database-url postgres://postgres:password@0.0.0.0:5435/namada-indexer --sleep-for 15 diff --git a/rewards/src/main.rs b/rewards/src/main.rs index 805affd44..926ef9511 100644 --- a/rewards/src/main.rs +++ b/rewards/src/main.rs @@ -5,12 +5,13 @@ use std::time::Duration; use clap::Parser; use clap_verbosity_flag::LevelFilter; use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl, SelectableHelper}; +use orm::migrations::run_migrations; use orm::pos_rewards::PosRewardInsertDb; use orm::validators::ValidatorDb; use rewards::config::AppConfig; use rewards::services::namada as namada_service; use rewards::state::AppState; -use shared::error::{AsDbError, AsRpcError, MainError}; +use shared::error::{AsDbError, AsRpcError, ContextDbInteractError, MainError}; use tendermint_rpc::HttpClient; use tokio::signal; use tokio::time::sleep; @@ -44,6 +45,13 @@ async fn main() -> anyhow::Result<()> { Arc::new(HttpClient::new(config.tendermint_url.as_str()).unwrap()); let app_state = AppState::new(config.database_url).into_db_error()?; + + let conn = Arc::new(app_state.get_db_connection().await.into_db_error()?); + // Run migrations + run_migrations(&conn) + .await + .context_db_interact_error() + .into_db_error()?; let retry_strategy = FixedInterval::from_millis(5000).map(jitter); let exit_handle = must_exit_handle(); diff --git a/shared/src/transaction.rs b/shared/src/transaction.rs index 2c990e34d..81a581e37 100644 --- a/shared/src/transaction.rs +++ b/shared/src/transaction.rs @@ -26,6 +26,7 @@ pub enum TransactionKind { impl TransactionKind { pub fn from(tx_kind_name: &str, data: &[u8]) -> Self { + println!("tx kind: {}", tx_kind_name); match tx_kind_name { "tx_transfer" => { let transfer_data = @@ -159,6 +160,7 @@ impl Transaction { let tx_data = transaction.data().unwrap_or_default(); TransactionKind::from(&tx_kind_name, &tx_data) } else { + println!("UNKNOWN: {}", id); TransactionKind::Unknown } } else { diff --git a/swagger.yml b/swagger.yml index 259169afa..7c6b3b52e 100644 --- a/swagger.yml +++ b/swagger.yml @@ -7,14 +7,14 @@ info: email: hello@heliax.dev url: https://github.com/anoma/namada-indexer servers: - - url: http://localhost:5000/v1/api + - url: http://localhost:5000 paths: /health: get: responses: '200': description: Health check - /pos/validator: + /api/v1/pos/validator: get: responses: '200': @@ -43,7 +43,7 @@ paths: total_items: type: integer minimum: 0 - /pos/reward/{address}: + /api/v1/pos/reward/{address}: get: summary: Get all the rewards for an address parameters: @@ -62,7 +62,7 @@ paths: type: array items: $ref: '#/components/schemas/Reward' - /pos/bond/{address}: + /api/v1/pos/bond/{address}: get: summary: Get all the bonds for an address parameters: @@ -81,7 +81,7 @@ paths: type: array items: $ref: '#/components/schemas/Bond' - /pos/unbonds/{address}: + /api/v1/pos/unbond/{address}: get: summary: Get all the unbonds for an an address parameters: @@ -100,7 +100,7 @@ paths: type: array items: $ref: '#/components/schemas/Unbond' - /pos/withdraw/{address}/{epoch}: + /api/v1/pos/withdraw/{address}/{epoch}: get: summary: Get all the withdraws for an address at a specific epoch parameters: @@ -125,7 +125,7 @@ paths: type: array items: $ref: '#/components/schemas/Withdraw' - /gov/proposal: + /api/v1/gov/proposal: get: summary: Get a list of governance proposals parameters: @@ -168,7 +168,7 @@ paths: total_items: type: integer minimum: 0 - /gov/search/{text}: + /api/v1/gov/search/{text}: get: summary: Get a list of governance proposals matching a text in the title parameters: @@ -212,7 +212,7 @@ paths: total_items: type: integer minimum: 0 - /gov/proposal/{id}: + /api/v1/gov/proposal/{id}: get: summary: Get a governance proposal by proposal id parameters: @@ -230,7 +230,7 @@ paths: application/json: schema: $ref: '#/components/schemas/Proposal' - /gov/proposal/{id}/votes: + /api/v1/gov/proposal/{id}/votes: get: summary: Get all the votes for a governance proposal parameters: @@ -268,7 +268,7 @@ paths: total_items: type: integer minimum: 0 - /gov/proposal/{id}/votes/{address}: + /api/v1/gov/proposal/{id}/votes/{address}: get: summary: Get all the votes for a governance proposal from an address parameters: @@ -294,7 +294,7 @@ paths: type: array items: $ref: '#/components/schemas/Vote' - /account/{address}: + /api/v1/account/{address}: get: summary: Get the all the tokens balances of an address parameters: diff --git a/webserver/run.sh b/webserver/run.sh index 12cef7119..07d58ef40 100755 --- a/webserver/run.sh +++ b/webserver/run.sh @@ -1 +1 @@ -cargo run -- --cargo-env development --database-url postgres://postgres:password@0.0.0.0:5435/indexer_local --cache-url redis://redis@0.0.0.0:6379 +cargo run -- --database-url postgres://postgres:password@0.0.0.0:5435/namada-indexer --cache-url redis://redis@0.0.0.0:6379 diff --git a/webserver/src/app.rs b/webserver/src/app.rs index 365b8e264..be4b4d8ab 100644 --- a/webserver/src/app.rs +++ b/webserver/src/app.rs @@ -43,30 +43,35 @@ impl ApplicationServer { Router::new() .route("/pos/validator", get(pos_handlers::get_validators)) - .route("/pos/bonds/{address}", get(pos_handlers::get_bonds)) - .route("/pos/unbonds/{address}", get(pos_handlers::get_unbonds)) + .route("/pos/bond/:address", get(pos_handlers::get_bonds)) + .route("/pos/unbond/:address", get(pos_handlers::get_unbonds)) + .route("/pos/withdraw/:address/:epoch", get(pos_handlers::get_withdraws)) .route( - "/pos/reward/{address}", - get(pos_handlers::get_withdraws), + "/pos/reward/:address", + get(pos_handlers::get_rewards), ) .route( "/gov/proposal", get(gov_handlers::get_governance_proposals), ) .route( - "/gov/proposal/{id}", + "/gov/search/:text", + get(gov_handlers::search_governance_proposals_by_pattern), + ) + .route( + "/gov/proposal/:id", get(gov_handlers::get_governance_proposal_by_id), ) .route( - "/gov/proposal/{id}/votes", + "/gov/proposal/:id/votes", get(gov_handlers::get_governance_proposal_votes), ) .route( - "/gov/proposal/{id}/votes/{address}", + "/gov/proposal/:id/votes/:address", get(gov_handlers::get_governance_proposal_votes_by_address), ) .route( - "/account/{address}", + "/account/:address", get(balance_handlers::get_address_balance), ) .with_state(common_state) diff --git a/webserver/src/dto/governance.rs b/webserver/src/dto/governance.rs index 687446192..97599d52d 100644 --- a/webserver/src/dto/governance.rs +++ b/webserver/src/dto/governance.rs @@ -22,3 +22,5 @@ pub struct ProposalVotesQueryparams { #[serde(flatten)] pub pagination: Option, } + +pub type ProposalSearchQueryParams = ProposalVotesQueryparams; \ No newline at end of file diff --git a/webserver/src/dto/mod.rs b/webserver/src/dto/mod.rs index 71d6cc904..b613118dc 100644 --- a/webserver/src/dto/mod.rs +++ b/webserver/src/dto/mod.rs @@ -1,2 +1,3 @@ pub mod governance; pub mod utils; +pub mod pos; diff --git a/webserver/src/dto/pos.rs b/webserver/src/dto/pos.rs index e69de29bb..3519cc798 100644 --- a/webserver/src/dto/pos.rs +++ b/webserver/src/dto/pos.rs @@ -0,0 +1,10 @@ +use serde::{Deserialize, Serialize}; +use validator::Validate; + +use super::utils::Pagination; + +#[derive(Clone, Serialize, Deserialize, Validate)] +pub struct PoSQueryParams { + #[serde(flatten)] + pub pagination: Option, +} \ No newline at end of file diff --git a/webserver/src/handler/governance.rs b/webserver/src/handler/governance.rs index 013f4f8f5..d04a08f9f 100644 --- a/webserver/src/handler/governance.rs +++ b/webserver/src/handler/governance.rs @@ -4,7 +4,7 @@ use axum::Json; use axum_macros::debug_handler; use axum_trace_id::TraceId; -use crate::dto::governance::{ProposalQueryParams, ProposalVotesQueryparams}; +use crate::dto::governance::{ProposalQueryParams, ProposalSearchQueryParams, ProposalVotesQueryparams}; use crate::error::api::ApiError; use crate::error::governance::GovernanceError; use crate::response::governance::{Proposal, ProposalVote}; @@ -18,7 +18,7 @@ pub async fn get_governance_proposals( Query(query): Query, State(state): State, ) -> Result>>, ApiError> { - let page = query.pagination.map(|p| p.page).unwrap_or(0); + let page = query.pagination.map(|p| p.page).unwrap_or(1); let (proposals, total_proposals) = state .gov_service .find_governance_proposals(query.status, page) @@ -53,10 +53,9 @@ pub async fn search_governance_proposals_by_pattern( _headers: HeaderMap, Path(pattern): Path, State(state): State, - Query(page): Query>, + Query(query): Query, ) -> Result>>, ApiError> { - let page = page.unwrap_or(0); - + let page = query.pagination.map(|p| p.page).unwrap_or(1); let (proposals, total_proposals) = state .gov_service .search_governance_proposals_by_pattern(pattern, page) @@ -74,7 +73,7 @@ pub async fn get_governance_proposal_votes( Query(query): Query, State(state): State, ) -> Result>>, ApiError> { - let page = query.pagination.map(|p| p.page).unwrap_or(0); + let page = query.pagination.map(|p| p.page).unwrap_or(1); let (proposal_votes, total_votes) = state .gov_service .find_governance_proposal_votes(proposal_id, page) diff --git a/webserver/src/handler/pos.rs b/webserver/src/handler/pos.rs index 1f9a72d4a..f8c3a64a7 100644 --- a/webserver/src/handler/pos.rs +++ b/webserver/src/handler/pos.rs @@ -4,6 +4,7 @@ use axum::Json; use axum_macros::debug_handler; use axum_trace_id::TraceId; +use crate::dto::pos::PoSQueryParams; use crate::error::api::ApiError; use crate::response::pos::{Bond, Reward, Unbond, ValidatorWithId, Withdraw}; use crate::response::utils::PaginatedResponse; @@ -13,9 +14,10 @@ use crate::state::common::CommonState; pub async fn get_validators( _trace_id: TraceId, _headers: HeaderMap, - Query(page): Query, + Query(query): Query, State(state): State, ) -> Result>>, ApiError> { + let page = query.pagination.map(|p| p.page).unwrap_or(1); let (validators, total_validators) = state.pos_service.get_all_validators(page).await?; From 884ba80ea5d8ca344127b5b141598598f28e4e99 Mon Sep 17 00:00:00 2001 From: Gianmarco Fraccaroli Date: Mon, 20 May 2024 16:19:23 +0200 Subject: [PATCH 4/8] upsert rewards --- orm/migrations/2024-05-09-131859_pos_rewards/up.sql | 2 ++ rewards/src/main.rs | 11 ++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/orm/migrations/2024-05-09-131859_pos_rewards/up.sql b/orm/migrations/2024-05-09-131859_pos_rewards/up.sql index fbd7fdb37..aea2b9d0d 100644 --- a/orm/migrations/2024-05-09-131859_pos_rewards/up.sql +++ b/orm/migrations/2024-05-09-131859_pos_rewards/up.sql @@ -8,4 +8,6 @@ CREATE TABLE pos_rewards ( CONSTRAINT fk_validator_id FOREIGN KEY(validator_id) REFERENCES validators(id) ON DELETE CASCADE ); +ALTER TABLE pos_rewards ADD UNIQUE (owner, validator_id); + CREATE INDEX index_pos_rewards_owner ON pos_rewards USING HASH (owner); \ No newline at end of file diff --git a/rewards/src/main.rs b/rewards/src/main.rs index 926ef9511..e0d5e546a 100644 --- a/rewards/src/main.rs +++ b/rewards/src/main.rs @@ -4,9 +4,10 @@ use std::time::Duration; use clap::Parser; use clap_verbosity_flag::LevelFilter; +use diesel::upsert::excluded; use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl, SelectableHelper}; use orm::migrations::run_migrations; -use orm::pos_rewards::PosRewardInsertDb; +use orm::pos_rewards::{self, PosRewardInsertDb}; use orm::validators::ValidatorDb; use rewards::config::AppConfig; use rewards::services::namada as namada_service; @@ -45,7 +46,7 @@ async fn main() -> anyhow::Result<()> { Arc::new(HttpClient::new(config.tendermint_url.as_str()).unwrap()); let app_state = AppState::new(config.database_url).into_db_error()?; - + let conn = Arc::new(app_state.get_db_connection().await.into_db_error()?); // Run migrations run_migrations(&conn) @@ -108,7 +109,11 @@ async fn main() -> anyhow::Result<()> { }) .collect::>(), ) - .on_conflict_do_nothing() + .on_conflict((orm::schema::pos_rewards::columns::owner, orm::schema::pos_rewards::columns::validator_id)) + .do_update() + .set( + orm::schema::pos_rewards::columns::raw_amount.eq(excluded(orm::schema::pos_rewards::columns::raw_amount)) + ) .execute(transaction_conn) } ) From 8d5bc7d8c3d85bf3b372e6aa7716c38f1b344e1c Mon Sep 17 00:00:00 2001 From: Gianmarco Fraccaroli Date: Mon, 20 May 2024 16:23:52 +0200 Subject: [PATCH 5/8] return 404 if proposal is not found --- webserver/src/service/governance.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/webserver/src/service/governance.rs b/webserver/src/service/governance.rs index e060b3577..0b06406af 100644 --- a/webserver/src/service/governance.rs +++ b/webserver/src/service/governance.rs @@ -96,6 +96,16 @@ impl GovernanceService { proposal_id: u64, page: u64, ) -> Result<(Vec, u64), GovernanceError> { + let db_proposal = self + .governance_repo + .find_governance_proposals_by_id(proposal_id as i32) + .await + .map_err(GovernanceError::Database)?; + + if db_proposal.is_none() { + return Err(GovernanceError::NotFound(proposal_id)); + } + let (db_proposal_votes, total_items) = self .governance_repo .find_governance_proposal_votes(proposal_id as i32, page as i64) @@ -116,6 +126,16 @@ impl GovernanceService { proposal_id: u64, voter_address: String, ) -> Result, GovernanceError> { + let db_proposal = self + .governance_repo + .find_governance_proposals_by_id(proposal_id as i32) + .await + .map_err(GovernanceError::Database)?; + + if db_proposal.is_none() { + return Err(GovernanceError::NotFound(proposal_id)); + } + let db_proposal_votes = self .governance_repo .find_governance_proposal_votes_by_address( From 9851dcd61962c29fa650bead8d4b776a32a4cb2d Mon Sep 17 00:00:00 2001 From: Gianmarco Fraccaroli Date: Mon, 20 May 2024 16:26:35 +0200 Subject: [PATCH 6/8] wip --- chain/src/main.rs | 2 +- rewards/src/main.rs | 2 +- webserver/src/app.rs | 6 +++--- webserver/src/dto/governance.rs | 2 +- webserver/src/dto/mod.rs | 2 +- webserver/src/dto/pos.rs | 2 +- webserver/src/handler/governance.rs | 4 +++- 7 files changed, 11 insertions(+), 9 deletions(-) diff --git a/chain/src/main.rs b/chain/src/main.rs index f8f592978..d4536b1f8 100644 --- a/chain/src/main.rs +++ b/chain/src/main.rs @@ -66,7 +66,7 @@ async fn main() -> Result<(), MainError> { let app_state = AppState::new(config.database_url).into_db_error()?; let conn = Arc::new(app_state.get_db_connection().await.into_db_error()?); - + // Run migrations run_migrations(&conn) .await diff --git a/rewards/src/main.rs b/rewards/src/main.rs index e0d5e546a..2dc8a99c6 100644 --- a/rewards/src/main.rs +++ b/rewards/src/main.rs @@ -7,7 +7,7 @@ use clap_verbosity_flag::LevelFilter; use diesel::upsert::excluded; use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl, SelectableHelper}; use orm::migrations::run_migrations; -use orm::pos_rewards::{self, PosRewardInsertDb}; +use orm::pos_rewards::PosRewardInsertDb; use orm::validators::ValidatorDb; use rewards::config::AppConfig; use rewards::services::namada as namada_service; diff --git a/webserver/src/app.rs b/webserver/src/app.rs index be4b4d8ab..2e4db6e5c 100644 --- a/webserver/src/app.rs +++ b/webserver/src/app.rs @@ -45,11 +45,11 @@ impl ApplicationServer { .route("/pos/validator", get(pos_handlers::get_validators)) .route("/pos/bond/:address", get(pos_handlers::get_bonds)) .route("/pos/unbond/:address", get(pos_handlers::get_unbonds)) - .route("/pos/withdraw/:address/:epoch", get(pos_handlers::get_withdraws)) .route( - "/pos/reward/:address", - get(pos_handlers::get_rewards), + "/pos/withdraw/:address/:epoch", + get(pos_handlers::get_withdraws), ) + .route("/pos/reward/:address", get(pos_handlers::get_rewards)) .route( "/gov/proposal", get(gov_handlers::get_governance_proposals), diff --git a/webserver/src/dto/governance.rs b/webserver/src/dto/governance.rs index 97599d52d..7b2872af7 100644 --- a/webserver/src/dto/governance.rs +++ b/webserver/src/dto/governance.rs @@ -23,4 +23,4 @@ pub struct ProposalVotesQueryparams { pub pagination: Option, } -pub type ProposalSearchQueryParams = ProposalVotesQueryparams; \ No newline at end of file +pub type ProposalSearchQueryParams = ProposalVotesQueryparams; diff --git a/webserver/src/dto/mod.rs b/webserver/src/dto/mod.rs index b613118dc..c571acbb1 100644 --- a/webserver/src/dto/mod.rs +++ b/webserver/src/dto/mod.rs @@ -1,3 +1,3 @@ pub mod governance; -pub mod utils; pub mod pos; +pub mod utils; diff --git a/webserver/src/dto/pos.rs b/webserver/src/dto/pos.rs index 3519cc798..5195e9eae 100644 --- a/webserver/src/dto/pos.rs +++ b/webserver/src/dto/pos.rs @@ -7,4 +7,4 @@ use super::utils::Pagination; pub struct PoSQueryParams { #[serde(flatten)] pub pagination: Option, -} \ No newline at end of file +} diff --git a/webserver/src/handler/governance.rs b/webserver/src/handler/governance.rs index d04a08f9f..cb5f544fc 100644 --- a/webserver/src/handler/governance.rs +++ b/webserver/src/handler/governance.rs @@ -4,7 +4,9 @@ use axum::Json; use axum_macros::debug_handler; use axum_trace_id::TraceId; -use crate::dto::governance::{ProposalQueryParams, ProposalSearchQueryParams, ProposalVotesQueryparams}; +use crate::dto::governance::{ + ProposalQueryParams, ProposalSearchQueryParams, ProposalVotesQueryparams, +}; use crate::error::api::ApiError; use crate::error::governance::GovernanceError; use crate::response::governance::{Proposal, ProposalVote}; From 1e21d94a997628ce03ce0cd5c9307f6c9db93d6a Mon Sep 17 00:00:00 2001 From: Gianmarco Fraccaroli Date: Mon, 20 May 2024 16:38:30 +0200 Subject: [PATCH 7/8] wip --- chain/Cargo.toml | 7 +++-- chain/src/services/namada.rs | 53 +++++++++++++++++++++--------------- 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/chain/Cargo.toml b/chain/Cargo.toml index f827ca23f..f2abd3522 100644 --- a/chain/Cargo.toml +++ b/chain/Cargo.toml @@ -13,11 +13,11 @@ name = "chain" path = "src/main.rs" [dependencies] -tokio.workspace = true +tokio.workspace = true tracing.workspace = true tracing-subscriber.workspace = true serde_json.workspace = true -clap.workspace = true +clap.workspace = true anyhow.workspace = true namada_sdk.workspace = true namada_core.workspace = true @@ -30,6 +30,7 @@ deadpool-diesel.workspace = true diesel.workspace = true orm.workspace = true clap-verbosity-flag.workspace = true +futures.workspace = true [build-dependencies] -vergen = { version = "8.0.0", features = ["build", "git", "gitcl"] } \ No newline at end of file +vergen = { version = "8.0.0", features = ["build", "git", "gitcl"] } diff --git a/chain/src/services/namada.rs b/chain/src/services/namada.rs index 4a4289b61..67aeb1b85 100644 --- a/chain/src/services/namada.rs +++ b/chain/src/services/namada.rs @@ -2,6 +2,7 @@ use std::collections::HashSet; use std::str::FromStr; use anyhow::{anyhow, Context}; +use futures::StreamExt; use namada_core::storage::{ BlockHeight as NamadaSdkBlockHeight, Epoch as NamadaSdkEpoch, }; @@ -72,28 +73,36 @@ pub async fn query_balance( client: &HttpClient, balance_changes: &HashSet, ) -> anyhow::Result { - let mut res: Balances = vec![]; - - for balance_change in balance_changes { - let owner = - NamadaSdkAddress::from_str(&balance_change.address.to_string()) - .context("Failed to parse owner address")?; - let token = - NamadaSdkAddress::from_str(&balance_change.token.to_string()) - .context("Failed to parse token address")?; - - let amount = rpc::get_token_balance(client, &token, &owner) - .await - .unwrap_or_default(); - - res.push(Balance { - owner: Id::from(owner), - token: Id::from(token), - amount: Amount::from(amount), - }); - } - - anyhow::Ok(res) + Ok(futures::stream::iter(balance_changes) + .filter_map(|balance_change| async move { + tracing::info!( + "Fetching balance change for {} ...", + balance_change.address + ); + + let owner = + NamadaSdkAddress::from_str(&balance_change.address.to_string()) + .context("Failed to parse owner address") + .ok()?; + let token = + NamadaSdkAddress::from_str(&balance_change.token.to_string()) + .context("Failed to parse token address") + .ok()?; + + let amount = rpc::get_token_balance(client, &token, &owner) + .await + .unwrap_or_default(); + + Some(Balance { + owner: Id::from(owner), + token: Id::from(token), + amount: Amount::from(amount), + }) + }) + .map(futures::future::ready) + .buffer_unordered(20) + .collect::>() + .await) } pub async fn query_all_balances( From f1c1da5cf7cc1c4d79eb8edf6e6bff4e7fc87dba Mon Sep 17 00:00:00 2001 From: Gianmarco Fraccaroli Date: Mon, 20 May 2024 16:39:17 +0200 Subject: [PATCH 8/8] wip --- chain/src/services/namada.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/chain/src/services/namada.rs b/chain/src/services/namada.rs index 67aeb1b85..1e7d1d8e5 100644 --- a/chain/src/services/namada.rs +++ b/chain/src/services/namada.rs @@ -68,7 +68,6 @@ pub async fn get_epoch_at_block_height( Ok(epoch.0 as Epoch) } -// TODO: remove unwraps pub async fn query_balance( client: &HttpClient, balance_changes: &HashSet,