Add import_abis binary implementation (#717)

eth-bytecode-db/eth-bytecode-db/Cargo.toml

@@ -56,6 +56,7 @@ url = "2.4.0"
 [features]
 # When activated includes helper methods for tests and benchmarking
 test-utils = []
+tokio-rt = ["tokio/rt-multi-thread"]
 
 [[bin]]
 name = "database_populate"
@@ -64,3 +65,7 @@ required-features = ["test-utils"]
 [[bin]]
 name = "database_search"
 required-features = ["test-utils"]
+
+[[bin]]
+name = "import_abis"
+required-features = ["tokio-rt"]

eth-bytecode-db/eth-bytecode-db/src/bin/import_abis.rs

@@ -0,0 +1,12 @@
+use eth_bytecode_db::verification::import_existing_abis;
+use sea_orm::Database;
+
+#[tokio::main]
+async fn main() {
+    let db_url = std::env::var("DATABASE_URL").expect("DATABASE_URL env was not provided");
+    let db_client = Database::connect(db_url)
+        .await
+        .expect("Error connecting to database");
+
+    import_existing_abis::import(&db_client).await;
+}

eth-bytecode-db/eth-bytecode-db/src/verification/db/eth_bytecode_db.rs

@@ -121,14 +121,16 @@ pub(crate) async fn insert_event_descriptions(
         })
         .collect();
 
-    let result = events::Entity::insert_many(active_models)
-        .on_conflict(OnConflict::new().do_nothing().to_owned())
-        .exec(db_client)
-        .await;
-    match result {
-        Ok(_) | Err(DbErr::RecordNotInserted) => {}
-        Err(err) => {
-            return Err(err).context("insert into \"events\"");
+    if !active_models.is_empty() {
+        let result = events::Entity::insert_many(active_models)
+            .on_conflict(OnConflict::new().do_nothing().to_owned())
+            .exec(db_client)
+            .await;
+        match result {
+            Ok(_) | Err(DbErr::RecordNotInserted) => {}
+            Err(err) => {
+                return Err(err).context("insert into \"events\"");
+            }
         }
     }
 

eth-bytecode-db/eth-bytecode-db/src/verification/handlers/import_existing_abis.rs

@@ -0,0 +1,44 @@
+use crate::verification::handlers::process_abi_data;
+use anyhow::Context;
+use entity::sources;
+use futures::TryStreamExt;
+use sea_orm::{DatabaseConnection, EntityTrait, FromQueryResult, QuerySelect};
+
+pub async fn import(db_client: &DatabaseConnection) {
+    #[derive(FromQueryResult)]
+    struct Abi {
+        pub abi: sea_orm::JsonValue,
+    }
+
+    let mut stream = sources::Entity::find()
+        .select_only()
+        .column(sources::Column::Abi)
+        .distinct()
+        .into_model::<Abi>()
+        .stream(db_client)
+        .await
+        .expect("creating a stream");
+
+    let mut processed = 0;
+
+    loop {
+        match stream.try_next().await.context("getting next abi value") {
+            Ok(None) => break,
+            Ok(Some(item)) => {
+                let abi = item.abi.to_string();
+                if let Err(err) = process_abi_data(Some(abi.clone()), db_client).await {
+                    println!("[ERROR] Error while processing abi; abi={abi}, err={err:#}");
+                }
+            }
+            Err(err) => {
+                println!("[ERROR] Error while processing next abi: {err}");
+            }
+        }
+        processed += 1;
+
+        if processed % 100 == 0 {
+            println!("Processed={processed}")
+        }
+    }
+    println!("\nabis processed successfully; total={processed}");
+}

eth-bytecode-db/eth-bytecode-db/src/verification/handlers/mod.rs

@@ -1,4 +1,5 @@
 pub mod compiler_versions;
+pub mod import_existing_abis;
 pub mod solidity_multi_part;
 pub mod solidity_standard_json;
 pub mod sourcify;

eth-bytecode-db/eth-bytecode-db/src/verification/mod.rs

@@ -9,7 +9,7 @@ mod verifier_alliance;
 pub use client::Client;
 pub use errors::Error;
 pub use handlers::{
-    compiler_versions, solidity_multi_part, solidity_standard_json, sourcify,
+    compiler_versions, import_existing_abis, solidity_multi_part, solidity_standard_json, sourcify,
     sourcify_from_etherscan, vyper_multi_part, vyper_standard_json,
 };
 pub use types::{