More Metrics (#495)

* metrics refactoring

* metrics refactoring

* Make brain hurt less by using a damn function

* patch through metrics to abacus_ethereum inbox and outbox

* WIP

* Minor cleanup

* work on time update

* added label arrays

* refactoring

* Metrics for sending txns

* Metrics middleware

* cleanup

* Cleanup after merge

* Add metrics config and setup prom metrics with core metrics

* Minor fixes to docs

* Better trait building

* rename

* Weave metrics through code

* Use core metrics instead

* Removed some unused code

* fmt

* change feature layout

* fix docker build

* Start scraping metrics on an interval

* fmt

* fix duplicate label error

* change features

* fix u256 conversion fn

* refactor

* Support default values in config

* Minor fixes

* Use f64 conversion from primitive-types

* remove unused code

* fix const name

* improve docs

* better var name

* remove unused code

* remove unused code

rust/Cargo.toml

@@ -11,4 +11,5 @@ members = [
     "agents/kathy",
     "agents/validator",
     "agents/relayer",
+    "ethers-prometheus"
 ]

rust/Dockerfile

@@ -16,6 +16,7 @@ COPY abacus-base ./abacus-base
 COPY abacus-cli ./abacus-cli
 COPY abacus-core ./abacus-core
 COPY abacus-test ./abacus-test
+COPY ethers-prometheus ./ethers-prometheus
 
 COPY Cargo.toml .
 COPY Cargo.lock .

rust/abacus-base/Cargo.toml

@@ -21,13 +21,14 @@ tracing-subscriber = { version = "0.3", features = ["json"] }
 rocksdb = { git = "https://github.com/rust-rocksdb/rust-rocksdb" }
 mockall = "0.10.2"
 
+ethers-prometheus = { path = "../ethers-prometheus", features = ["serde"] }
 abacus-core = { path = "../abacus-core" }
-abacus-ethereum = { path = "../chains/abacus-ethereum"}
+abacus-ethereum = { path = "../chains/abacus-ethereum" }
 abacus-test = { path = "../abacus-test" }
 paste = "1.0"
 tracing-error = "0.2"
 
-prometheus = "0.12"
+prometheus = "0.13"
 
 warp = "0.3"
 

rust/abacus-base/src/agent.rs

@@ -11,6 +11,7 @@ use futures_util::future::select_all;
 use tracing::instrument::Instrumented;
 use tracing::{info_span, Instrument};
 
+use std::fmt::Debug;
 use std::{collections::HashMap, sync::Arc};
 use tokio::task::JoinHandle;
 
@@ -37,12 +38,12 @@ pub struct AbacusAgentCore {
     /// The height at which to start indexing the Outbox
     pub indexer: IndexSettings,
     /// Settings this agent was created with
-    pub settings: crate::settings::Settings,
+    pub settings: Settings,
 }
 
 /// A trait for an abacus agent
 #[async_trait]
-pub trait Agent: Send + Sync + std::fmt::Debug + AsRef<AbacusAgentCore> {
+pub trait Agent: Send + Sync + Debug + AsRef<AbacusAgentCore> {
     /// The agent's name
     const AGENT_NAME: &'static str;
 

rust/abacus-base/src/contract_sync/metrics.rs

@@ -1,5 +1,5 @@
 use crate::CoreMetrics;
-use prometheus::{IntGauge, IntGaugeVec};
+use prometheus::{IntCounterVec, IntGauge, IntGaugeVec};
 use std::sync::Arc;
 
 /// Struct encapsulating prometheus metrics used by the ContractSync.
@@ -12,7 +12,7 @@ pub struct ContractSyncMetrics {
     pub stored_events: IntGaugeVec,
     /// Unique occasions when agent missed an event (label values
     /// differentiate checkpoints vs. messages)
-    pub missed_events: IntGaugeVec,
+    pub missed_events: IntCounterVec,
     /// An optional gauge for tracking the latest message leafs that are being indexed
     pub message_leaf_index: Option<IntGauge>,
 }
@@ -37,7 +37,7 @@ impl ContractSyncMetrics {
             .expect("failed to register stored_events metric");
 
         let missed_events = metrics
-            .new_int_gauge(
+            .new_int_counter(
                 "contract_sync_missed_events",
                 "Number of unique occasions when agent missed an event",
                 &["data_type", "contract_name", "agent"],