Skip to content

Commit

Permalink
Merge pull request #35 from vulcanize/metrics
Browse files Browse the repository at this point in the history
Metrics
  • Loading branch information
i-norden authored Oct 21, 2021
2 parents 6a912ee + 7dc6c52 commit a057940
Show file tree
Hide file tree
Showing 13 changed files with 468 additions and 104 deletions.
10 changes: 10 additions & 0 deletions cmd/env.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,11 @@ const (
LVLDB_ANCIENT = "LVLDB_ANCIENT"
STATEDIFF_WORKERS = "STATEDIFF_WORKERS"
WRITE_SERVER = "WRITE_SERVER"

PROM_METRICS = "PROM_METRICS"
PROM_HTTP = "PROM_HTTP"
PROM_HTTP_ADDR = "PROM_HTTP_ADDR"
PROM_HTTP_PORT = "PROM_HTTP_PORT"
)

// Bind env vars for eth node and DB configuration
Expand All @@ -60,5 +65,10 @@ func init() {
viper.BindEnv("leveldb.path", LVLDB_PATH)
viper.BindEnv("leveldb.ancient", LVLDB_ANCIENT)

viper.BindEnv("prom.metrics", PROM_METRICS)
viper.BindEnv("prom.http", PROM_HTTP)
viper.BindEnv("prom.httpAddr", PROM_HTTP_ADDR)
viper.BindEnv("prom.httpPort", PROM_HTTP_PORT)

viper.BindEnv("statediff.workers", STATEDIFF_WORKERS)
}
26 changes: 26 additions & 0 deletions cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"

"github.com/vulcanize/eth-statediff-service/pkg/prom"
)

var (
Expand Down Expand Up @@ -64,6 +66,19 @@ func initFuncs(cmd *cobra.Command, args []string) {
if err := logLevel(); err != nil {
log.Fatal("Could not set log level: ", err)
}

if viper.GetBool("prom.metrics") {
prom.Init()
}

if viper.GetBool("prom.http") {
addr := fmt.Sprintf(
"%s:%s",
viper.GetString("prom.httpAddr"),
viper.GetString("prom.httpPort"),
)
prom.Listen(addr)
}
}

func logLevel() error {
Expand Down Expand Up @@ -106,9 +121,16 @@ func init() {
"0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3", "eth genesis block hash")
rootCmd.PersistentFlags().String("eth-network-id", "1", "eth network id")
rootCmd.PersistentFlags().String("eth-chain-id", "1", "eth chain id")

rootCmd.PersistentFlags().Int("cache-db", 1024, "megabytes of memory allocated to database cache")
rootCmd.PersistentFlags().Int("cache-trie", 1024, "Megabytes of memory allocated to trie cache")

rootCmd.PersistentFlags().Bool("prom-http", false, "enable prometheus http service")
rootCmd.PersistentFlags().String("prom-http-addr", "127.0.0.1", "prometheus http host")
rootCmd.PersistentFlags().String("prom-http-port", "8080", "prometheus http port")

rootCmd.PersistentFlags().Bool("metrics", false, "enable metrics")

viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log-file"))
viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
viper.BindPFlag("statediff.workers", rootCmd.PersistentFlags().Lookup("workers"))
Expand All @@ -126,6 +148,10 @@ func init() {
viper.BindPFlag("ethereum.chainID", rootCmd.PersistentFlags().Lookup("eth-chain-id"))
viper.BindPFlag("cache.database", rootCmd.PersistentFlags().Lookup("cache-db"))
viper.BindPFlag("cache.trie", rootCmd.PersistentFlags().Lookup("cache-trie"))
viper.BindPFlag("prom.http", rootCmd.PersistentFlags().Lookup("prom-http"))
viper.BindPFlag("prom.httpAddr", rootCmd.PersistentFlags().Lookup("prom-http-addr"))
viper.BindPFlag("prom.httpPort", rootCmd.PersistentFlags().Lookup("prom-http-port"))
viper.BindPFlag("prom.metrics", rootCmd.PersistentFlags().Lookup("metrics"))
}

func initConfig() {
Expand Down
51 changes: 2 additions & 49 deletions cmd/serve.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,10 @@ import (
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/trie"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"

ind "github.com/ethereum/go-ethereum/statediff/indexer"
"github.com/ethereum/go-ethereum/statediff/indexer/postgres"

sd "github.com/vulcanize/eth-statediff-service/pkg"
)

Expand All @@ -50,52 +46,9 @@ var serveCmd = &cobra.Command{
}

func serve() {
logWithCommand.Info("starting statediff RPC service")

// load params
path := viper.GetString("leveldb.path")
ancientPath := viper.GetString("leveldb.ancient")
if path == "" || ancientPath == "" {
logWithCommand.Fatal("require a valid eth leveldb primary datastore path and ancient datastore path")
}

nodeInfo := GetEthNodeInfo()
config, err := chainConfig(nodeInfo.ChainID)
if err != nil {
logWithCommand.Fatal(err)
}

// create leveldb reader
logWithCommand.Info("Creating leveldb reader")
conf := sd.ReaderConfig{
TrieConfig: &trie.Config{
Cache: viper.GetInt("cache.trie"),
Journal: "",
Preimages: false,
},
ChainConfig: config,
Path: path,
AncientPath: ancientPath,
DBCacheSize: viper.GetInt("cache.database"),
}
lvlDBReader, err := sd.NewLvlDBReader(conf)
if err != nil {
logWithCommand.Fatal(err)
}

// create statediff service
logWithCommand.Info("Creating statediff service")
db, err := postgres.NewDB(postgres.DbConnectionString(GetDBParams()), GetDBConfig(), nodeInfo)
if err != nil {
logWithCommand.Fatal(err)
}

indexer, err := ind.NewStateDiffIndexer(config, db)
if err != nil {
logWithCommand.Fatal(err)
}
logWithCommand.Info("Running eth-statediff-service serve command")

statediffService, err := sd.NewStateDiffService(lvlDBReader, indexer, viper.GetUint("statediff.workers"))
statediffService, err := createStateDiffService()
if err != nil {
logWithCommand.Fatal(err)
}
Expand Down
67 changes: 67 additions & 0 deletions cmd/util.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
package cmd

import (
ind "github.com/ethereum/go-ethereum/statediff/indexer"
"github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
"github.com/ethereum/go-ethereum/trie"
"github.com/spf13/viper"

sd "github.com/vulcanize/eth-statediff-service/pkg"
"github.com/vulcanize/eth-statediff-service/pkg/prom"
)

func createStateDiffService() (sd.IService, error) {
logWithCommand.Info("Loading statediff service parameters")
path := viper.GetString("leveldb.path")
ancientPath := viper.GetString("leveldb.ancient")
if path == "" || ancientPath == "" {
logWithCommand.Fatal("require a valid eth leveldb primary datastore path and ancient datastore path")
}

nodeInfo := GetEthNodeInfo()
config, err := chainConfig(nodeInfo.ChainID)
if err != nil {
logWithCommand.Fatal(err)
}

// create leveldb reader
logWithCommand.Info("Creating leveldb reader")
conf := sd.ReaderConfig{
TrieConfig: &trie.Config{
Cache: viper.GetInt("cache.trie"),
Journal: "",
Preimages: false,
},
ChainConfig: config,
Path: path,
AncientPath: ancientPath,
DBCacheSize: viper.GetInt("cache.database"),
}
lvlDBReader, err := sd.NewLvlDBReader(conf)
if err != nil {
logWithCommand.Fatal(err)
}

// create statediff service
logWithCommand.Info("Creating statediff service")
db, err := setupPostgres(nodeInfo)
if err != nil {
logWithCommand.Fatal(err)
}
indexer, err := ind.NewStateDiffIndexer(config, db)
if err != nil {
logWithCommand.Fatal(err)
}
return sd.NewStateDiffService(lvlDBReader, indexer, viper.GetUint("statediff.workers"))
}

func setupPostgres(nodeInfo node.Info) (*postgres.DB, error) {
params := GetDBParams()
db, err := postgres.NewDB(postgres.DbConnectionString(params), GetDBConfig(), nodeInfo)
if err != nil {
return nil, err
}
prom.RegisterDBCollector(params.Name, db.DB)
return db, nil
}
51 changes: 2 additions & 49 deletions cmd/write.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,9 @@ import (
"time"

gethsd "github.com/ethereum/go-ethereum/statediff"
ind "github.com/ethereum/go-ethereum/statediff/indexer"
"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
"github.com/ethereum/go-ethereum/trie"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"

sd "github.com/vulcanize/eth-statediff-service/pkg"
)

var writeCmd = &cobra.Command{
Expand All @@ -55,53 +50,11 @@ func init() {
}

func write() {
logWithCommand.Info("Starting statediff writer")
// load params
logWithCommand.Info("Running eth-statediff-service write command")
viper.BindEnv("write.serve", WRITE_SERVER)
addr := viper.GetString("write.serve")
path := viper.GetString("leveldb.path")
ancientPath := viper.GetString("leveldb.ancient")
if path == "" || ancientPath == "" {
logWithCommand.Fatal("require a valid eth leveldb primary datastore path and ancient datastore path")
}

nodeInfo := GetEthNodeInfo()
config, err := chainConfig(nodeInfo.ChainID)
if err != nil {
logWithCommand.Fatal(err)
}

// create leveldb reader
logWithCommand.Info("Creating leveldb reader")
conf := sd.ReaderConfig{
TrieConfig: &trie.Config{
Cache: viper.GetInt("cache.trie"),
Journal: "",
Preimages: false,
},
ChainConfig: config,
Path: path,
AncientPath: ancientPath,
DBCacheSize: viper.GetInt("cache.database"),
}
lvlDBReader, err := sd.NewLvlDBReader(conf)
if err != nil {
logWithCommand.Fatal(err)
}

// create statediff service
logWithCommand.Info("Creating statediff service")
db, err := postgres.NewDB(postgres.DbConnectionString(GetDBParams()), GetDBConfig(), nodeInfo)
if err != nil {
logWithCommand.Fatal(err)
}

indexer, err := ind.NewStateDiffIndexer(config, db)
if err != nil {
logWithCommand.Fatal(err)
}

statediffService, err := sd.NewStateDiffService(lvlDBReader, indexer, viper.GetUint("statediff.workers"))
statediffService, err := createStateDiffService()
if err != nil {
logWithCommand.Fatal(err)
}
Expand Down
6 changes: 6 additions & 0 deletions environments/example.toml
Original file line number Diff line number Diff line change
Expand Up @@ -32,3 +32,9 @@
[cache]
database = 1024
trie = 1024

[prom]
metrics = true
http = true
addr = "localhost"
port = "8889"
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ go 1.13

require (
github.com/ethereum/go-ethereum v1.10.9
github.com/jmoiron/sqlx v1.2.0
github.com/prometheus/client_golang v1.0.0
github.com/sirupsen/logrus v1.7.0
github.com/spf13/cobra v1.1.1
github.com/spf13/viper v1.7.1
Expand All @@ -12,4 +14,3 @@ require (
)

replace github.com/ethereum/go-ethereum v1.10.9 => github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27

6 changes: 6 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ github.com/aws/aws-sdk-go-v2/service/sso v1.1.1/go.mod h1:SuZJxklHxLAXgLTc1iFXbE
github.com/aws/aws-sdk-go-v2/service/sts v1.1.1/go.mod h1:Wi0EBZwiz/K44YliU0EKxqTCJGUfYTWXrrBwkq736bM=
github.com/aws/smithy-go v1.1.0/go.mod h1:EzMw8dbp/YJL4A5/sbhGddag+NPT7q084agLbB9LgIw=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
Expand Down Expand Up @@ -364,6 +365,7 @@ github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsO
github.com/mattn/go-sqlite3 v1.14.7 h1:fxWBnXkxfM6sRiuH3bqJ4CfzZojMOLVc0UTsTglEghA=
github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g=
Expand Down Expand Up @@ -441,16 +443,20 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo=
github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
Expand Down
7 changes: 7 additions & 0 deletions monitoring/prometheus.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
global:
scrape_interval: 10s

scrape_configs:
- job_name: 'eth-statediff-service'
static_configs:
- targets: ['localhost:8100']
Loading

0 comments on commit a057940

Please sign in to comment.