diff --git a/cl/persistence/beacon_indicies/indicies.go b/cl/persistence/beacon_indicies/indicies.go index da08b445b8f..f9cd4e2fb5c 100644 --- a/cl/persistence/beacon_indicies/indicies.go +++ b/cl/persistence/beacon_indicies/indicies.go @@ -94,6 +94,38 @@ func MarkRootCanonical(ctx context.Context, tx kv.RwTx, slot uint64, blockRoot l return tx.Put(kv.CanonicalBlockRoots, base_encoding.Encode64(slot), blockRoot[:]) } +func WriteExecutionBlockNumber(tx kv.RwTx, blockRoot libcommon.Hash, blockNumber uint64) error { + return tx.Put(kv.BlockRootToBlockNumber, blockRoot[:], base_encoding.Encode64(blockNumber)) +} + +func WriteExecutionBlockHash(tx kv.RwTx, blockRoot, blockHash libcommon.Hash) error { + return tx.Put(kv.BlockRootToBlockHash, blockRoot[:], blockHash[:]) +} + +func ReadExecutionBlockNumber(tx kv.Tx, blockRoot libcommon.Hash) (*uint64, error) { + val, err := tx.GetOne(kv.BlockRootToBlockNumber, blockRoot[:]) + if err != nil { + return nil, err + } + if len(val) == 0 { + return nil, nil + } + ret := new(uint64) + *ret = base_encoding.Decode64(val) + return ret, nil +} + +func ReadExecutionBlockHash(tx kv.Tx, blockRoot libcommon.Hash) (libcommon.Hash, error) { + val, err := tx.GetOne(kv.BlockRootToBlockHash, blockRoot[:]) + if err != nil { + return libcommon.Hash{}, err + } + if len(val) == 0 { + return libcommon.Hash{}, nil + } + return libcommon.BytesToHash(val), nil +} + func WriteBeaconBlockHeader(ctx context.Context, tx kv.RwTx, signedHeader *cltypes.SignedBeaconBlockHeader) error { headersBytes, err := signedHeader.EncodeSSZ(nil) if err != nil { diff --git a/cl/persistence/beacon_indicies/indicies_test.go b/cl/persistence/beacon_indicies/indicies_test.go index 7451327b89b..3db10d48eca 100644 --- a/cl/persistence/beacon_indicies/indicies_test.go +++ b/cl/persistence/beacon_indicies/indicies_test.go @@ -140,3 +140,35 @@ func TestReadBeaconBlockHeader(t *testing.T) { require.Equal(t, headerRoot, blockRoot) } + +func TestWriteExecutionBlockNumber(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + tx, _ := db.BeginRw(context.Background()) + defer tx.Rollback() + + tHash := libcommon.HexToHash("0x2") + require.NoError(t, WriteExecutionBlockNumber(tx, tHash, 1)) + require.NoError(t, WriteExecutionBlockNumber(tx, tHash, 2)) + require.NoError(t, WriteExecutionBlockNumber(tx, tHash, 3)) + + // Try to retrieve the block's slot by its blockRoot and verify + blockNumber, err := ReadExecutionBlockNumber(tx, tHash) + require.NoError(t, err) + require.Equal(t, uint64(3), *blockNumber) +} + +func TestWriteExecutionBlockHash(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + tx, _ := db.BeginRw(context.Background()) + defer tx.Rollback() + + tHash := libcommon.HexToHash("0x2") + tHash2 := libcommon.HexToHash("0x3") + require.NoError(t, WriteExecutionBlockHash(tx, tHash, tHash2)) + // Try to retrieve the block's slot by its blockRoot and verify + tHash3, err := ReadExecutionBlockHash(tx, tHash) + require.NoError(t, err) + require.Equal(t, tHash2, tHash3) +} diff --git a/cl/persistence/block_saver.go b/cl/persistence/block_saver.go index 6bc35ae3c57..15a02986a8b 100644 --- a/cl/persistence/block_saver.go +++ b/cl/persistence/block_saver.go @@ -2,6 +2,7 @@ package persistence import ( "context" + "errors" "fmt" "io" "path" @@ -18,6 +19,8 @@ import ( "github.com/spf13/afero" ) +const subDivisionFolderSize = 10_000 + type beaconChainDatabaseFilesystem struct { rawDB RawBeaconBlockChain cfg *clparams.BeaconChainConfig @@ -73,6 +76,9 @@ func (b beaconChainDatabaseFilesystem) GetRange(ctx context.Context, tx kv.Tx, f slot := slots[idx] r, err := b.rawDB.BlockReader(ctx, slot, blockRoot) + if errors.Is(err, afero.ErrFileNotFound) { + continue + } if err != nil { return nil, err } @@ -98,7 +104,7 @@ func (b beaconChainDatabaseFilesystem) PurgeRange(ctx context.Context, tx kv.RwT return err } - return beacon_indicies.PruneBlockRoots(ctx, tx, from, from+count) + return nil } func (b beaconChainDatabaseFilesystem) WriteBlock(ctx context.Context, tx kv.RwTx, block *cltypes.SignedBeaconBlock, canonical bool) error { @@ -139,6 +145,15 @@ func (b beaconChainDatabaseFilesystem) WriteBlock(ctx context.Context, tx kv.RwT if err != nil { return err } + if block.Version() >= clparams.BellatrixVersion { + if err := beacon_indicies.WriteExecutionBlockNumber(tx, blockRoot, block.Block.Body.ExecutionPayload.BlockNumber); err != nil { + return err + } + if err := beacon_indicies.WriteExecutionBlockHash(tx, blockRoot, block.Block.Body.ExecutionPayload.BlockHash); err != nil { + return err + } + } + if err := beacon_indicies.WriteBeaconBlockHeaderAndIndicies(ctx, tx, &cltypes.SignedBeaconBlockHeader{ Signature: block.Signature, Header: &cltypes.BeaconBlockHeader{ @@ -156,28 +171,9 @@ func (b beaconChainDatabaseFilesystem) WriteBlock(ctx context.Context, tx kv.RwT // SlotToPaths define the file structure to store a block // -// superEpoch = floor(slot / (epochSize ^ 2)) -// epoch = floot(slot / epochSize) -// file is to be stored at -// "/signedBeaconBlock/{superEpoch}/{epoch}/{root}.ssz_snappy" +// "/signedBeaconBlock/{slot/10_000}/{root}.ssz_snappy" func RootToPaths(slot uint64, root libcommon.Hash, config *clparams.BeaconChainConfig) (folderPath string, filePath string) { - folderPath = path.Clean(fmt.Sprintf("%d/%d", slot/(config.SlotsPerEpoch*config.SlotsPerEpoch), slot/config.SlotsPerEpoch)) + folderPath = path.Clean(fmt.Sprintf("%d", slot/subDivisionFolderSize)) filePath = path.Clean(fmt.Sprintf("%s/%x.sz", folderPath, root)) return } - -func ValidateEpoch(fs afero.Fs, epoch uint64, config *clparams.BeaconChainConfig) error { - superEpoch := epoch / (config.SlotsPerEpoch) - - // the folder path is superEpoch/epoch - folderPath := path.Clean(fmt.Sprintf("%d/%d", superEpoch, epoch)) - - fi, err := afero.ReadDir(fs, folderPath) - if err != nil { - return err - } - for _, fn := range fi { - fn.Name() - } - return nil -} diff --git a/cl/persistence/format/snapshot_format/blocks.go b/cl/persistence/format/snapshot_format/blocks.go index 66e4077318d..12e8170ec3b 100644 --- a/cl/persistence/format/snapshot_format/blocks.go +++ b/cl/persistence/format/snapshot_format/blocks.go @@ -7,6 +7,8 @@ import ( "io" "sync" + libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon/cl/clparams" "github.com/ledgerwatch/erigon/cl/cltypes" "github.com/ledgerwatch/erigon/cl/persistence/format/chunk_encoding" @@ -17,7 +19,8 @@ var buffersPool = sync.Pool{ } type ExecutionBlockReaderByNumber interface { - BlockByNumber(number uint64) (*cltypes.Eth1Block, error) + TransactionsSSZ(w io.Writer, number uint64, hash libcommon.Hash) error + WithdrawalsSZZ(w io.Writer, number uint64, hash libcommon.Hash) error } const ( @@ -33,21 +36,22 @@ const ( ) func writeExecutionBlockPtr(w io.Writer, p *cltypes.Eth1Block) error { - temp := make([]byte, 8) + temp := make([]byte, 40) binary.BigEndian.PutUint64(temp, p.BlockNumber) + copy(temp[8:], p.BlockHash[:]) return chunk_encoding.WriteChunk(w, temp, chunk_encoding.PointerDataType) } -func readExecutionBlockPtr(r io.Reader) (uint64, error) { +func readExecutionBlockPtr(r io.Reader) (uint64, libcommon.Hash, error) { b, dT, err := chunk_encoding.ReadChunkToBytes(r) if err != nil { - return 0, err + return 0, libcommon.Hash{}, err } if dT != chunk_encoding.PointerDataType { - return 0, fmt.Errorf("malformed beacon block, invalid block pointer type %d, expected: %d", dT, chunk_encoding.ChunkDataType) + return 0, libcommon.Hash{}, fmt.Errorf("malformed beacon block, invalid block pointer type %d, expected: %d", dT, chunk_encoding.ChunkDataType) } - return binary.BigEndian.Uint64(b), nil + return binary.BigEndian.Uint64(b[:8]), libcommon.BytesToHash(b[8:]), nil } func computeInitialOffset(version clparams.StateVersion) uint64 { @@ -68,22 +72,25 @@ func computeInitialOffset(version clparams.StateVersion) uint64 { } // WriteBlockForSnapshot writes a block to the given writer in the format expected by the snapshot. -func WriteBlockForSnapshot(block *cltypes.SignedBeaconBlock, w io.Writer) error { +// buf is just a reusable buffer. if it had to grow it will be returned back as grown. +func WriteBlockForSnapshot(w io.Writer, block *cltypes.SignedBeaconBlock, reusable []byte) ([]byte, error) { bodyRoot, err := block.Block.Body.HashSSZ() if err != nil { - return err + return reusable, err } + reusable = reusable[:0] // Maybe reuse the buffer? - encoded, err := block.EncodeSSZ(nil) + encoded, err := block.EncodeSSZ(reusable) if err != nil { - return err + return reusable, err } + reusable = encoded version := block.Version() if _, err := w.Write([]byte{byte(version)}); err != nil { - return err + return reusable, err } if _, err := w.Write(bodyRoot[:]); err != nil { - return err + return reusable, err } currentChunkLength := computeInitialOffset(version) @@ -96,20 +103,21 @@ func WriteBlockForSnapshot(block *cltypes.SignedBeaconBlock, w io.Writer) error currentChunkLength += uint64(body.VoluntaryExits.EncodingSizeSSZ()) // Write the chunk and chunk attestations if err := chunk_encoding.WriteChunk(w, encoded[:currentChunkLength], chunk_encoding.ChunkDataType); err != nil { - return err + return reusable, err } // we are done if we are before altair if version <= clparams.AltairVersion { - return nil + return reusable, nil } - encoded = encoded[currentChunkLength+uint64(body.ExecutionPayload.EncodingSizeSSZ()):] - if err := writeExecutionBlockPtr(w, body.ExecutionPayload); err != nil { - return err + encoded = encoded[currentChunkLength:] + if err := writeEth1BlockForSnapshot(w, encoded[:body.ExecutionPayload.EncodingSizeSSZ()], body.ExecutionPayload); err != nil { + return reusable, err } + encoded = encoded[body.ExecutionPayload.EncodingSizeSSZ():] if version <= clparams.BellatrixVersion { - return nil + return reusable, nil } - return chunk_encoding.WriteChunk(w, encoded, chunk_encoding.ChunkDataType) + return reusable, chunk_encoding.WriteChunk(w, encoded, chunk_encoding.ChunkDataType) } func readMetadataForBlock(r io.Reader, b []byte) (clparams.StateVersion, error) { @@ -153,23 +161,7 @@ func ReadRawBlockFromSnapshot(r io.Reader, out io.Writer, executionReader Execut return v, nil } // Read the block pointer and retrieve chunk4 from the execution reader - blockPointer, err := readExecutionBlockPtr(r) - if err != nil { - return v, err - } - executionBlock, err := executionReader.BlockByNumber(blockPointer) - if err != nil { - return v, err - } - if executionBlock == nil { - return v, fmt.Errorf("execution block %d not found", blockPointer) - } - // TODO(Giulio2002): optimize GC - eth1Bytes, err := executionBlock.EncodeSSZ(nil) - if err != nil { - return v, err - } - if _, err := out.Write(eth1Bytes); err != nil { + if _, err := readEth1BlockFromSnapshot(r, out, executionReader, cfg); err != nil { return v, err } if v <= clparams.BellatrixVersion { diff --git a/cl/persistence/format/snapshot_format/blocks_test.go b/cl/persistence/format/snapshot_format/blocks_test.go index 8021c3fcc38..807cfeb9ecb 100644 --- a/cl/persistence/format/snapshot_format/blocks_test.go +++ b/cl/persistence/format/snapshot_format/blocks_test.go @@ -57,7 +57,8 @@ func TestBlockSnapshotEncoding(t *testing.T) { br = snapshot_format.MockBlockReader{Block: blk.Block.Body.ExecutionPayload} } var b bytes.Buffer - require.NoError(t, snapshot_format.WriteBlockForSnapshot(blk, &b)) + _, err := snapshot_format.WriteBlockForSnapshot(&b, blk, nil) + require.NoError(t, err) blk2, err := snapshot_format.ReadBlockFromSnapshot(&b, &br, &clparams.MainnetBeaconConfig) require.NoError(t, err) _ = blk2 diff --git a/cl/persistence/format/snapshot_format/eth1_blocks.go b/cl/persistence/format/snapshot_format/eth1_blocks.go new file mode 100644 index 00000000000..053c075aa22 --- /dev/null +++ b/cl/persistence/format/snapshot_format/eth1_blocks.go @@ -0,0 +1,92 @@ +package snapshot_format + +import ( + "fmt" + "io" + + "github.com/ledgerwatch/erigon-lib/common/length" + "github.com/ledgerwatch/erigon/cl/clparams" + "github.com/ledgerwatch/erigon/cl/cltypes" + "github.com/ledgerwatch/erigon/cl/persistence/format/chunk_encoding" + "github.com/ledgerwatch/erigon/core/types" +) + +// WriteEth1BlockForSnapshot writes an execution block to the given writer in the format expected by the snapshot. +func writeEth1BlockForSnapshot(w io.Writer, encoded []byte, block *cltypes.Eth1Block) error { + pos := (length.Hash /*ParentHash*/ + length.Addr /*Miner*/ + length.Hash /*StateRoot*/ + length.Hash /*ReceiptsRoot*/ + types.BloomByteLength /*Bloom*/ + + length.Hash /*PrevRandao*/ + 32 /*BlockNumber + Timestamp + GasLimit + GasUsed */ + 4 /*ExtraDataOffset*/ + length.Hash /*BaseFee*/ + + length.Hash /*BlockHash*/ + 4 /*TransactionOffset*/) + + if block.Version() >= clparams.CapellaVersion { + pos += 4 /*WithdrawalsOffset*/ + } + if block.Version() >= clparams.DenebVersion { + pos += 16 /*BlobGasUsed + ExcessBlobGas*/ + } + // Add metadata first for Eth1Block, aka. version + if _, err := w.Write([]byte{byte(block.Version())}); err != nil { + return err + } + + // Maybe reuse the buffer? + pos += block.Extra.EncodingSizeSSZ() + if err := chunk_encoding.WriteChunk(w, encoded[:pos], chunk_encoding.ChunkDataType); err != nil { + return err + } + pos += block.Withdrawals.EncodingSizeSSZ() + pos += block.Transactions.EncodingSizeSSZ() + encoded = encoded[pos:] + //pos = 0 + // write the block pointer + if err := writeExecutionBlockPtr(w, block); err != nil { + return err + } + // From now on here, just finish up + return chunk_encoding.WriteChunk(w, encoded, chunk_encoding.ChunkDataType) +} + +func readEth1BlockFromSnapshot(r io.Reader, out io.Writer, executionReader ExecutionBlockReaderByNumber, cfg *clparams.BeaconChainConfig) (clparams.StateVersion, error) { + // Metadata section is just the current hardfork of the block. + vArr := make([]byte, 1) + if _, err := r.Read(vArr); err != nil { + return 0, err + } + v := clparams.StateVersion(vArr[0]) + + // Read the first chunk + dT1, err := chunk_encoding.ReadChunk(r, out) + if err != nil { + return v, err + } + if dT1 != chunk_encoding.ChunkDataType { + return v, fmt.Errorf("malformed beacon block, invalid chunk 1 type %d, expected: %d", dT1, chunk_encoding.ChunkDataType) + } + // Read the block pointer and retrieve chunk4 from the execution reader + blockNumber, blockHash, err := readExecutionBlockPtr(r) + if err != nil { + return v, err + } + err = executionReader.TransactionsSSZ(out, blockNumber, blockHash) + if err != nil { + return v, err + } + + if v < clparams.CapellaVersion { + return v, nil + } + err = executionReader.WithdrawalsSZZ(out, blockNumber, blockHash) + if err != nil { + return v, err + } + + // Read the 5h chunk + dT2, err := chunk_encoding.ReadChunk(r, out) + if err != nil { + return v, err + } + if dT2 != chunk_encoding.ChunkDataType { + return v, fmt.Errorf("malformed beacon block, invalid chunk 5 type %d, expected: %d", dT2, chunk_encoding.ChunkDataType) + } + + return v, nil +} diff --git a/cl/persistence/format/snapshot_format/test_util.go b/cl/persistence/format/snapshot_format/test_util.go index 3993c1648b7..1bf45999522 100644 --- a/cl/persistence/format/snapshot_format/test_util.go +++ b/cl/persistence/format/snapshot_format/test_util.go @@ -1,11 +1,30 @@ package snapshot_format -import "github.com/ledgerwatch/erigon/cl/cltypes" +import ( + "io" + + libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon/cl/cltypes" +) type MockBlockReader struct { Block *cltypes.Eth1Block } -func (t *MockBlockReader) BlockByNumber(number uint64) (*cltypes.Eth1Block, error) { - return t.Block, nil +func (t *MockBlockReader) WithdrawalsSZZ(out io.Writer, number uint64, hash libcommon.Hash) error { + l, err := t.Block.Withdrawals.EncodeSSZ(nil) + if err != nil { + return err + } + _, err = out.Write(l) + return err +} + +func (t *MockBlockReader) TransactionsSSZ(out io.Writer, number uint64, hash libcommon.Hash) error { + l, err := t.Block.Transactions.EncodeSSZ(nil) + if err != nil { + return err + } + _, err = out.Write(l) + return err } diff --git a/cl/sentinel/sentinel.go b/cl/sentinel/sentinel.go index 51fab5101fb..839906fb1bb 100644 --- a/cl/sentinel/sentinel.go +++ b/cl/sentinel/sentinel.go @@ -168,7 +168,7 @@ func (s *Sentinel) createListener() (*discover.UDPv5, error) { // Start stream handlers handlers.NewConsensusHandlers(s.ctx, s.db, s.host, s.peers, s.cfg.BeaconConfig, s.cfg.GenesisConfig, s.metadataV2).Start() - net, err := discover.ListenV5(s.ctx, conn, localNode, discCfg) + net, err := discover.ListenV5(s.ctx, "any", conn, localNode, discCfg) if err != nil { return nil, err } diff --git a/cl/spectest/consensus_tests/ssz_static.go b/cl/spectest/consensus_tests/ssz_static.go index b38e71df336..939078efe19 100644 --- a/cl/spectest/consensus_tests/ssz_static.go +++ b/cl/spectest/consensus_tests/ssz_static.go @@ -67,7 +67,8 @@ func getSSZStaticConsensusTest[T unmarshalerMarshalerHashable](ref T) spectest.H // Now let it do the encoding in snapshot format if blk, ok := object.(*cltypes.SignedBeaconBlock); ok { var b bytes.Buffer - require.NoError(t, snapshot_format.WriteBlockForSnapshot(blk, &b)) + _, err := snapshot_format.WriteBlockForSnapshot(&b, blk, nil) + require.NoError(t, err) var br snapshot_format.MockBlockReader if blk.Version() >= clparams.BellatrixVersion { br = snapshot_format.MockBlockReader{Block: blk.Block.Body.ExecutionPayload} diff --git a/cmd/bootnode/main.go b/cmd/bootnode/main.go index 7339cf06ab1..eedde266ad4 100644 --- a/cmd/bootnode/main.go +++ b/cmd/bootnode/main.go @@ -131,11 +131,11 @@ func main() { } if *runv5 { - if _, err := discover.ListenV5(ctx, conn, ln, cfg); err != nil { + if _, err := discover.ListenV5(ctx, "any", conn, ln, cfg); err != nil { utils.Fatalf("%v", err) } } else { - if _, err := discover.ListenUDP(ctx, conn, ln, cfg); err != nil { + if _, err := discover.ListenUDP(ctx, "any", conn, ln, cfg); err != nil { utils.Fatalf("%v", err) } } diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go index cfd04bb9426..f7b68eee0ab 100644 --- a/cmd/capcli/cli.go +++ b/cmd/capcli/cli.go @@ -456,7 +456,7 @@ func (c *DumpSnapshots) Run(ctx *Context) error { return }) - return freezeblocks.DumpBeaconBlocks(ctx, db, beaconDB, 0, to, snaptype.Erigon2MergeLimit, dirs.Tmp, dirs.Snap, 8, log.LvlInfo, log.Root()) + return freezeblocks.DumpBeaconBlocks(ctx, db, beaconDB, 0, to, snaptype.Erigon2RecentMergeLimit, dirs.Tmp, dirs.Snap, 8, log.LvlInfo, log.Root()) } type CheckSnapshots struct { @@ -495,7 +495,7 @@ func (c *CheckSnapshots) Run(ctx *Context) error { return err } - to = (to / snaptype.Erigon2MergeLimit) * snaptype.Erigon2MergeLimit + to = (to / snaptype.Erigon2RecentMergeLimit) * snaptype.Erigon2RecentMergeLimit csn := freezeblocks.NewCaplinSnapshots(ethconfig.BlocksFreezing{}, dirs.Snap, log.Root()) if err := csn.ReopenFolder(); err != nil { @@ -580,7 +580,7 @@ func (c *LoopSnapshots) Run(ctx *Context) error { return err } - to = (to / snaptype.Erigon2MergeLimit) * snaptype.Erigon2MergeLimit + to = (to / snaptype.Erigon2RecentMergeLimit) * snaptype.Erigon2RecentMergeLimit csn := freezeblocks.NewCaplinSnapshots(ethconfig.BlocksFreezing{}, dirs.Snap, log.Root()) if err := csn.ReopenFolder(); err != nil { diff --git a/cmd/devnet/args/node_args.go b/cmd/devnet/args/node_args.go index 9684d11c899..c5f7373a8bd 100644 --- a/cmd/devnet/args/node_args.go +++ b/cmd/devnet/args/node_args.go @@ -49,7 +49,7 @@ type NodeArgs struct { MetricsAddr string `arg:"--metrics.addr" json:"metrics.addr,omitempty"` StaticPeers string `arg:"--staticpeers" json:"staticpeers,omitempty"` WithoutHeimdall bool `arg:"--bor.withoutheimdall" flag:"" default:"false" json:"bor.withoutheimdall,omitempty"` - HeimdallGRpc string `arg:"--bor.heimdallgRPC" json:"bor.heimdallgRPC,omitempty"` + HeimdallGrpcAddr string `arg:"--bor.heimdallgRPC" json:"bor.heimdallgRPC,omitempty"` WithHeimdallMilestones bool `arg:"--bor.milestone" json:"bor.milestone"` VMDebug bool `arg:"--vmdebug" flag:"" default:"false" json:"dmdebug"` @@ -126,6 +126,11 @@ func (node *NodeArgs) GetEnodeURL() string { return enode.NewV4(&node.NodeKey.PublicKey, net.ParseIP("127.0.0.1"), port, port).URLv4() } +func (node *NodeArgs) EnableMetrics(port int) { + node.Metrics = true + node.MetricsPort = port +} + type BlockProducer struct { NodeArgs Mine bool `arg:"--mine" flag:"true"` diff --git a/cmd/devnet/devnet/context.go b/cmd/devnet/devnet/context.go index 97348746e75..54d9faccbc7 100644 --- a/cmd/devnet/devnet/context.go +++ b/cmd/devnet/devnet/context.go @@ -5,7 +5,6 @@ import ( "math/big" "github.com/ledgerwatch/log/v3" - "github.com/urfave/cli/v2" ) type ctxKey int @@ -14,7 +13,6 @@ const ( ckLogger ctxKey = iota ckNetwork ckNode - ckCliContext ckDevnet ) @@ -71,11 +69,10 @@ type cnet struct { network *Network } -func WithDevnet(ctx context.Context, cliCtx *cli.Context, devnet Devnet, logger log.Logger) Context { - return WithCliContext( - context.WithValue( - context.WithValue(ctx, ckDevnet, devnet), - ckLogger, logger), cliCtx) +func WithDevnet(ctx context.Context, devnet Devnet, logger log.Logger) Context { + ctx = context.WithValue(ctx, ckDevnet, devnet) + ctx = context.WithValue(ctx, ckLogger, logger) + return devnetContext{ctx} } func WithCurrentNetwork(ctx context.Context, selector interface{}) Context { @@ -107,14 +104,6 @@ func WithCurrentNode(ctx context.Context, selector interface{}) Context { return devnetContext{context.WithValue(ctx, ckNode, &cnode{selector: selector})} } -func WithCliContext(ctx context.Context, cliCtx *cli.Context) Context { - return devnetContext{context.WithValue(ctx, ckCliContext, cliCtx)} -} - -func CliContext(ctx context.Context) *cli.Context { - return ctx.Value(ckCliContext).(*cli.Context) -} - func CurrentChainID(ctx context.Context) *big.Int { if network := CurrentNetwork(ctx); network != nil { return network.ChainID() diff --git a/cmd/devnet/devnet/devnet.go b/cmd/devnet/devnet/devnet.go index 310db056802..3fc09bc6b7e 100644 --- a/cmd/devnet/devnet/devnet.go +++ b/cmd/devnet/devnet/devnet.go @@ -7,7 +7,6 @@ import ( "sync" "github.com/ledgerwatch/log/v3" - "github.com/urfave/cli/v2" ) type Devnet []*Network @@ -22,12 +21,12 @@ func (f NetworkSelectorFunc) Test(ctx context.Context, network *Network) bool { return f(ctx, network) } -func (d Devnet) Start(ctx *cli.Context, logger log.Logger) (Context, error) { +func (d Devnet) Start(logger log.Logger) (Context, error) { var wg sync.WaitGroup errors := make(chan error, len(d)) - runCtx := WithDevnet(context.Background(), ctx, d, logger) + runCtx := WithDevnet(context.Background(), d, logger) for _, network := range d { wg.Add(1) diff --git a/cmd/devnet/devnet/network.go b/cmd/devnet/devnet/network.go index 2f6e948d827..2cb282b781b 100644 --- a/cmd/devnet/devnet/network.go +++ b/cmd/devnet/devnet/network.go @@ -12,7 +12,7 @@ import ( "time" "github.com/ledgerwatch/erigon-lib/common/dbg" - "github.com/ledgerwatch/erigon/cmd/devnet/args" + devnet_args "github.com/ledgerwatch/erigon/cmd/devnet/args" "github.com/ledgerwatch/erigon/cmd/devnet/requests" "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/params" @@ -41,6 +41,9 @@ type Network struct { wg sync.WaitGroup peers []string namedNodes map[string]Node + + // max number of blocks to look for a transaction in + MaxNumberOfEmptyBlockChecks int } func (nw *Network) ChainID() *big.Int { @@ -60,7 +63,7 @@ func (nw *Network) Start(ctx context.Context) error { } } - baseNode := args.NodeArgs{ + baseNode := devnet_args.NodeArgs{ DataDir: nw.DataDir, Chain: nw.Chain, Port: nw.BasePort, @@ -75,22 +78,13 @@ func (nw *Network) Start(ctx context.Context) error { baseNode.WithHeimdallMilestones = utils.WithHeimdallMilestones.Value } - cliCtx := CliContext(ctx) - - metricsEnabled := cliCtx.Bool("metrics") - metricsNode := cliCtx.Int("metrics.node") nw.namedNodes = map[string]Node{} for i, nodeArgs := range nw.Nodes { { - base := baseNode - if metricsEnabled && metricsNode == i { - base.Metrics = true - base.MetricsPort = cliCtx.Int("metrics.port") - } - base.StaticPeers = strings.Join(nw.peers, ",") + baseNode.StaticPeers = strings.Join(nw.peers, ",") - err := nodeArgs.Configure(base, i) + err := nodeArgs.Configure(baseNode, i) if err != nil { nw.Stop() return err @@ -182,7 +176,7 @@ func (nw *Network) startNode(n Node) error { node := n.(*devnetNode) - args, err := args.AsArgs(node.nodeArgs) + args, err := devnet_args.AsArgs(node.nodeArgs) if err != nil { return err } diff --git a/cmd/devnet/devnet/node.go b/cmd/devnet/devnet/node.go index abba7715d68..4c372721a03 100644 --- a/cmd/devnet/devnet/node.go +++ b/cmd/devnet/devnet/node.go @@ -30,6 +30,7 @@ type Node interface { Account() *accounts.Account IsBlockProducer() bool Configure(baseNode args.NodeArgs, nodeNumber int) error + EnableMetrics(port int) } type NodeSelector interface { @@ -129,6 +130,10 @@ func (n *devnetNode) GetEnodeURL() string { return n.nodeArgs.GetEnodeURL() } +func (n *devnetNode) EnableMetrics(int) { + panic("not implemented") +} + // run configures, creates and serves an erigon node func (n *devnetNode) run(ctx *cli.Context) error { var logger log.Logger diff --git a/cmd/devnet/main.go b/cmd/devnet/main.go index 5de9d579018..d241040c09a 100644 --- a/cmd/devnet/main.go +++ b/cmd/devnet/main.go @@ -1,9 +1,7 @@ package main import ( - "context" "fmt" - "github.com/ledgerwatch/erigon/cmd/utils" "os" "os/signal" "path/filepath" @@ -12,28 +10,25 @@ import ( "syscall" "time" + "github.com/ledgerwatch/erigon/cmd/devnet/services" + "github.com/ledgerwatch/erigon/cmd/devnet/services/polygon" + "github.com/ledgerwatch/erigon-lib/chain/networkname" + "github.com/ledgerwatch/erigon-lib/common/metrics" "github.com/ledgerwatch/erigon/cmd/devnet/accounts" _ "github.com/ledgerwatch/erigon/cmd/devnet/accounts/steps" _ "github.com/ledgerwatch/erigon/cmd/devnet/admin" _ "github.com/ledgerwatch/erigon/cmd/devnet/contracts/steps" - account_services "github.com/ledgerwatch/erigon/cmd/devnet/services/accounts" - "github.com/ledgerwatch/erigon/cmd/devnet/services/polygon" - "github.com/ledgerwatch/erigon/cmd/devnet/transactions" - "github.com/ledgerwatch/erigon/core/types" - - "github.com/ledgerwatch/erigon-lib/common/metrics" - "github.com/ledgerwatch/erigon/cmd/devnet/args" "github.com/ledgerwatch/erigon/cmd/devnet/devnet" "github.com/ledgerwatch/erigon/cmd/devnet/devnetutils" "github.com/ledgerwatch/erigon/cmd/devnet/requests" "github.com/ledgerwatch/erigon/cmd/devnet/scenarios" - "github.com/ledgerwatch/erigon/cmd/devnet/services" + "github.com/ledgerwatch/erigon/cmd/devnet/tests" "github.com/ledgerwatch/log/v3" "github.com/ledgerwatch/erigon/cmd/utils/flags" "github.com/ledgerwatch/erigon/params" - "github.com/ledgerwatch/erigon/turbo/app" + erigon_app "github.com/ledgerwatch/erigon/turbo/app" "github.com/ledgerwatch/erigon/turbo/debug" "github.com/ledgerwatch/erigon/turbo/logging" "github.com/urfave/cli/v2" @@ -81,10 +76,10 @@ var ( Usage: "Run with a devnet local Heimdall service", } - HeimdallgRPCAddressFlag = cli.StringFlag{ + HeimdallGrpcAddressFlag = cli.StringFlag{ Name: "bor.heimdallgRPC", Usage: "Address of Heimdall gRPC service", - Value: "localhost:8540", + Value: polygon.HeimdallGrpcAddressDefault, } BorSprintSizeFlag = cli.IntFlag{ @@ -135,19 +130,15 @@ type PanicHandler struct { func (ph PanicHandler) Log(r *log.Record) error { fmt.Printf("Msg: %s\nStack: %s\n", r.Msg, dbg.Stack()) - os.Exit(1) + os.Exit(2) return nil } func main() { - - debug.RaiseFdLimit() - app := cli.NewApp() app.Version = params.VersionWithCommit(params.GitCommit) - app.Action = func(ctx *cli.Context) error { - return action(ctx) - } + app.Action = mainContext + app.Flags = []cli.Flag{ &DataDirFlag, &ChainFlag, @@ -156,7 +147,7 @@ func main() { &BaseRpcPortFlag, &WithoutHeimdallFlag, &LocalHeimdallFlag, - &HeimdallgRPCAddressFlag, + &HeimdallGrpcAddressFlag, &BorSprintSizeFlag, &MetricsEnabledFlag, &MetricsNodeFlag, @@ -170,27 +161,18 @@ func main() { &logging.LogDirVerbosityFlag, } - app.After = func(ctx *cli.Context) error { - // unsubscribe from all the subscriptions made - services.UnsubscribeAll() - return nil - } if err := app.Run(os.Args); err != nil { - fmt.Fprintln(os.Stderr, err) + _, _ = fmt.Fprintln(os.Stderr, err) + os.Exit(1) } } -const ( - recipientAddress = "0x71562b71999873DB5b286dF957af199Ec94617F7" - sendValue uint64 = 10000 -) - -func action(ctx *cli.Context) error { - dataDir := ctx.String("datadir") +func setupLogger(ctx *cli.Context) (log.Logger, error) { + dataDir := ctx.String(DataDirFlag.Name) logsDir := filepath.Join(dataDir, "logs") if err := os.MkdirAll(logsDir, 0755); err != nil { - return err + return nil, err } logger := logging.SetupLoggerCtx("devnet", ctx, false /* rootLogger */) @@ -198,65 +180,92 @@ func action(ctx *cli.Context) error { // Make root logger fail log.Root().SetHandler(PanicHandler{}) + return logger, nil +} + +func handleTerminationSignals(stopFunc func(), logger log.Logger) { + signalCh := make(chan os.Signal, 1) + signal.Notify(signalCh, syscall.SIGTERM, syscall.SIGINT) + + switch s := <-signalCh; s { + case syscall.SIGTERM: + logger.Info("Stopping networks") + stopFunc() + case syscall.SIGINT: + logger.Info("Terminating network") + os.Exit(-int(syscall.SIGINT)) + } +} + +func connectDiagnosticsIfEnabled(ctx *cli.Context, logger log.Logger) { + metricsEnabled := ctx.Bool(MetricsEnabledFlag.Name) + diagnosticsUrl := ctx.String(DiagnosticsURLFlag.Name) + if metricsEnabled && len(diagnosticsUrl) > 0 { + err := erigon_app.ConnectDiagnostics(ctx, logger) + if err != nil { + logger.Error("app.ConnectDiagnostics failed", "err", err) + } + } +} + +func mainContext(ctx *cli.Context) error { + debug.RaiseFdLimit() + + logger, err := setupLogger(ctx) + if err != nil { + return err + } + // clear all the dev files + dataDir := ctx.String(DataDirFlag.Name) if err := devnetutils.ClearDevDB(dataDir, logger); err != nil { return err } network, err := initDevnet(ctx, logger) - if err != nil { return err } - metrics := ctx.Bool("metrics") - - if metrics { - // TODO should get this from the network as once we have multiple nodes we'll need to iterate the - // nodes and create a series of urls - for the moment only one is supported - ctx.Set("metrics.urls", fmt.Sprintf("http://localhost:%d/debug/", ctx.Int("metrics.port"))) + if err = initDevnetMetrics(ctx, network); err != nil { + return err } - // start the network with each node in a go routine logger.Info("Starting Devnet") - - runCtx, err := network.Start(ctx, logger) - + runCtx, err := network.Start(logger) if err != nil { - return fmt.Errorf("Devnet start failed: %w", err) + return fmt.Errorf("devnet start failed: %w", err) } - go func() { - signalCh := make(chan os.Signal, 1) - signal.Notify(signalCh, syscall.SIGTERM, syscall.SIGINT) + go handleTerminationSignals(network.Stop, logger) + go connectDiagnosticsIfEnabled(ctx, logger) - switch s := <-signalCh; s { - case syscall.SIGTERM: - logger.Info("Stopping networks") - network.Stop() - case syscall.SIGINT: - logger.Info("Terminating network") - os.Exit(-int(syscall.SIGINT)) - } - }() - - diagnosticsUrl := ctx.String("diagnostics.url") - - if metrics && len(diagnosticsUrl) > 0 { - go func() { - app.ConnectDiagnostics(ctx, logger) - }() + enabledScenarios := strings.Split(ctx.String(ScenariosFlag.Name), ",") + if err = allScenarios(runCtx).Run(runCtx, enabledScenarios...); err != nil { + return err } - if ctx.String(ChainFlag.Name) == networkname.DevChainName { - transactions.MaxNumberOfEmptyBlockChecks = 30 + if ctx.Bool(WaitFlag.Name) { + logger.Info("Waiting") + network.Wait() + } else { + logger.Info("Stopping Networks") + network.Stop() } - scenarios.Scenarios{ + return nil +} + +func allScenarios(runCtx devnet.Context) scenarios.Scenarios { + // unsubscribe from all the subscriptions made + defer services.UnsubscribeAll() + + const recipientAddress = "0x71562b71999873DB5b286dF957af199Ec94617F7" + const sendValue uint64 = 10000 + + return scenarios.Scenarios{ "dynamic-tx-node-0": { - Context: runCtx. - WithCurrentNetwork(0). - WithCurrentNode(0), + Context: runCtx.WithCurrentNetwork(0).WithCurrentNode(0), Steps: []*scenarios.Step{ {Text: "InitSubscriptions", Args: []any{[]requests.SubMethod{requests.Methods.ETHNewHeads}}}, {Text: "PingErigonRpc"}, @@ -304,205 +313,52 @@ func action(ctx *cli.Context) error { //{Text: "BatchProcessTransfers", Args: []any{"child-funder", 1, 10, 2, 2}}, }, }, - }.Run(runCtx, strings.Split(ctx.String("scenarios"), ",")...) - - if ctx.Bool("wait") || (metrics && len(diagnosticsUrl) > 0) { - logger.Info("Waiting") - network.Wait() - } else { - logger.Info("Stopping Networks") - network.Stop() } - - return nil } func initDevnet(ctx *cli.Context, logger log.Logger) (devnet.Devnet, error) { dataDir := ctx.String(DataDirFlag.Name) - chain := ctx.String(ChainFlag.Name) + chainName := ctx.String(ChainFlag.Name) baseRpcHost := ctx.String(BaseRpcHostFlag.Name) baseRpcPort := ctx.Int(BaseRpcPortFlag.Name) - faucetSource := accounts.NewAccount("faucet-source") - - switch chain { + switch chainName { case networkname.BorDevnetChainName: if ctx.Bool(WithoutHeimdallFlag.Name) { - return []*devnet.Network{ - { - DataDir: dataDir, - Chain: networkname.BorDevnetChainName, - Logger: logger, - BasePort: 40303, - BasePrivateApiAddr: "localhost:10090", - BaseRPCHost: baseRpcHost, - BaseRPCPort: baseRpcPort, - //Snapshots: true, - Alloc: types.GenesisAlloc{ - faucetSource.Address: {Balance: accounts.EtherAmount(200_000)}, - }, - Services: []devnet.Service{ - account_services.NewFaucet(networkname.BorDevnetChainName, faucetSource), - }, - Nodes: []devnet.Node{ - &args.BlockProducer{ - NodeArgs: args.NodeArgs{ - ConsoleVerbosity: "0", - DirVerbosity: "5", - WithoutHeimdall: true, - }, - AccountSlots: 200, - }, - &args.NonBlockProducer{ - NodeArgs: args.NodeArgs{ - ConsoleVerbosity: "0", - DirVerbosity: "5", - WithoutHeimdall: true, - }, - }, - }, - }}, nil + return tests.NewBorDevnetWithoutHeimdall(dataDir, baseRpcHost, baseRpcPort, logger), nil + } else if ctx.Bool(LocalHeimdallFlag.Name) { + heimdallGrpcAddr := ctx.String(HeimdallGrpcAddressFlag.Name) + sprintSize := uint64(ctx.Int(BorSprintSizeFlag.Name)) + return tests.NewBorDevnetWithLocalHeimdall(dataDir, baseRpcHost, baseRpcPort, heimdallGrpcAddr, sprintSize, logger), nil } else { - var heimdallGrpc string - var services []devnet.Service - var withMilestones = utils.WithHeimdallMilestones.Value + return tests.NewBorDevnetWithRemoteHeimdall(dataDir, baseRpcHost, baseRpcPort, logger), nil + } - checkpointOwner := accounts.NewAccount("checkpoint-owner") + case networkname.DevChainName: + return tests.NewDevDevnet(dataDir, baseRpcHost, baseRpcPort, logger), nil - if ctx.Bool(LocalHeimdallFlag.Name) { - config := *params.BorDevnetChainConfig - // milestones are not supported yet on the local heimdall - withMilestones = false + default: + return nil, fmt.Errorf("unknown network: '%s'", chainName) + } +} - if sprintSize := uint64(ctx.Int(BorSprintSizeFlag.Name)); sprintSize > 0 { - config.Bor.Sprint = map[string]uint64{"0": sprintSize} - } +func initDevnetMetrics(ctx *cli.Context, network devnet.Devnet) error { + metricsEnabled := ctx.Bool(MetricsEnabledFlag.Name) + metricsNode := ctx.Int(MetricsNodeFlag.Name) + metricsPort := ctx.Int(MetricsPortFlag.Name) - services = append(services, polygon.NewHeimdall(&config, - &polygon.CheckpointConfig{ - CheckpointBufferTime: 60 * time.Second, - CheckpointAccount: checkpointOwner, - }, - logger)) + if !metricsEnabled { + return nil + } - heimdallGrpc = polygon.HeimdallGRpc(devnet.WithCliContext(context.Background(), ctx)) + for _, nw := range network { + for i, nodeArgs := range nw.Nodes { + if metricsEnabled && (metricsNode == i) { + nodeArgs.EnableMetrics(metricsPort) + return nil } - - return []*devnet.Network{ - { - DataDir: dataDir, - Chain: networkname.BorDevnetChainName, - Logger: logger, - BasePort: 40303, - BasePrivateApiAddr: "localhost:10090", - BaseRPCHost: baseRpcHost, - BaseRPCPort: baseRpcPort, - BorStateSyncDelay: 5 * time.Second, - BorWithMilestones: &withMilestones, - Services: append(services, account_services.NewFaucet(networkname.BorDevnetChainName, faucetSource)), - Alloc: types.GenesisAlloc{ - faucetSource.Address: {Balance: accounts.EtherAmount(200_000)}, - }, - Nodes: []devnet.Node{ - &args.BlockProducer{ - NodeArgs: args.NodeArgs{ - ConsoleVerbosity: "0", - DirVerbosity: "5", - HeimdallGRpc: heimdallGrpc, - }, - AccountSlots: 200, - }, - &args.BlockProducer{ - NodeArgs: args.NodeArgs{ - ConsoleVerbosity: "0", - DirVerbosity: "5", - HeimdallGRpc: heimdallGrpc, - }, - AccountSlots: 200, - }, - /*&args.BlockProducer{ - Node: args.Node{ - ConsoleVerbosity: "0", - DirVerbosity: "5", - HeimdallGRpc: heimdallGrpc, - }, - AccountSlots: 200, - },*/ - &args.NonBlockProducer{ - NodeArgs: args.NodeArgs{ - ConsoleVerbosity: "0", - DirVerbosity: "5", - HeimdallGRpc: heimdallGrpc, - }, - }, - }, - }, - { - DataDir: dataDir, - Chain: networkname.DevChainName, - Logger: logger, - BasePort: 30403, - BasePrivateApiAddr: "localhost:10190", - BaseRPCHost: baseRpcHost, - BaseRPCPort: baseRpcPort + 1000, - Services: append(services, account_services.NewFaucet(networkname.DevChainName, faucetSource)), - Alloc: types.GenesisAlloc{ - faucetSource.Address: {Balance: accounts.EtherAmount(200_000)}, - checkpointOwner.Address: {Balance: accounts.EtherAmount(10_000)}, - }, - Nodes: []devnet.Node{ - &args.BlockProducer{ - NodeArgs: args.NodeArgs{ - ConsoleVerbosity: "0", - DirVerbosity: "5", - VMDebug: true, - HttpCorsDomain: "*", - }, - DevPeriod: 5, - AccountSlots: 200, - }, - &args.NonBlockProducer{ - NodeArgs: args.NodeArgs{ - ConsoleVerbosity: "0", - DirVerbosity: "3", - }, - }, - }, - }}, nil } + } - case networkname.DevChainName: - return []*devnet.Network{ - { - DataDir: dataDir, - Chain: networkname.DevChainName, - Logger: logger, - BasePrivateApiAddr: "localhost:10090", - BaseRPCHost: baseRpcHost, - BaseRPCPort: baseRpcPort, - Alloc: types.GenesisAlloc{ - faucetSource.Address: {Balance: accounts.EtherAmount(200_000)}, - }, - Services: []devnet.Service{ - account_services.NewFaucet(networkname.DevChainName, faucetSource), - }, - Nodes: []devnet.Node{ - &args.BlockProducer{ - NodeArgs: args.NodeArgs{ - ConsoleVerbosity: "0", - DirVerbosity: "5", - }, - AccountSlots: 200, - }, - &args.NonBlockProducer{ - NodeArgs: args.NodeArgs{ - ConsoleVerbosity: "0", - DirVerbosity: "5", - }, - }, - }, - }}, nil - } - - return nil, fmt.Errorf(`Unknown network: "%s"`, chain) + return fmt.Errorf("initDevnetMetrics: not found %s=%d", MetricsNodeFlag.Name, metricsNode) } diff --git a/cmd/devnet/services/polygon/heimdall.go b/cmd/devnet/services/polygon/heimdall.go index a55ce00491e..0581a703949 100644 --- a/cmd/devnet/services/polygon/heimdall.go +++ b/cmd/devnet/services/polygon/heimdall.go @@ -52,6 +52,8 @@ const ( DefaultCheckpointBufferTime time.Duration = 1000 * time.Second ) +const HeimdallGrpcAddressDefault = "localhost:8540" + type CheckpointConfig struct { RootChainTxConfirmations uint64 ChildChainTxConfirmations uint64 @@ -65,6 +67,7 @@ type CheckpointConfig struct { type Heimdall struct { sync.Mutex chainConfig *chain.Config + grpcAddr string validatorSet *valset.ValidatorSet pendingCheckpoint *checkpoint.Checkpoint latestCheckpoint *CheckpointAck @@ -85,9 +88,15 @@ type Heimdall struct { startTime time.Time } -func NewHeimdall(chainConfig *chain.Config, checkpointConfig *CheckpointConfig, logger log.Logger) *Heimdall { +func NewHeimdall( + chainConfig *chain.Config, + grpcAddr string, + checkpointConfig *CheckpointConfig, + logger log.Logger, +) *Heimdall { heimdall := &Heimdall{ chainConfig: chainConfig, + grpcAddr: grpcAddr, checkpointConfig: *checkpointConfig, spans: map[uint64]*span.HeimdallSpan{}, pendingSyncRecords: map[syncRecordKey]*EventRecordWithBlock{}, @@ -368,19 +377,7 @@ func (h *Heimdall) Start(ctx context.Context) error { // if this is a restart h.unsubscribe() - return heimdallgrpc.StartHeimdallServer(ctx, h, HeimdallGRpc(ctx), h.logger) -} - -func HeimdallGRpc(ctx context.Context) string { - addr := "localhost:8540" - - if cli := devnet.CliContext(ctx); cli != nil { - if grpcAddr := cli.String("bor.heimdallgRPC"); len(grpcAddr) > 0 { - addr = grpcAddr - } - } - - return addr + return heimdallgrpc.StartHeimdallServer(ctx, h, h.grpcAddr, h.logger) } func (h *Heimdall) Stop() { diff --git a/cmd/devnet/tests/bor/devnet_test.go b/cmd/devnet/tests/bor/devnet_test.go new file mode 100644 index 00000000000..fa8d66ec005 --- /dev/null +++ b/cmd/devnet/tests/bor/devnet_test.go @@ -0,0 +1,87 @@ +//go:build integration + +package bor + +import ( + "context" + "github.com/ledgerwatch/erigon-lib/chain/networkname" + accounts_steps "github.com/ledgerwatch/erigon/cmd/devnet/accounts/steps" + contracts_steps "github.com/ledgerwatch/erigon/cmd/devnet/contracts/steps" + "github.com/ledgerwatch/erigon/cmd/devnet/requests" + "github.com/ledgerwatch/erigon/cmd/devnet/services" + "github.com/ledgerwatch/erigon/cmd/devnet/tests" + "github.com/stretchr/testify/require" + "testing" +) + +func TestStateSync(t *testing.T) { + t.Skip("FIXME: hangs in GenerateSyncEvents without any visible progress") + + runCtx, err := tests.ContextStart(t, networkname.BorDevnetChainName) + require.Nil(t, err) + var ctx context.Context = runCtx + + t.Run("InitSubscriptions", func(t *testing.T) { + services.InitSubscriptions(ctx, []requests.SubMethod{requests.Methods.ETHNewHeads}) + }) + t.Run("CreateAccountWithFunds", func(t *testing.T) { + _, err := accounts_steps.CreateAccountWithFunds(ctx, networkname.DevChainName, "root-funder", 200.0) + require.Nil(t, err) + }) + t.Run("CreateAccountWithFunds", func(t *testing.T) { + _, err := accounts_steps.CreateAccountWithFunds(ctx, networkname.BorDevnetChainName, "child-funder", 200.0) + require.Nil(t, err) + }) + t.Run("DeployChildChainReceiver", func(t *testing.T) { + var err error + ctx, err = contracts_steps.DeployChildChainReceiver(ctx, "child-funder") + require.Nil(t, err) + }) + t.Run("DeployRootChainSender", func(t *testing.T) { + var err error + ctx, err = contracts_steps.DeployRootChainSender(ctx, "root-funder") + require.Nil(t, err) + }) + t.Run("GenerateSyncEvents", func(t *testing.T) { + require.Nil(t, contracts_steps.GenerateSyncEvents(ctx, "root-funder", 10, 2, 2)) + }) + t.Run("ProcessRootTransfers", func(t *testing.T) { + require.Nil(t, contracts_steps.ProcessRootTransfers(ctx, "root-funder", 10, 2, 2)) + }) + t.Run("BatchProcessRootTransfers", func(t *testing.T) { + require.Nil(t, contracts_steps.BatchProcessRootTransfers(ctx, "root-funder", 1, 10, 2, 2)) + }) +} + +func TestChildChainExit(t *testing.T) { + t.Skip("FIXME: step CreateAccountWithFunds fails: Failed to get transfer tx: failed to search reserves for hashes: no block heads subscription") + + runCtx, err := tests.ContextStart(t, networkname.BorDevnetChainName) + require.Nil(t, err) + var ctx context.Context = runCtx + + t.Run("CreateAccountWithFunds", func(t *testing.T) { + _, err := accounts_steps.CreateAccountWithFunds(ctx, networkname.DevChainName, "root-funder", 200.0) + require.Nil(t, err) + }) + t.Run("CreateAccountWithFunds", func(t *testing.T) { + _, err := accounts_steps.CreateAccountWithFunds(ctx, networkname.BorDevnetChainName, "child-funder", 200.0) + require.Nil(t, err) + }) + t.Run("DeployRootChainReceiver", func(t *testing.T) { + var err error + ctx, err = contracts_steps.DeployRootChainReceiver(ctx, "root-funder") + require.Nil(t, err) + }) + t.Run("DeployChildChainSender", func(t *testing.T) { + var err error + ctx, err = contracts_steps.DeployChildChainSender(ctx, "child-funder") + require.Nil(t, err) + }) + t.Run("ProcessChildTransfers", func(t *testing.T) { + require.Nil(t, contracts_steps.ProcessChildTransfers(ctx, "child-funder", 1, 2, 2)) + }) + //t.Run("BatchProcessTransfers", func(t *testing.T) { + // require.Nil(t, contracts_steps.BatchProcessTransfers(ctx, "child-funder", 1, 10, 2, 2)) + //}) +} diff --git a/cmd/devnet/tests/context.go b/cmd/devnet/tests/context.go new file mode 100644 index 00000000000..6658ed80125 --- /dev/null +++ b/cmd/devnet/tests/context.go @@ -0,0 +1,65 @@ +package tests + +import ( + "fmt" + "github.com/ledgerwatch/erigon-lib/chain/networkname" + "github.com/ledgerwatch/erigon/cmd/devnet/devnet" + "github.com/ledgerwatch/erigon/cmd/devnet/services" + "github.com/ledgerwatch/erigon/cmd/devnet/services/polygon" + "github.com/ledgerwatch/erigon/turbo/debug" + "github.com/ledgerwatch/log/v3" + "os" + "runtime" + "testing" +) + +func initDevnet(chainName string, dataDir string, logger log.Logger) (devnet.Devnet, error) { + const baseRpcHost = "localhost" + const baseRpcPort = 8545 + + switch chainName { + case networkname.BorDevnetChainName: + heimdallGrpcAddr := polygon.HeimdallGrpcAddressDefault + const sprintSize uint64 = 0 + return NewBorDevnetWithLocalHeimdall(dataDir, baseRpcHost, baseRpcPort, heimdallGrpcAddr, sprintSize, logger), nil + + case networkname.DevChainName: + return NewDevDevnet(dataDir, baseRpcHost, baseRpcPort, logger), nil + + case "": + envChainName, _ := os.LookupEnv("DEVNET_CHAIN") + if envChainName == "" { + envChainName = networkname.DevChainName + } + return initDevnet(envChainName, dataDir, logger) + + default: + return nil, fmt.Errorf("unknown network: '%s'", chainName) + } +} + +func ContextStart(t *testing.T, chainName string) (devnet.Context, error) { + if runtime.GOOS == "windows" { + t.Skip("FIXME: TempDir RemoveAll cleanup error: remove dev-0\\clique\\db\\clique\\mdbx.dat: The process cannot access the file because it is being used by another process") + } + + debug.RaiseFdLimit() + logger := log.New() + dataDir := t.TempDir() + + var network devnet.Devnet + network, err := initDevnet(chainName, dataDir, logger) + if err != nil { + return nil, fmt.Errorf("ContextStart initDevnet failed: %w", err) + } + + runCtx, err := network.Start(logger) + if err != nil { + return nil, fmt.Errorf("ContextStart devnet start failed: %w", err) + } + + t.Cleanup(services.UnsubscribeAll) + t.Cleanup(network.Stop) + + return runCtx, nil +} diff --git a/cmd/devnet/tests/devnet_bor.go b/cmd/devnet/tests/devnet_bor.go new file mode 100644 index 00000000000..f1b88a197ac --- /dev/null +++ b/cmd/devnet/tests/devnet_bor.go @@ -0,0 +1,221 @@ +package tests + +import ( + "github.com/ledgerwatch/erigon-lib/chain/networkname" + "github.com/ledgerwatch/erigon/cmd/devnet/accounts" + "github.com/ledgerwatch/erigon/cmd/devnet/args" + "github.com/ledgerwatch/erigon/cmd/devnet/devnet" + account_services "github.com/ledgerwatch/erigon/cmd/devnet/services/accounts" + "github.com/ledgerwatch/erigon/cmd/devnet/services/polygon" + "github.com/ledgerwatch/erigon/cmd/utils" + "github.com/ledgerwatch/erigon/core/types" + "github.com/ledgerwatch/erigon/params" + "github.com/ledgerwatch/log/v3" + "time" +) + +func NewBorDevnetWithoutHeimdall( + dataDir string, + baseRpcHost string, + baseRpcPort int, + logger log.Logger, +) devnet.Devnet { + faucetSource := accounts.NewAccount("faucet-source") + + network := devnet.Network{ + DataDir: dataDir, + Chain: networkname.BorDevnetChainName, + Logger: logger, + BasePort: 40303, + BasePrivateApiAddr: "localhost:10090", + BaseRPCHost: baseRpcHost, + BaseRPCPort: baseRpcPort, + //Snapshots: true, + Alloc: types.GenesisAlloc{ + faucetSource.Address: {Balance: accounts.EtherAmount(200_000)}, + }, + Services: []devnet.Service{ + account_services.NewFaucet(networkname.BorDevnetChainName, faucetSource), + }, + Nodes: []devnet.Node{ + &args.BlockProducer{ + NodeArgs: args.NodeArgs{ + ConsoleVerbosity: "0", + DirVerbosity: "5", + WithoutHeimdall: true, + }, + AccountSlots: 200, + }, + &args.NonBlockProducer{ + NodeArgs: args.NodeArgs{ + ConsoleVerbosity: "0", + DirVerbosity: "5", + WithoutHeimdall: true, + }, + }, + }, + } + + return devnet.Devnet{&network} +} + +func NewBorDevnetWithHeimdall( + dataDir string, + baseRpcHost string, + baseRpcPort int, + heimdall *polygon.Heimdall, + heimdallGrpcAddr string, + checkpointOwner *accounts.Account, + withMilestones bool, + logger log.Logger, +) devnet.Devnet { + faucetSource := accounts.NewAccount("faucet-source") + + var services []devnet.Service + if heimdall != nil { + services = append(services, heimdall) + } + + borNetwork := devnet.Network{ + DataDir: dataDir, + Chain: networkname.BorDevnetChainName, + Logger: logger, + BasePort: 40303, + BasePrivateApiAddr: "localhost:10090", + BaseRPCHost: baseRpcHost, + BaseRPCPort: baseRpcPort, + BorStateSyncDelay: 5 * time.Second, + BorWithMilestones: &withMilestones, + Services: append(services, account_services.NewFaucet(networkname.BorDevnetChainName, faucetSource)), + Alloc: types.GenesisAlloc{ + faucetSource.Address: {Balance: accounts.EtherAmount(200_000)}, + }, + Nodes: []devnet.Node{ + &args.BlockProducer{ + NodeArgs: args.NodeArgs{ + ConsoleVerbosity: "0", + DirVerbosity: "5", + HeimdallGrpcAddr: heimdallGrpcAddr, + }, + AccountSlots: 200, + }, + &args.BlockProducer{ + NodeArgs: args.NodeArgs{ + ConsoleVerbosity: "0", + DirVerbosity: "5", + HeimdallGrpcAddr: heimdallGrpcAddr, + }, + AccountSlots: 200, + }, + /*&args.BlockProducer{ + Node: args.Node{ + ConsoleVerbosity: "0", + DirVerbosity: "5", + HeimdallGrpcAddr: heimdallGrpcAddr, + }, + AccountSlots: 200, + },*/ + &args.NonBlockProducer{ + NodeArgs: args.NodeArgs{ + ConsoleVerbosity: "0", + DirVerbosity: "5", + HeimdallGrpcAddr: heimdallGrpcAddr, + }, + }, + }, + } + + devNetwork := devnet.Network{ + DataDir: dataDir, + Chain: networkname.DevChainName, + Logger: logger, + BasePort: 30403, + BasePrivateApiAddr: "localhost:10190", + BaseRPCHost: baseRpcHost, + BaseRPCPort: baseRpcPort + 1000, + Services: append(services, account_services.NewFaucet(networkname.DevChainName, faucetSource)), + Alloc: types.GenesisAlloc{ + faucetSource.Address: {Balance: accounts.EtherAmount(200_000)}, + checkpointOwner.Address: {Balance: accounts.EtherAmount(10_000)}, + }, + Nodes: []devnet.Node{ + &args.BlockProducer{ + NodeArgs: args.NodeArgs{ + ConsoleVerbosity: "0", + DirVerbosity: "5", + VMDebug: true, + HttpCorsDomain: "*", + }, + DevPeriod: 5, + AccountSlots: 200, + }, + &args.NonBlockProducer{ + NodeArgs: args.NodeArgs{ + ConsoleVerbosity: "0", + DirVerbosity: "3", + }, + }, + }, + } + + return devnet.Devnet{ + &borNetwork, + &devNetwork, + } +} + +func NewBorDevnetWithRemoteHeimdall( + dataDir string, + baseRpcHost string, + baseRpcPort int, + logger log.Logger, +) devnet.Devnet { + heimdallGrpcAddr := "" + checkpointOwner := accounts.NewAccount("checkpoint-owner") + withMilestones := utils.WithHeimdallMilestones.Value + return NewBorDevnetWithHeimdall( + dataDir, + baseRpcHost, + baseRpcPort, + nil, + heimdallGrpcAddr, + checkpointOwner, + withMilestones, + logger) +} + +func NewBorDevnetWithLocalHeimdall( + dataDir string, + baseRpcHost string, + baseRpcPort int, + heimdallGrpcAddr string, + sprintSize uint64, + logger log.Logger, +) devnet.Devnet { + config := *params.BorDevnetChainConfig + if sprintSize > 0 { + config.Bor.Sprint = map[string]uint64{"0": sprintSize} + } + + checkpointOwner := accounts.NewAccount("checkpoint-owner") + + heimdall := polygon.NewHeimdall( + &config, + heimdallGrpcAddr, + &polygon.CheckpointConfig{ + CheckpointBufferTime: 60 * time.Second, + CheckpointAccount: checkpointOwner, + }, + logger) + + return NewBorDevnetWithHeimdall( + dataDir, + baseRpcHost, + baseRpcPort, + heimdall, + heimdallGrpcAddr, + checkpointOwner, + // milestones are not supported yet on the local heimdall + false, + logger) +} diff --git a/cmd/devnet/tests/devnet_dev.go b/cmd/devnet/tests/devnet_dev.go new file mode 100644 index 00000000000..f4aeed1d0f7 --- /dev/null +++ b/cmd/devnet/tests/devnet_dev.go @@ -0,0 +1,53 @@ +package tests + +import ( + "github.com/ledgerwatch/erigon-lib/chain/networkname" + "github.com/ledgerwatch/erigon/cmd/devnet/accounts" + "github.com/ledgerwatch/erigon/cmd/devnet/args" + "github.com/ledgerwatch/erigon/cmd/devnet/devnet" + account_services "github.com/ledgerwatch/erigon/cmd/devnet/services/accounts" + "github.com/ledgerwatch/erigon/core/types" + "github.com/ledgerwatch/log/v3" +) + +func NewDevDevnet( + dataDir string, + baseRpcHost string, + baseRpcPort int, + logger log.Logger, +) devnet.Devnet { + faucetSource := accounts.NewAccount("faucet-source") + + network := devnet.Network{ + DataDir: dataDir, + Chain: networkname.DevChainName, + Logger: logger, + BasePrivateApiAddr: "localhost:10090", + BaseRPCHost: baseRpcHost, + BaseRPCPort: baseRpcPort, + Alloc: types.GenesisAlloc{ + faucetSource.Address: {Balance: accounts.EtherAmount(200_000)}, + }, + Services: []devnet.Service{ + account_services.NewFaucet(networkname.DevChainName, faucetSource), + }, + MaxNumberOfEmptyBlockChecks: 30, + Nodes: []devnet.Node{ + &args.BlockProducer{ + NodeArgs: args.NodeArgs{ + ConsoleVerbosity: "0", + DirVerbosity: "5", + }, + AccountSlots: 200, + }, + &args.NonBlockProducer{ + NodeArgs: args.NodeArgs{ + ConsoleVerbosity: "0", + DirVerbosity: "5", + }, + }, + }, + } + + return devnet.Devnet{&network} +} diff --git a/cmd/devnet/tests/generic/devnet_test.go b/cmd/devnet/tests/generic/devnet_test.go new file mode 100644 index 00000000000..a3229652ae1 --- /dev/null +++ b/cmd/devnet/tests/generic/devnet_test.go @@ -0,0 +1,66 @@ +//go:build integration + +package generic + +import ( + "context" + "github.com/ledgerwatch/erigon/cmd/devnet/accounts" + "github.com/ledgerwatch/erigon/cmd/devnet/admin" + "github.com/ledgerwatch/erigon/cmd/devnet/contracts/steps" + "github.com/ledgerwatch/erigon/cmd/devnet/requests" + "github.com/ledgerwatch/erigon/cmd/devnet/services" + "github.com/ledgerwatch/erigon/cmd/devnet/tests" + "github.com/ledgerwatch/erigon/cmd/devnet/transactions" + "github.com/stretchr/testify/require" + "testing" + "time" +) + +func testDynamicTx(t *testing.T, ctx context.Context) { + t.Run("InitSubscriptions", func(t *testing.T) { + services.InitSubscriptions(ctx, []requests.SubMethod{requests.Methods.ETHNewHeads}) + }) + t.Run("PingErigonRpc", func(t *testing.T) { + require.Nil(t, admin.PingErigonRpc(ctx)) + }) + t.Run("CheckTxPoolContent", func(t *testing.T) { + transactions.CheckTxPoolContent(ctx, 0, 0, 0) + }) + t.Run("SendTxWithDynamicFee", func(t *testing.T) { + const recipientAddress = "0x71562b71999873DB5b286dF957af199Ec94617F7" + const sendValue uint64 = 10000 + _, err := transactions.SendTxWithDynamicFee(ctx, recipientAddress, accounts.DevAddress, sendValue) + require.Nil(t, err) + }) + t.Run("AwaitBlocks", func(t *testing.T) { + require.Nil(t, transactions.AwaitBlocks(ctx, 2*time.Second)) + }) +} + +func TestDynamicTxNode0(t *testing.T) { + runCtx, err := tests.ContextStart(t, "") + require.Nil(t, err) + testDynamicTx(t, runCtx.WithCurrentNetwork(0).WithCurrentNode(0)) +} + +func TestDynamicTxAnyNode(t *testing.T) { + runCtx, err := tests.ContextStart(t, "") + require.Nil(t, err) + testDynamicTx(t, runCtx.WithCurrentNetwork(0)) +} + +func TestCallContract(t *testing.T) { + t.Skip("FIXME: DeployAndCallLogSubscriber step fails: Log result is incorrect expected txIndex: 1, actual txIndex 2") + + runCtx, err := tests.ContextStart(t, "") + require.Nil(t, err) + ctx := runCtx.WithCurrentNetwork(0) + + t.Run("InitSubscriptions", func(t *testing.T) { + services.InitSubscriptions(ctx, []requests.SubMethod{requests.Methods.ETHNewHeads}) + }) + t.Run("DeployAndCallLogSubscriber", func(t *testing.T) { + _, err := contracts_steps.DeployAndCallLogSubscriber(ctx, accounts.DevAddress) + require.Nil(t, err) + }) +} diff --git a/cmd/devnet/transactions/block.go b/cmd/devnet/transactions/block.go index 7219658c36b..e759bdbaac8 100644 --- a/cmd/devnet/transactions/block.go +++ b/cmd/devnet/transactions/block.go @@ -16,8 +16,8 @@ import ( "github.com/ledgerwatch/erigon/rpc" ) -// MaxNumberOfBlockChecks is the max number of blocks to look for a transaction in -var MaxNumberOfEmptyBlockChecks = 25 +// max number of blocks to look for a transaction in +const defaultMaxNumberOfEmptyBlockChecks = 25 func AwaitTransactions(ctx context.Context, hashes ...libcommon.Hash) (map[libcommon.Hash]uint64, error) { devnet.Logger(ctx).Info("Awaiting transactions in confirmed blocks...") @@ -28,7 +28,13 @@ func AwaitTransactions(ctx context.Context, hashes ...libcommon.Hash) (map[libco hashmap[hash] = true } - m, err := searchBlockForHashes(ctx, hashmap) + maxNumberOfEmptyBlockChecks := defaultMaxNumberOfEmptyBlockChecks + network := devnet.CurrentNetwork(ctx) + if (network != nil) && (network.MaxNumberOfEmptyBlockChecks > 0) { + maxNumberOfEmptyBlockChecks = network.MaxNumberOfEmptyBlockChecks + } + + m, err := searchBlockForHashes(ctx, hashmap, maxNumberOfEmptyBlockChecks) if err != nil { return nil, fmt.Errorf("failed to search reserves for hashes: %v", err) } @@ -36,7 +42,11 @@ func AwaitTransactions(ctx context.Context, hashes ...libcommon.Hash) (map[libco return m, nil } -func searchBlockForHashes(ctx context.Context, hashmap map[libcommon.Hash]bool) (map[libcommon.Hash]uint64, error) { +func searchBlockForHashes( + ctx context.Context, + hashmap map[libcommon.Hash]bool, + maxNumberOfEmptyBlockChecks int, +) (map[libcommon.Hash]uint64, error) { logger := devnet.Logger(ctx) if len(hashmap) == 0 { @@ -72,7 +82,7 @@ func searchBlockForHashes(ctx context.Context, hashmap map[libcommon.Hash]bool) blockCount++ // increment the number of blocks seen to check against the max number of blocks to iterate over } - if blockCount == MaxNumberOfEmptyBlockChecks { + if blockCount == maxNumberOfEmptyBlockChecks { for h := range hashmap { logger.Error("Missing Tx", "txHash", h) } diff --git a/cmd/integration/commands/stages.go b/cmd/integration/commands/stages.go index 5ac02ddfb6f..804c2b46555 100644 --- a/cmd/integration/commands/stages.go +++ b/cmd/integration/commands/stages.go @@ -11,6 +11,8 @@ import ( "time" "github.com/c2h5oh/datasize" + lru "github.com/hashicorp/golang-lru/arc/v2" + "github.com/ledgerwatch/erigon/consensus/bor" "github.com/ledgerwatch/erigon/consensus/bor/heimdall" "github.com/ledgerwatch/erigon/consensus/bor/heimdallgrpc" "github.com/ledgerwatch/erigon/core/rawdb/blockio" @@ -26,6 +28,7 @@ import ( chain2 "github.com/ledgerwatch/erigon-lib/chain" "github.com/ledgerwatch/erigon-lib/commitment" common2 "github.com/ledgerwatch/erigon-lib/common" + libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/cmp" "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/common/dir" @@ -1553,7 +1556,18 @@ func newSync(ctx context.Context, db kv.RwDB, miningConfig *params.MiningConfig, notifications := &shards.Notifications{} blockRetire := freezeblocks.NewBlockRetire(1, dirs, blockReader, blockWriter, db, notifications.Events, logger) - stages := stages2.NewDefaultStages(context.Background(), db, p2p.Config{}, &cfg, sentryControlServer, notifications, nil, blockReader, blockRetire, agg, nil, nil, heimdallClient, logger) + var ( + snapDb kv.RwDB + recents *lru.ARCCache[libcommon.Hash, *bor.Snapshot] + signatures *lru.ARCCache[libcommon.Hash, libcommon.Address] + ) + if bor, ok := engine.(*bor.Bor); ok { + snapDb = bor.DB + recents = bor.Recents + signatures = bor.Signatures + } + stages := stages2.NewDefaultStages(context.Background(), db, snapDb, p2p.Config{}, &cfg, sentryControlServer, notifications, nil, blockReader, blockRetire, agg, nil, nil, + heimdallClient, recents, signatures, logger) sync := stagedsync.New(stages, stagedsync.DefaultUnwindOrder, stagedsync.DefaultPruneOrder, logger) miner := stagedsync.NewMiningState(&cfg.Miner) @@ -1566,7 +1580,7 @@ func newSync(ctx context.Context, db kv.RwDB, miningConfig *params.MiningConfig, miningSync := stagedsync.New( stagedsync.MiningStages(ctx, stagedsync.StageMiningCreateBlockCfg(db, miner, *chainConfig, engine, nil, nil, dirs.Tmp, blockReader), - stagedsync.StageBorHeimdallCfg(db, miner, *chainConfig, heimdallClient, blockReader, nil, nil), + stagedsync.StageBorHeimdallCfg(db, snapDb, miner, *chainConfig, heimdallClient, blockReader, nil, nil, recents, signatures), stagedsync.StageMiningExecCfg(db, miner, events, *chainConfig, engine, &vm.Config{}, dirs.Tmp, nil, 0, nil, nil, blockReader), stagedsync.StageHashStateCfg(db, dirs, historyV3), stagedsync.StageTrieCfg(db, false, true, false, dirs.Tmp, blockReader, nil, historyV3, agg), diff --git a/cmd/observer/observer/server.go b/cmd/observer/observer/server.go index 4c017f3379d..99c2cb4bbc2 100644 --- a/cmd/observer/observer/server.go +++ b/cmd/observer/observer/server.go @@ -183,5 +183,5 @@ func (server *Server) Listen(ctx context.Context) (*discover.UDPv4, error) { server.logger.Debug("Discovery UDP listener is up", "addr", realAddr) - return discover.ListenV4(ctx, conn, server.localNode, server.discConfig) + return discover.ListenV4(ctx, "any", conn, server.localNode, server.discConfig) } diff --git a/cmd/rpcdaemon/rpcservices/eth_backend.go b/cmd/rpcdaemon/rpcservices/eth_backend.go index 44f1d91e61d..aa4f8192ee0 100644 --- a/cmd/rpcdaemon/rpcservices/eth_backend.go +++ b/cmd/rpcdaemon/rpcservices/eth_backend.go @@ -271,6 +271,9 @@ func (back *RemoteBackend) EventLookup(ctx context.Context, tx kv.Getter, txnHas func (back *RemoteBackend) EventsByBlock(ctx context.Context, tx kv.Tx, hash common.Hash, blockNum uint64) ([]rlp.RawValue, error) { return back.blockReader.EventsByBlock(ctx, tx, hash, blockNum) } +func (back *RemoteBackend) Span(ctx context.Context, tx kv.Getter, spanId uint64) ([]byte, error) { + return back.blockReader.Span(ctx, tx, spanId) +} func (back *RemoteBackend) NodeInfo(ctx context.Context, limit uint32) ([]p2p.NodeInfo, error) { nodes, err := back.remoteEthBackend.NodeInfo(ctx, &remote.NodesInfoRequest{Limit: limit}) diff --git a/cmd/rpctest/main.go b/cmd/rpctest/main.go index 5ce6a6c71ab..78eb87af272 100644 --- a/cmd/rpctest/main.go +++ b/cmd/rpctest/main.go @@ -290,6 +290,19 @@ func main() { } with(benchTraceFilterCmd, withGethUrl, withErigonUrl, withNeedCompare, withBlockNum, withRecord, withErrorFile) + var benchDebugTraceBlockByNumberCmd = &cobra.Command{ + Use: "benchDebugTraceBlockByNumber", + Short: "", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + err := rpctest.BenchDebugTraceBlockByNumber(erigonURL, gethURL, needCompare, blockFrom, blockTo, recordFile, errorFile) + if err != nil { + logger.Error(err.Error()) + } + }, + } + with(benchDebugTraceBlockByNumberCmd, withErigonUrl, withGethUrl, withNeedCompare, withBlockNum, withRecord, withErrorFile, withLatest) + var benchTxReceiptCmd = &cobra.Command{ Use: "benchTxReceipt", Short: "", @@ -384,6 +397,7 @@ func main() { benchTraceCallManyCmd, benchTraceBlockCmd, benchTraceFilterCmd, + benchDebugTraceBlockByNumberCmd, benchTxReceiptCmd, compareAccountRange, benchTraceReplayTransactionCmd, diff --git a/cmd/rpctest/rpctest/bench_debugTraceBlockByNumber.go b/cmd/rpctest/rpctest/bench_debugTraceBlockByNumber.go new file mode 100644 index 00000000000..39b92bd1d79 --- /dev/null +++ b/cmd/rpctest/rpctest/bench_debugTraceBlockByNumber.go @@ -0,0 +1,58 @@ +package rpctest + +import ( + "bufio" + "fmt" + "net/http" + "os" + "time" +) + +func BenchDebugTraceBlockByNumber(erigonUrl, gethUrl string, needCompare bool, blockFrom uint64, blockTo uint64, recordFileName string, errorFileName string) error { + setRoutes(erigonUrl, gethUrl) + var client = &http.Client{ + Timeout: time.Second * 600, + } + + var rec *bufio.Writer + if recordFileName != "" { + f, err := os.Create(recordFileName) + if err != nil { + return fmt.Errorf("Cannot create file %s for recording: %v\n", recordFileName, err) + } + defer f.Close() + rec = bufio.NewWriter(f) + defer rec.Flush() + } + var errs *bufio.Writer + if errorFileName != "" { + ferr, err := os.Create(errorFileName) + if err != nil { + return fmt.Errorf("Cannot create file %s for error output: %v\n", errorFileName, err) + } + defer ferr.Close() + errs = bufio.NewWriter(ferr) + defer errs.Flush() + } + + var resultsCh chan CallResult = nil + if !needCompare { + resultsCh = make(chan CallResult, 1000) + defer close(resultsCh) + go vegetaWrite(true, []string{"debug_traceBlockByNumber"}, resultsCh) + } + + reqGen := &RequestGenerator{ + client: client, + } + + for bn := blockFrom; bn < blockTo; bn++ { + reqGen.reqID++ + request := reqGen.debugTraceBlockByNumber(bn) + errCtx := fmt.Sprintf("block %d", bn) + if err := requestAndCompare(request, "debug_traceBlockByNumber", errCtx, reqGen, needCompare, rec, errs, resultsCh); err != nil { + return err + } + } + return nil +} diff --git a/cmd/rpctest/rpctest/request_generator.go b/cmd/rpctest/rpctest/request_generator.go index 67c699529f9..4016bebc5d6 100644 --- a/cmd/rpctest/rpctest/request_generator.go +++ b/cmd/rpctest/rpctest/request_generator.go @@ -3,11 +3,12 @@ package rpctest import ( "encoding/base64" "fmt" - "github.com/ledgerwatch/erigon-lib/common/hexutil" "net/http" "strings" "time" + "github.com/ledgerwatch/erigon-lib/common/hexutil" + "github.com/valyala/fastjson" libcommon "github.com/ledgerwatch/erigon-lib/common" @@ -58,6 +59,11 @@ func (g *RequestGenerator) traceBlockByHash(hash string) string { return fmt.Sprintf(template, hash, g.reqID) } +func (g *RequestGenerator) debugTraceBlockByNumber(blockNum uint64) string { + const template = `{"jsonrpc":"2.0","method":"debug_traceBlockByNumber","params":[%d],"id":%d}` + return fmt.Sprintf(template, blockNum, g.reqID) +} + func (g *RequestGenerator) traceTransaction(hash string) string { const template = `{"jsonrpc":"2.0","method":"debug_traceTransaction","params":["%s"],"id":%d}` return fmt.Sprintf(template, hash, g.reqID) diff --git a/cmd/state/exec3/state.go b/cmd/state/exec3/state.go index c351aa7143d..3f4a9383b6d 100644 --- a/cmd/state/exec3/state.go +++ b/cmd/state/exec3/state.go @@ -287,6 +287,7 @@ func (cr ChainReader) HasBlock(hash libcommon.Hash, number uint64) bool { func (cr ChainReader) BorEventsByBlock(hash libcommon.Hash, number uint64) []rlp.RawValue { panic("") } +func (cr ChainReader) BorSpan(spanId uint64) []byte { panic("") } func NewWorkersPool(lock sync.Locker, ctx context.Context, background bool, chainDb kv.RoDB, rs *state.StateV3, in *exec22.QueueWithRetry, blockReader services.FullBlockReader, chainConfig *chain.Config, genesis *types.Genesis, engine consensus.Engine, workerCount int) (reconWorkers []*Worker, applyWorker *Worker, rws *exec22.ResultsQueue, clear func(), wait func()) { reconWorkers = make([]*Worker, workerCount) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 47ce913cac9..46deded7de2 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -140,12 +140,12 @@ var ( } InternalConsensusFlag = cli.BoolFlag{ Name: "internalcl", - Usage: "enables internal consensus", + Usage: "Enables internal consensus", } // Transaction pool settings TxPoolDisableFlag = cli.BoolFlag{ Name: "txpool.disable", - Usage: "experimental external pool and block producer, see ./cmd/txpool/readme.md for more info. Disabling internal txpool and block producer.", + Usage: "Experimental external pool and block producer, see ./cmd/txpool/readme.md for more info. Disabling internal txpool and block producer.", } TxPoolLocalsFlag = cli.StringFlag{ Name: "txpool.locals", @@ -413,7 +413,7 @@ var ( TxpoolApiAddrFlag = cli.StringFlag{ Name: "txpool.api.addr", - Usage: "txpool api network address, for example: 127.0.0.1:9090 (default: use value of --private.api.addr)", + Usage: "TxPool api network address, for example: 127.0.0.1:9090 (default: use value of --private.api.addr)", } TraceMaxtracesFlag = cli.UintFlag{ @@ -530,7 +530,7 @@ var ( } SentryAddrFlag = cli.StringFlag{ Name: "sentry.api.addr", - Usage: "comma separated sentry addresses ':,:'", + Usage: "Comma separated sentry addresses ':,:'", } SentryLogPeerInfoFlag = cli.BoolFlag{ Name: "sentry.log-peer-info", @@ -566,14 +566,14 @@ var ( NATFlag = cli.StringFlag{ Name: "nat", Usage: `NAT port mapping mechanism (any|none|upnp|pmp|stun|extip:) - "" or "none" default - do not nat - "extip:77.12.33.4" will assume the local machine is reachable on the given IP - "any" uses the first auto-detected mechanism - "upnp" uses the Universal Plug and Play protocol - "pmp" uses NAT-PMP with an auto-detected gateway address - "pmp:192.168.0.1" uses NAT-PMP with the given gateway address - "stun" uses STUN to detect an external IP using a default server - "stun:" uses STUN to detect an external IP using the given server (host:port) + "" or "none" Default - do not nat + "extip:77.12.33.4" Will assume the local machine is reachable on the given IP + "any" Uses the first auto-detected mechanism + "upnp" Uses the Universal Plug and Play protocol + "pmp" Uses NAT-PMP with an auto-detected gateway address + "pmp:192.168.0.1" Uses NAT-PMP with the given gateway address + "stun" Uses STUN to detect an external IP using a default server + "stun:" Uses STUN to detect an external IP using the given server (host:port) `, Value: "", } @@ -640,27 +640,27 @@ var ( } HistoryV3Flag = cli.BoolFlag{ Name: "experimental.history.v3", - Usage: "(also known as Erigon3) Not recommended yet: Can't change this flag after node creation. New DB and Snapshots format of history allows: parallel blocks execution, get state as of given transaction without executing whole block.", + Usage: "(Also known as Erigon3) Not recommended yet: Can't change this flag after node creation. New DB and Snapshots format of history allows: parallel blocks execution, get state as of given transaction without executing whole block.", } CliqueSnapshotCheckpointIntervalFlag = cli.UintFlag{ Name: "clique.checkpoint", - Usage: "number of blocks after which to save the vote snapshot to the database", + Usage: "Number of blocks after which to save the vote snapshot to the database", Value: 10, } CliqueSnapshotInmemorySnapshotsFlag = cli.IntFlag{ Name: "clique.snapshots", - Usage: "number of recent vote snapshots to keep in memory", + Usage: "Number of recent vote snapshots to keep in memory", Value: 1024, } CliqueSnapshotInmemorySignaturesFlag = cli.IntFlag{ Name: "clique.signatures", - Usage: "number of recent block signatures to keep in memory", + Usage: "Number of recent block signatures to keep in memory", Value: 16384, } CliqueDataDirFlag = flags.DirectoryFlag{ Name: "clique.datadir", - Usage: "a path to clique db folder", + Usage: "Path to clique db folder", Value: "", } @@ -680,17 +680,17 @@ var ( TorrentDownloadRateFlag = cli.StringFlag{ Name: "torrent.download.rate", Value: "16mb", - Usage: "bytes per second, example: 32mb", + Usage: "Bytes per second, example: 32mb", } TorrentUploadRateFlag = cli.StringFlag{ Name: "torrent.upload.rate", Value: "4mb", - Usage: "bytes per second, example: 32mb", + Usage: "Bytes per second, example: 32mb", } TorrentDownloadSlotsFlag = cli.IntFlag{ Name: "torrent.download.slots", Value: 3, - Usage: "amount of files to download in parallel. If network has enough seeders 1-3 slot enough, if network has lack of seeders increase to 5-7 (too big value will slow down everything).", + Usage: "Amount of files to download in parallel. If network has enough seeders 1-3 slot enough, if network has lack of seeders increase to 5-7 (too big value will slow down everything).", } TorrentStaticPeersFlag = cli.StringFlag{ Name: "torrent.staticpeers", @@ -699,37 +699,37 @@ var ( } NoDownloaderFlag = cli.BoolFlag{ Name: "no-downloader", - Usage: "to disable downloader component", + Usage: "Disables downloader component", } DownloaderVerifyFlag = cli.BoolFlag{ Name: "downloader.verify", - Usage: "verify snapshots on startup. it will not report founded problems but just re-download broken pieces", + Usage: "Verify snapshots on startup. It will not report problems found, but re-download broken pieces.", } DisableIPV6 = cli.BoolFlag{ Name: "downloader.disable.ipv6", - Usage: "Turns off ipv6 for the downlaoder", + Usage: "Turns off ipv6 for the downloader", Value: false, } DisableIPV4 = cli.BoolFlag{ Name: "downloader.disable.ipv4", - Usage: "Turn off ipv4 for the downloader", + Usage: "Turns off ipv4 for the downloader", Value: false, } TorrentPortFlag = cli.IntFlag{ Name: "torrent.port", Value: 42069, - Usage: "port to listen and serve BitTorrent protocol", + Usage: "Port to listen and serve BitTorrent protocol", } TorrentMaxPeersFlag = cli.IntFlag{ Name: "torrent.maxpeers", Value: 100, - Usage: "unused parameter (reserved for future use)", + Usage: "Unused parameter (reserved for future use)", } TorrentConnsPerFileFlag = cli.IntFlag{ Name: "torrent.conns.perfile", Value: 10, - Usage: "connections per file", + Usage: "Number of connections per file", } DbPageSizeFlag = cli.StringFlag{ Name: "db.pagesize", @@ -738,7 +738,7 @@ var ( } DbSizeLimitFlag = cli.StringFlag{ Name: "db.size.limit", - Usage: "runtime limit of chandata db size. you can change value of this flag at any time", + Usage: "Runtime limit of chaindata db size. You can change value of this flag at any time.", Value: (3 * datasize.TB).String(), } ForcePartialCommitFlag = cli.BoolFlag{ @@ -760,7 +760,7 @@ var ( WebSeedsFlag = cli.StringFlag{ Name: "webseed", - Usage: "comma-separated URL's, holding metadata about network-support infrastructure (like S3 buckets with snapshots, bootnodes, etc...)", + Usage: "Comma-separated URL's, holding metadata about network-support infrastructure (like S3 buckets with snapshots, bootnodes, etc...)", Value: "", } diff --git a/consensus/bor/bor.go b/consensus/bor/bor.go index b9434fa9e2b..40dc9654aec 100644 --- a/consensus/bor/bor.go +++ b/consensus/bor/bor.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/hex" + "encoding/json" "errors" "fmt" "io" @@ -15,9 +16,7 @@ import ( "sync/atomic" "time" - "github.com/google/btree" lru "github.com/hashicorp/golang-lru/arc/v2" - "github.com/ledgerwatch/erigon/eth/ethconfig/estimate" "github.com/ledgerwatch/log/v3" "github.com/xsleonard/go-merkle" "golang.org/x/crypto/sha3" @@ -44,6 +43,8 @@ import ( "github.com/ledgerwatch/erigon/core/types/accounts" "github.com/ledgerwatch/erigon/crypto" "github.com/ledgerwatch/erigon/crypto/cryptopool" + "github.com/ledgerwatch/erigon/eth/ethconfig/estimate" + "github.com/ledgerwatch/erigon/params" "github.com/ledgerwatch/erigon/rlp" "github.com/ledgerwatch/erigon/rpc" "github.com/ledgerwatch/erigon/turbo/services" @@ -116,7 +117,7 @@ var ( // errInvalidSpanValidators is returned if a block contains an // invalid list of validators (i.e. non divisible by 40 bytes). - errInvalidSpanValidators = errors.New("invalid validator list on sprint end block") + ErrInvalidSpanValidators = errors.New("invalid validator list on sprint end block") // errInvalidMixDigest is returned if a block's mix digest is non-zero. errInvalidMixDigest = errors.New("non-zero mix digest") @@ -146,7 +147,7 @@ var ( type SignerFn func(signer libcommon.Address, mimeType string, message []byte) ([]byte, error) // ecrecover extracts the Ethereum account address from a signed header. -func ecrecover(header *types.Header, sigcache *lru.ARCCache[libcommon.Hash, libcommon.Address], c *chain.BorConfig) (libcommon.Address, error) { +func Ecrecover(header *types.Header, sigcache *lru.ARCCache[libcommon.Hash, libcommon.Address], c *chain.BorConfig) (libcommon.Address, error) { // If the signature's already cached, return that hash := header.Hash() if address, known := sigcache.Get(hash); known { @@ -250,8 +251,8 @@ type Bor struct { DB kv.RwDB // Database to store and retrieve snapshot checkpoints blockReader services.FullBlockReader - recents *lru.ARCCache[libcommon.Hash, *Snapshot] // Snapshots for recent block to speed up reorgs - signatures *lru.ARCCache[libcommon.Hash, libcommon.Address] // Signatures of recent blocks to speed up mining + Recents *lru.ARCCache[libcommon.Hash, *Snapshot] // Snapshots for recent block to speed up reorgs + Signatures *lru.ARCCache[libcommon.Hash, libcommon.Address] // Signatures of recent blocks to speed up mining authorizedSigner atomic.Pointer[signer] // Ethereum address and sign function of the signing key @@ -263,8 +264,7 @@ type Bor struct { // scope event.SubscriptionScope // The fields below are for testing only - fakeDiff bool // Skip difficulty verifications - spanCache *btree.BTree + fakeDiff bool // Skip difficulty verifications closeOnce sync.Once logger log.Logger @@ -395,12 +395,11 @@ func New( config: borConfig, DB: db, blockReader: blockReader, - recents: recents, - signatures: signatures, + Recents: recents, + Signatures: signatures, spanner: spanner, GenesisContractsClient: genesisContracts, HeimdallClient: heimdallClient, - spanCache: btree.New(32), execCtx: context.Background(), logger: logger, closeCh: make(chan struct{}), @@ -464,9 +463,8 @@ func NewRo(chainConfig *chain.Config, db kv.RoDB, blockReader services.FullBlock DB: rwWrapper{db}, blockReader: blockReader, logger: logger, - recents: recents, - signatures: signatures, - spanCache: btree.New(32), + Recents: recents, + Signatures: signatures, execCtx: context.Background(), closeCh: make(chan struct{}), } @@ -490,7 +488,7 @@ func (c *Bor) HeaderProgress(p HeaderProgress) { // This is thread-safe (only access the header and config (which is never updated), // as well as signatures, which are lru.ARCCache, which is thread-safe) func (c *Bor) Author(header *types.Header) (libcommon.Address, error) { - return ecrecover(header, c.signatures, c.config) + return Ecrecover(header, c.Signatures, c.config) } // VerifyHeader checks whether a header conforms to the consensus rules. @@ -550,7 +548,7 @@ func (c *Bor) verifyHeader(chain consensus.ChainHeaderReader, header *types.Head } if isSprintEnd && signersBytes%validatorHeaderBytesLength != 0 { - return errInvalidSpanValidators + return ErrInvalidSpanValidators } // Ensure that the mix digest is zero as we don't have fork protection currently @@ -571,9 +569,8 @@ func (c *Bor) verifyHeader(chain consensus.ChainHeaderReader, header *types.Head } // Verify that the gas limit is <= 2^63-1 - gasCap := uint64(0x7fffffffffffffff) - if header.GasLimit > gasCap { - return fmt.Errorf("invalid gasLimit: have %v, max %v", header.GasLimit, gasCap) + if header.GasLimit > params.MaxGasLimit { + return fmt.Errorf("invalid gasLimit: have %v, max %v", header.GasLimit, params.MaxGasLimit) } if header.WithdrawalsHash != nil { @@ -644,67 +641,7 @@ func (c *Bor) verifyCascadingFields(chain consensus.ChainHeaderReader, header *t if parent.Time+c.config.CalculatePeriod(number) > header.Time { return ErrInvalidTimestamp } - - sprintLength := c.config.CalculateSprint(number) - - // Verify the validator list match the local contract - // - // Note: Here we fetch the data from span instead of contract - // as done in bor client. The contract (validator set) returns - // a fixed span for 0th span i.e. 0 - 255 blocks. Hence, the - // contract data and span data won't match for that. Skip validating - // for 0th span. TODO: Remove `number > zerothSpanEnd` check - // once we start fetching validator data from contract. - if number > zerothSpanEnd && isSprintStart(number+1, sprintLength) { - producerSet, err := c.spanner.GetCurrentProducers(number+1, c.authorizedSigner.Load().signer, c.getSpanForBlock) - - if err != nil { - return err - } - - sort.Sort(valset.ValidatorsByAddress(producerSet)) - - headerVals, err := valset.ParseValidators(header.Extra[extraVanity : len(header.Extra)-extraSeal]) - - if err != nil { - return err - } - - if len(producerSet) != len(headerVals) { - return errInvalidSpanValidators - } - - for i, val := range producerSet { - if !bytes.Equal(val.HeaderBytes(), headerVals[i].HeaderBytes()) { - return errInvalidSpanValidators - } - } - } - snap, err := c.snapshot(chain, number-1, header.ParentHash, parents) - if err != nil { - return err - } - - // verify the validator list in the last sprint block - if isSprintStart(number, sprintLength) { - // Retrieve the snapshot needed to verify this header and cache it - parentValidatorBytes := parent.Extra[extraVanity : len(parent.Extra)-extraSeal] - validatorsBytes := make([]byte, len(snap.ValidatorSet.Validators)*validatorHeaderBytesLength) - - currentValidators := snap.ValidatorSet.Copy().Validators - // sort validator by address - sort.Sort(valset.ValidatorsByAddress(currentValidators)) - for i, validator := range currentValidators { - copy(validatorsBytes[i*validatorHeaderBytesLength:], validator.HeaderBytes()) - } - // len(header.Extra) >= extraVanity+extraSeal has already been validated in ValidateHeaderExtraField, so this won't result in a panic - if !bytes.Equal(parentValidatorBytes, validatorsBytes) { - return &MismatchingValidatorsError{number - 1, validatorsBytes, parentValidatorBytes} - } - } - - // All basic checks passed, verify the seal and return - return c.verifySeal(chain, header, parents, snap) + return nil } func (c *Bor) initFrozenSnapshot(chain consensus.ChainHeaderReader, number uint64, logEvery *time.Ticker) (snap *Snapshot, err error) { @@ -723,16 +660,16 @@ func (c *Bor) initFrozenSnapshot(chain consensus.ChainHeaderReader, number uint6 // get validators and current span var validators []*valset.Validator - validators, err = c.spanner.GetCurrentValidators(1, c.authorizedSigner.Load().signer, c.getSpanForBlock) + validators, err = c.spanner.GetCurrentValidators(0, c.authorizedSigner.Load().signer, chain) if err != nil { return nil, err } // new snap shot - snap = newSnapshot(c.config, c.signatures, 0, hash, validators, c.logger) + snap = NewSnapshot(c.config, c.Signatures, 0, hash, validators, c.logger) - if err = snap.store(c.DB); err != nil { + if err = snap.Store(c.DB); err != nil { return nil, err } @@ -753,13 +690,13 @@ func (c *Bor) initFrozenSnapshot(chain consensus.ChainHeaderReader, number uint6 // `batchSize` < `inmemorySignatures`: means all current batch will fit in cache - and `snap.apply` will find it there. snap := snap g.Go(func() error { - _, _ = ecrecover(header, snap.sigcache, snap.config) + _, _ = Ecrecover(header, snap.sigcache, snap.config) return nil }) } initialHeaders = append(initialHeaders, header) if len(initialHeaders) == cap(initialHeaders) { - snap, err = snap.apply(initialHeaders, c.logger) + snap, err = snap.Apply(nil, initialHeaders, c.logger) if err != nil { return nil, err @@ -774,7 +711,7 @@ func (c *Bor) initFrozenSnapshot(chain consensus.ChainHeaderReader, number uint6 } } - if snap, err = snap.apply(initialHeaders, c.logger); err != nil { + if snap, err = snap.Apply(nil, initialHeaders, c.logger); err != nil { return nil, err } } @@ -794,14 +731,14 @@ func (c *Bor) snapshot(chain consensus.ChainHeaderReader, number uint64, hash li //nolint:govet for snap == nil { // If an in-memory snapshot was found, use that - if s, ok := c.recents.Get(hash); ok { + if s, ok := c.Recents.Get(hash); ok { snap = s break } // If an on-disk snapshot can be found, use that if number%snapshotPersistInterval == 0 { - if s, err := loadSnapshot(c.config, c.signatures, c.DB, hash); err == nil { + if s, err := LoadSnapshot(c.config, c.Signatures, c.DB, hash); err == nil { c.logger.Trace("Loaded snapshot from disk", "number", number, "hash", hash) snap = s @@ -852,7 +789,6 @@ func (c *Bor) snapshot(chain consensus.ChainHeaderReader, number uint64, hash li if snap == nil && chain != nil && number <= chain.FrozenBlocks() { var err error - c.frozenSnapshotsInit.Do(func() { snap, err = c.initFrozenSnapshot(chain, number, logEvery) }) @@ -873,15 +809,15 @@ func (c *Bor) snapshot(chain consensus.ChainHeaderReader, number uint64, hash li } var err error - if snap, err = snap.apply(headers, c.logger); err != nil { + if snap, err = snap.Apply(nil, headers, c.logger); err != nil { return nil, err } - c.recents.Add(snap.Hash, snap) + c.Recents.Add(snap.Hash, snap) // If we've generated a new persistent snapshot, save to disk if snap.Number%snapshotPersistInterval == 0 && len(headers) > 0 { - if err = snap.store(c.DB); err != nil { + if err = snap.Store(c.DB); err != nil { return nil, err } @@ -922,7 +858,7 @@ func (c *Bor) verifySeal(chain consensus.ChainHeaderReader, header *types.Header return errUnknownBlock } // Resolve the authorization key and check against signers - signer, err := ecrecover(header, c.signatures, c.config) + signer, err := Ecrecover(header, c.Signatures, c.config) if err != nil { return err } @@ -993,7 +929,11 @@ func (c *Bor) Prepare(chain consensus.ChainHeaderReader, header *types.Header, s // where it fetches producers internally. As we fetch data from span // in Erigon, use directly the `GetCurrentProducers` function. if isSprintStart(number+1, c.config.CalculateSprint(number)) { - newValidators, err := c.spanner.GetCurrentProducers(number+1, c.authorizedSigner.Load().signer, c.getSpanForBlock) + var spanID uint64 + if number+1 > zerothSpanEnd { + spanID = 1 + (number+1-zerothSpanEnd-1)/spanLength + } + newValidators, err := c.spanner.GetCurrentProducers(spanID, c.authorizedSigner.Load().signer, chain) if err != nil { return errUnknownValidators } @@ -1054,13 +994,13 @@ func (c *Bor) Finalize(config *chain.Config, header *types.Header, state *state. if isSprintStart(headerNumber, c.config.CalculateSprint(headerNumber)) { cx := statefull.ChainContext{Chain: chain, Bor: c} - // check and commit span - if err := c.checkAndCommitSpan(state, header, cx, syscall); err != nil { - c.logger.Error("Error while committing span", "err", err) - return nil, types.Receipts{}, err - } if c.blockReader != nil { + // check and commit span + if err := c.checkAndCommitSpan(state, header, cx, syscall); err != nil { + c.logger.Error("Error while committing span", "err", err) + return nil, types.Receipts{}, err + } // commit states if err := c.CommitStates(state, header, cx, syscall); err != nil { c.logger.Error("Error while committing states", "err", err) @@ -1119,16 +1059,14 @@ func (c *Bor) FinalizeAndAssemble(chainConfig *chain.Config, header *types.Heade if isSprintStart(headerNumber, c.config.CalculateSprint(headerNumber)) { cx := statefull.ChainContext{Chain: chain, Bor: c} - // check and commit span - err := c.checkAndCommitSpan(state, header, cx, syscall) - if err != nil { - c.logger.Error("Error while committing span", "err", err) - return nil, nil, types.Receipts{}, err - } - - if c.HeimdallClient != nil { + if c.blockReader != nil { + // check and commit span + if err := c.checkAndCommitSpan(state, header, cx, syscall); err != nil { + c.logger.Error("Error while committing span", "err", err) + return nil, nil, types.Receipts{}, err + } // commit states - if err = c.CommitStates(state, header, cx, syscall); err != nil { + if err := c.CommitStates(state, header, cx, syscall); err != nil { c.logger.Error("Error while committing states", "err", err) return nil, nil, types.Receipts{}, err } @@ -1421,46 +1359,6 @@ func (c *Bor) needToCommitSpan(currentSpan *span.Span, headerNumber uint64) bool return false } -func (c *Bor) getSpanForBlock(blockNum uint64) (*span.HeimdallSpan, error) { - c.logger.Debug("Getting span", "for block", blockNum) - var borSpan *span.HeimdallSpan - c.spanCache.AscendGreaterOrEqual(&span.HeimdallSpan{Span: span.Span{EndBlock: blockNum}}, func(item btree.Item) bool { - borSpan = item.(*span.HeimdallSpan) - return false - }) - - if borSpan != nil && borSpan.StartBlock <= blockNum && borSpan.EndBlock >= blockNum { - return borSpan, nil - } - - // Span with given block block number is not loaded - // As span has fixed set of blocks (except 0th span), we can - // formulate it and get the exact ID we'd need to fetch. - var spanID uint64 - if blockNum > zerothSpanEnd { - spanID = 1 + (blockNum-zerothSpanEnd-1)/spanLength - } - - if c.HeimdallClient == nil { - return nil, fmt.Errorf("span with given block number is not loaded: %d", spanID) - } - - c.logger.Debug("Span with given block number is not loaded", "fetching span", spanID) - - response, err := c.HeimdallClient.Span(c.execCtx, spanID) - if err != nil { - return nil, err - } - borSpan = response - c.spanCache.ReplaceOrInsert(borSpan) - - for c.spanCache.Len() > 128 { - c.spanCache.DeleteMin() - } - - return borSpan, nil -} - func (c *Bor) fetchAndCommitSpan( newSpanID uint64, state *state.IntraBlockState, @@ -1479,12 +1377,10 @@ func (c *Bor) fetchAndCommitSpan( heimdallSpan = *s } else { - response, err := c.HeimdallClient.Span(c.execCtx, newSpanID) - if err != nil { + spanJson := chain.Chain.BorSpan(newSpanID) + if err := json.Unmarshal(spanJson, &heimdallSpan); err != nil { return err } - - heimdallSpan = *response } // check if chain id matches with heimdall span @@ -1594,10 +1490,6 @@ func (c *Bor) SetHeimdallClient(h heimdall.IHeimdallClient) { c.HeimdallClient = h } -func (c *Bor) GetCurrentValidators(blockNumber uint64, signer libcommon.Address, getSpanForBlock func(blockNum uint64) (*span.HeimdallSpan, error)) ([]*valset.Validator, error) { - return c.spanner.GetCurrentValidators(blockNumber, signer, getSpanForBlock) -} - // // Private methods // diff --git a/consensus/bor/bor_test.go b/consensus/bor/bor_test.go index 937868cab29..352686e5034 100644 --- a/consensus/bor/bor_test.go +++ b/consensus/bor/bor_test.go @@ -2,11 +2,13 @@ package bor_test import ( "context" + "encoding/json" "fmt" "math/big" "testing" "github.com/ledgerwatch/erigon-lib/chain" + "github.com/ledgerwatch/erigon-lib/common" libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/gointerfaces/sentry" "github.com/ledgerwatch/erigon-lib/kv/memdb" @@ -173,9 +175,15 @@ func (r headerReader) GetTd(libcommon.Hash, uint64) *big.Int { return nil } +func (r headerReader) BorSpan(spanId uint64) []byte { + b, _ := json.Marshal(&r.validator.heimdall.currentSpan) + return b +} + type spanner struct { *span.ChainSpanner - currentSpan span.Span + validatorAddress common.Address + currentSpan span.Span } func (c spanner) GetCurrentSpan(_ consensus.SystemCall) (*span.Span, error) { @@ -187,6 +195,16 @@ func (c *spanner) CommitSpan(heimdallSpan span.HeimdallSpan, syscall consensus.S return nil } +func (c *spanner) GetCurrentValidators(spanId uint64, signer libcommon.Address, chain consensus.ChainHeaderReader) ([]*valset.Validator, error) { + return []*valset.Validator{ + { + ID: 1, + Address: c.validatorAddress, + VotingPower: 1000, + ProposerPriority: 1, + }}, nil +} + type validator struct { *mock.MockSentry heimdall *test_heimdall @@ -248,19 +266,18 @@ func (v validator) verifyBlocks(blocks []*types.Block) error { func newValidator(t *testing.T, heimdall *test_heimdall, blocks map[uint64]*types.Block) validator { logger := log.Root() + validatorKey, _ := crypto.GenerateKey() + validatorAddress := crypto.PubkeyToAddress(validatorKey.PublicKey) bor := bor.New( heimdall.chainConfig, memdb.New(""), nil, /* blockReader */ - &spanner{span.NewChainSpanner(contract.ValidatorSet(), heimdall.chainConfig, false, logger), span.Span{}}, + &spanner{span.NewChainSpanner(contract.ValidatorSet(), heimdall.chainConfig, false, logger), validatorAddress, span.Span{}}, heimdall, test_genesisContract{}, logger, ) - validatorKey, _ := crypto.GenerateKey() - validatorAddress := crypto.PubkeyToAddress(validatorKey.PublicKey) - /*fmt.Printf("Private: 0x%s\nPublic: 0x%s\nAddress: %s\n", hex.EncodeToString(crypto.FromECDSA(validatorKey)), hex.EncodeToString(crypto.MarshalPubkey(&validatorKey.PublicKey)), diff --git a/consensus/bor/heimdall/span/spanner.go b/consensus/bor/heimdall/span/spanner.go index b7f50ff796b..968aeff65bf 100644 --- a/consensus/bor/heimdall/span/spanner.go +++ b/consensus/bor/heimdall/span/spanner.go @@ -2,6 +2,7 @@ package span import ( "encoding/hex" + "encoding/json" "math/big" "github.com/ledgerwatch/erigon-lib/chain" @@ -67,28 +68,30 @@ func (c *ChainSpanner) GetCurrentSpan(syscall consensus.SystemCall) (*Span, erro return &span, nil } -func (c *ChainSpanner) GetCurrentValidators(blockNumber uint64, signer libcommon.Address, getSpanForBlock func(blockNum uint64) (*HeimdallSpan, error)) ([]*valset.Validator, error) { +func (c *ChainSpanner) GetCurrentValidators(spanId uint64, signer libcommon.Address, chain consensus.ChainHeaderReader) ([]*valset.Validator, error) { // Use hardcoded bor devnet valset if chain-name = bor-devnet if NetworkNameVals[c.chainConfig.ChainName] != nil && c.withoutHeimdall { return NetworkNameVals[c.chainConfig.ChainName], nil } - span, err := getSpanForBlock(blockNumber) - if err != nil { + spanBytes := chain.BorSpan(spanId) + var span HeimdallSpan + if err := json.Unmarshal(spanBytes, &span); err != nil { return nil, err } return span.ValidatorSet.Validators, nil } -func (c *ChainSpanner) GetCurrentProducers(blockNumber uint64, signer libcommon.Address, getSpanForBlock func(blockNum uint64) (*HeimdallSpan, error)) ([]*valset.Validator, error) { +func (c *ChainSpanner) GetCurrentProducers(spanId uint64, signer libcommon.Address, chain consensus.ChainHeaderReader) ([]*valset.Validator, error) { // Use hardcoded bor devnet valset if chain-name = bor-devnet if NetworkNameVals[c.chainConfig.ChainName] != nil && c.withoutHeimdall { return NetworkNameVals[c.chainConfig.ChainName], nil } - span, err := getSpanForBlock(blockNumber) - if err != nil { + spanBytes := chain.BorSpan(spanId) + var span HeimdallSpan + if err := json.Unmarshal(spanBytes, &span); err != nil { return nil, err } diff --git a/consensus/bor/snapshot.go b/consensus/bor/snapshot.go index 8a60b4bd683..5edaf596efc 100644 --- a/consensus/bor/snapshot.go +++ b/consensus/bor/snapshot.go @@ -37,7 +37,7 @@ const BorSeparate = "BorSeparate" // newSnapshot creates a new snapshot with the specified startup parameters. This // method does not initialize the set of recent signers, so only ever use if for // the genesis block. -func newSnapshot( +func NewSnapshot( config *chain.BorConfig, sigcache *lru.ARCCache[common.Hash, common.Address], number uint64, @@ -57,7 +57,7 @@ func newSnapshot( } // loadSnapshot loads an existing snapshot from the database. -func loadSnapshot(config *chain.BorConfig, sigcache *lru.ARCCache[common.Hash, common.Address], db kv.RwDB, hash common.Hash) (*Snapshot, error) { +func LoadSnapshot(config *chain.BorConfig, sigcache *lru.ARCCache[common.Hash, common.Address], db kv.RwDB, hash common.Hash) (*Snapshot, error) { tx, err := db.BeginRo(context.Background()) if err != nil { return nil, err @@ -90,7 +90,7 @@ func loadSnapshot(config *chain.BorConfig, sigcache *lru.ARCCache[common.Hash, c } // store inserts the snapshot into the database. -func (s *Snapshot) store(db kv.RwDB) error { +func (s *Snapshot) Store(db kv.RwDB) error { blob, err := json.Marshal(s) if err != nil { return err @@ -118,7 +118,7 @@ func (s *Snapshot) copy() *Snapshot { return cpy } -func (s *Snapshot) apply(headers []*types.Header, logger log.Logger) (*Snapshot, error) { +func (s *Snapshot) Apply(parent *types.Header, headers []*types.Header, logger log.Logger) (*Snapshot, error) { // Allow passing in no headers for cleaner code if len(headers) == 0 { return s, nil @@ -146,30 +146,36 @@ func (s *Snapshot) apply(headers []*types.Header, logger log.Logger) (*Snapshot, delete(snap.Recents, number-sprintLen) } // Resolve the authorization key and check against signers - signer, err := ecrecover(header, s.sigcache, s.config) + signer, err := Ecrecover(header, s.sigcache, s.config) if err != nil { return nil, err } var validSigner bool + var succession int // check if signer is in validator set - if snap.ValidatorSet.HasAddress(signer) { - if _, err = snap.GetSignerSuccessionNumber(signer); err != nil { - return nil, err - } + if !snap.ValidatorSet.HasAddress(signer) { + return snap, &UnauthorizedSignerError{number, signer.Bytes()} + } + if succession, err = snap.GetSignerSuccessionNumber(signer); err != nil { + return snap, err + } - // add recents - snap.Recents[number] = signer + // add recents + snap.Recents[number] = signer - validSigner = true + validSigner = true + + if parent != nil && header.Time < parent.Time+CalcProducerDelay(number, succession, s.config) { + return snap, &BlockTooSoonError{number, succession} } // change validator set and change proposer if number > 0 && (number+1)%sprintLen == 0 { if err := ValidateHeaderExtraField(header.Extra); err != nil { - return nil, err + return snap, err } validatorBytes := header.Extra[extraVanity : len(header.Extra)-extraSeal] @@ -181,13 +187,13 @@ func (s *Snapshot) apply(headers []*types.Header, logger log.Logger) (*Snapshot, } if number > 64 && !validSigner { - return nil, &UnauthorizedSignerError{number, signer.Bytes()} + return snap, &UnauthorizedSignerError{number, signer.Bytes()} } + parent = header + snap.Number = number + snap.Hash = header.Hash() } - snap.Number += uint64(len(headers)) - snap.Hash = headers[len(headers)-1].Hash() - return snap, nil } diff --git a/consensus/bor/span.go b/consensus/bor/span.go index 7365fd10c80..41e8abec8db 100644 --- a/consensus/bor/span.go +++ b/consensus/bor/span.go @@ -10,7 +10,7 @@ import ( //go:generate mockgen -destination=./span_mock.go -package=bor . Spanner type Spanner interface { GetCurrentSpan(syscall consensus.SystemCall) (*span.Span, error) - GetCurrentValidators(blockNumber uint64, signer libcommon.Address, getSpanForBlock func(blockNum uint64) (*span.HeimdallSpan, error)) ([]*valset.Validator, error) - GetCurrentProducers(blockNumber uint64, signer libcommon.Address, getSpanForBlock func(blockNum uint64) (*span.HeimdallSpan, error)) ([]*valset.Validator, error) + GetCurrentValidators(spanId uint64, signer libcommon.Address, chain consensus.ChainHeaderReader) ([]*valset.Validator, error) + GetCurrentProducers(spanId uint64, signer libcommon.Address, chain consensus.ChainHeaderReader) ([]*valset.Validator, error) CommitSpan(heimdallSpan span.HeimdallSpan, syscall consensus.SystemCall) error } diff --git a/consensus/chain_reader.go b/consensus/chain_reader.go index 795e2a856e4..f79de40c4cc 100644 --- a/consensus/chain_reader.go +++ b/consensus/chain_reader.go @@ -78,3 +78,11 @@ func (cr ChainReaderImpl) GetTd(hash libcommon.Hash, number uint64) *big.Int { func (cr ChainReaderImpl) FrozenBlocks() uint64 { return cr.BlockReader.FrozenBlocks() } + +func (cr ChainReaderImpl) BorSpan(spanId uint64) []byte { + spanBytes, err := cr.BlockReader.Span(context.Background(), cr.Db, spanId) + if err != nil { + log.Error("BorSpan failed", "err", err) + } + return spanBytes +} diff --git a/consensus/consensus.go b/consensus/consensus.go index 1165c95bbef..0a98706fa34 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -55,6 +55,9 @@ type ChainHeaderReader interface { // Number of blocks frozen in the block snapshots FrozenBlocks() uint64 + + // Byte string representation of a bor span with given ID + BorSpan(spanId uint64) []byte } // ChainReader defines a small collection of methods needed to access the local diff --git a/consensus/merge/merge_test.go b/consensus/merge/merge_test.go index bf0558211d3..aee7810cd2f 100644 --- a/consensus/merge/merge_test.go +++ b/consensus/merge/merge_test.go @@ -41,6 +41,10 @@ func (r readerMock) FrozenBlocks() uint64 { return 0 } +func (r readerMock) BorSpan(spanId uint64) []byte { + return nil +} + // The thing only that changes beetwen normal ethash checks other than POW, is difficulty // and nonce so we are gonna test those func TestVerifyHeaderDifficulty(t *testing.T) { diff --git a/consensus/misc/gaslimit.go b/consensus/misc/gaslimit.go index 440a1629e26..16fab48373c 100644 --- a/consensus/misc/gaslimit.go +++ b/consensus/misc/gaslimit.go @@ -17,7 +17,6 @@ package misc import ( - "errors" "fmt" "github.com/ledgerwatch/erigon/params" @@ -36,7 +35,7 @@ func VerifyGaslimit(parentGasLimit, headerGasLimit uint64) error { return fmt.Errorf("invalid gas limit: have %d, want %d +-= %d", headerGasLimit, parentGasLimit, limit-1) } if headerGasLimit < params.MinGasLimit { - return errors.New("invalid gas limit below 5000") + return fmt.Errorf("invalid gas limit below %d", params.MinGasLimit) } return nil } diff --git a/core/chain_makers.go b/core/chain_makers.go index e9926701b7d..3532c14b611 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -657,3 +657,4 @@ func (cr *FakeChainReader) FrozenBlocks() uint64 func (cr *FakeChainReader) BorEventsByBlock(hash libcommon.Hash, number uint64) []rlp.RawValue { return nil } +func (cr *FakeChainReader) BorSpan(spanId uint64) []byte { return nil } diff --git a/diagnostics/diagnostic.go b/diagnostics/diagnostic.go new file mode 100644 index 00000000000..c8013ba90cd --- /dev/null +++ b/diagnostics/diagnostic.go @@ -0,0 +1,56 @@ +package diagnostics + +import ( + "context" + "net/http" + + "github.com/ledgerwatch/erigon-lib/common" + diaglib "github.com/ledgerwatch/erigon-lib/diagnostics" + "github.com/ledgerwatch/erigon/turbo/node" + "github.com/ledgerwatch/log/v3" + "github.com/urfave/cli/v2" +) + +type DiagnosticClient struct { + ctx *cli.Context + metricsMux *http.ServeMux + node *node.ErigonNode + + snapshotDownload map[string]diaglib.DownloadStatistics +} + +func NewDiagnosticClient(ctx *cli.Context, metricsMux *http.ServeMux, node *node.ErigonNode) *DiagnosticClient { + return &DiagnosticClient{ctx: ctx, metricsMux: metricsMux, node: node, snapshotDownload: map[string]diaglib.DownloadStatistics{}} +} + +func (d *DiagnosticClient) Setup() { + d.runSnapshotListener() +} + +func (d *DiagnosticClient) runSnapshotListener() { + go func() { + ctx, ch, cancel := diaglib.Context[diaglib.DownloadStatistics](context.Background(), 1) + defer cancel() + + rootCtx, _ := common.RootContext() + + diaglib.StartProviders(ctx, diaglib.TypeOf(diaglib.DownloadStatistics{}), log.Root()) + for { + select { + case <-rootCtx.Done(): + cancel() + return + case info := <-ch: + d.snapshotDownload[info.StagePrefix] = info + if info.DownloadFinished { + return + } + } + } + + }() +} + +func (d *DiagnosticClient) SnapshotDownload() map[string]diaglib.DownloadStatistics { + return d.snapshotDownload +} diff --git a/diagnostics/peers.go b/diagnostics/peers.go index 260c60b3456..e65e3713d2f 100644 --- a/diagnostics/peers.go +++ b/diagnostics/peers.go @@ -36,11 +36,11 @@ type PeerResponse struct { Protocols map[string]interface{} `json:"protocols"` // Sub-protocol specific metadata fields } -func SetupPeersAccess(ctx *cli.Context, metricsMux *http.ServeMux, node *node.ErigonNode) { +func SetupPeersAccess(ctxclient *cli.Context, metricsMux *http.ServeMux, node *node.ErigonNode) { metricsMux.HandleFunc("/peers", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Access-Control-Allow-Origin", "*") w.Header().Set("Content-Type", "application/json") - writePeers(w, ctx, node) + writePeers(w, ctxclient, node) }) } diff --git a/diagnostics/setup.go b/diagnostics/setup.go index 022e7eb7e56..44fc74570fc 100644 --- a/diagnostics/setup.go +++ b/diagnostics/setup.go @@ -11,6 +11,9 @@ import ( func Setup(ctx *cli.Context, metricsMux *http.ServeMux, node *node.ErigonNode) { debugMux := http.NewServeMux() + diagnostic := NewDiagnosticClient(ctx, debugMux, node) + diagnostic.Setup() + metricsMux.HandleFunc("/debug/", func(w http.ResponseWriter, r *http.Request) { r.URL.Path = strings.TrimPrefix(r.URL.Path, "/debug") r.URL.RawPath = strings.TrimPrefix(r.URL.RawPath, "/debug") @@ -27,5 +30,6 @@ func Setup(ctx *cli.Context, metricsMux *http.ServeMux, node *node.ErigonNode) { SetupNodeInfoAccess(debugMux, node) SetupPeersAccess(ctx, debugMux, node) SetupBootnodesAccess(debugMux, node) + SetupStagesAccess(debugMux, diagnostic) } diff --git a/diagnostics/snapshot_sync.go b/diagnostics/snapshot_sync.go new file mode 100644 index 00000000000..66bb2a8a392 --- /dev/null +++ b/diagnostics/snapshot_sync.go @@ -0,0 +1,18 @@ +package diagnostics + +import ( + "encoding/json" + "net/http" +) + +func SetupStagesAccess(metricsMux *http.ServeMux, diag *DiagnosticClient) { + metricsMux.HandleFunc("/snapshot-sync", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Content-Type", "application/json") + writeStages(w, diag) + }) +} + +func writeStages(w http.ResponseWriter, diag *DiagnosticClient) { + json.NewEncoder(w).Encode(diag.SnapshotDownload()) +} diff --git a/erigon-lib/common/fixedgas/protocol.go b/erigon-lib/common/fixedgas/protocol.go index 989aaf4e5dd..038e8bd0586 100644 --- a/erigon-lib/common/fixedgas/protocol.go +++ b/erigon-lib/common/fixedgas/protocol.go @@ -17,144 +17,16 @@ package fixedgas const ( - GasLimitBoundDivisor uint64 = 1024 // The bound divisor of the gas limit, used in update calculations. - MinGasLimit uint64 = 5000 // Minimum the gas limit may ever be. - GenesisGasLimit uint64 = 4712388 // Gas limit of the Genesis block. - - MaximumExtraDataSize uint64 = 32 // Maximum size extra data may be after Genesis. - ExpByteGas uint64 = 10 // Times ceil(log256(exponent)) for the EXP instruction. - SloadGas uint64 = 50 // Multiplied by the number of 32-byte words that are copied (round up) for any *COPY operation and added. - CallValueTransferGas uint64 = 9000 // Paid for CALL when the value transfer is non-zero. - CallNewAccountGas uint64 = 25000 // Paid for CALL when the destination address didn't exist prior. - TxGas uint64 = 21000 // Per transaction not creating a contract. NOTE: Not payable on data of calls between transactions. - TxGasContractCreation uint64 = 53000 // Per transaction that creates a contract. NOTE: Not payable on data of calls between transactions. - TxDataZeroGas uint64 = 4 // Per byte of data attached to a transaction that equals zero. NOTE: Not payable on data of calls between transactions. - QuadCoeffDiv uint64 = 512 // Divisor for the quadratic particle of the memory cost equation. - LogDataGas uint64 = 8 // Per byte in a LOG* operation's data. - CallStipend uint64 = 2300 // Free gas given at beginning of call. - - Sha3Gas uint64 = 30 // Once per SHA3 operation. - Sha3WordGas uint64 = 6 // Once per word of the SHA3 operation's data. - - SstoreSetGas uint64 = 20000 // Once per SLOAD operation. - SstoreResetGas uint64 = 5000 // Once per SSTORE operation if the zeroness changes from zero. - SstoreClearGas uint64 = 5000 // Once per SSTORE operation if the zeroness doesn't change. - SstoreRefundGas uint64 = 15000 // Once per SSTORE operation if the zeroness changes to zero. - - NetSstoreNoopGas uint64 = 200 // Once per SSTORE operation if the value doesn't change. - NetSstoreInitGas uint64 = 20000 // Once per SSTORE operation from clean zero. - NetSstoreCleanGas uint64 = 5000 // Once per SSTORE operation from clean non-zero. - NetSstoreDirtyGas uint64 = 200 // Once per SSTORE operation from dirty. - - NetSstoreClearRefund uint64 = 15000 // Once per SSTORE operation for clearing an originally existing storage slot - NetSstoreResetRefund uint64 = 4800 // Once per SSTORE operation for resetting to the original non-zero value - NetSstoreResetClearRefund uint64 = 19800 // Once per SSTORE operation for resetting to the original zero value - - SstoreSentryGasEIP2200 uint64 = 2300 // Minimum gas required to be present for an SSTORE call, not consumed - SstoreSetGasEIP2200 uint64 = 20000 // Once per SSTORE operation from clean zero to non-zero - SstoreResetGasEIP2200 uint64 = 5000 // Once per SSTORE operation from clean non-zero to something else - SstoreClearsScheduleRefundEIP2200 uint64 = 15000 // Once per SSTORE operation for clearing an originally existing storage slot - - ColdAccountAccessCostEIP2929 = uint64(2600) // COLD_ACCOUNT_ACCESS_COST - ColdSloadCostEIP2929 = uint64(2100) // COLD_SLOAD_COST - WarmStorageReadCostEIP2929 = uint64(100) // WARM_STORAGE_READ_COST - - // In EIP-2200: SstoreResetGas was 5000. - // In EIP-2929: SstoreResetGas was changed to '5000 - COLD_SLOAD_COST'. - // In EIP-3529: SSTORE_CLEARS_SCHEDULE is defined as SSTORE_RESET_GAS + ACCESS_LIST_STORAGE_KEY_COST - // Which becomes: 5000 - 2100 + 1900 = 4800 - SstoreClearsScheduleRefundEIP3529 uint64 = SstoreResetGasEIP2200 - ColdSloadCostEIP2929 + TxAccessListStorageKeyGas - - JumpdestGas uint64 = 1 // Once per JUMPDEST operation. - EpochDuration uint64 = 30000 // Duration between proof-of-work epochs. - - CreateDataGas uint64 = 200 // - CallCreateDepth uint64 = 1024 // Maximum depth of call/create stack. - ExpGas uint64 = 10 // Once per EXP instruction - LogGas uint64 = 375 // Per LOG* operation. - CopyGas uint64 = 3 // - StackLimit uint64 = 1024 // Maximum size of VM stack allowed. - TierStepGas uint64 = 0 // Once per operation, for a selection of them. - LogTopicGas uint64 = 375 // Multiplied by the * of the LOG*, per LOG transaction. e.g. LOG0 incurs 0 * c_txLogTopicGas, LOG4 incurs 4 * c_txLogTopicGas. - CreateGas uint64 = 32000 // Once per CREATE operation & contract-creation transaction. - Create2Gas uint64 = 32000 // Once per CREATE2 operation - SelfdestructRefundGas uint64 = 24000 // Refunded following a selfdestruct operation. - MemoryGas uint64 = 3 // Times the address of the (highest referenced byte in memory + 1). NOTE: referencing happens on read, write and in instructions such as RETURN and CALL. - - TxDataNonZeroGasFrontier uint64 = 68 // Per byte of data attached to a transaction that is not equal to zero. NOTE: Not payable on data of calls between transactions. - TxDataNonZeroGasEIP2028 uint64 = 16 // Per byte of non zero data attached to a transaction after EIP 2028 (part in Istanbul) - TxAccessListAddressGas uint64 = 2400 // Per address specified in EIP 2930 access list - TxAccessListStorageKeyGas uint64 = 1900 // Per storage key specified in EIP 2930 access list - - // These have been changed during the course of the chain - CallGasFrontier uint64 = 40 // Once per CALL operation & message call transaction. - CallGasEIP150 uint64 = 700 // Static portion of gas for CALL-derivates after EIP 150 (Tangerine) - BalanceGasFrontier uint64 = 20 // The cost of a BALANCE operation - BalanceGasEIP150 uint64 = 400 // The cost of a BALANCE operation after Tangerine - BalanceGasEIP1884 uint64 = 700 // The cost of a BALANCE operation after EIP 1884 (part of Istanbul) - ExtcodeSizeGasFrontier uint64 = 20 // Cost of EXTCODESIZE before EIP 150 (Tangerine) - ExtcodeSizeGasEIP150 uint64 = 700 // Cost of EXTCODESIZE after EIP 150 (Tangerine) - SloadGasFrontier uint64 = 50 - SloadGasEIP150 uint64 = 200 - SloadGasEIP1884 uint64 = 800 // Cost of SLOAD after EIP 1884 (part of Istanbul) - SloadGasEIP2200 uint64 = 800 // Cost of SLOAD after EIP 2200 (part of Istanbul) - ExtcodeHashGasConstantinople uint64 = 400 // Cost of EXTCODEHASH (introduced in Constantinople) - ExtcodeHashGasEIP1884 uint64 = 700 // Cost of EXTCODEHASH after EIP 1884 (part in Istanbul) - SelfdestructGasEIP150 uint64 = 5000 // Cost of SELFDESTRUCT post EIP 150 (Tangerine) - - // EXP has a dynamic portion depending on the size of the exponent - ExpByteFrontier uint64 = 10 // was set to 10 in Frontier - ExpByteEIP158 uint64 = 50 // was raised to 50 during Eip158 (Spurious Dragon) - - // Extcodecopy has a dynamic AND a static cost. This represents only the - // static portion of the gas. It was changed during EIP 150 (Tangerine) - ExtcodeCopyBaseFrontier uint64 = 20 - ExtcodeCopyBaseEIP150 uint64 = 700 - - // CreateBySelfdestructGas is used when the refunded account is one that does - // not exist. This logic is similar to call. - // Introduced in Tangerine Whistle (Eip 150) - CreateBySelfdestructGas uint64 = 25000 - - BaseFeeChangeDenominator = 8 // Bounds the amount the base fee can change between blocks. - ElasticityMultiplier = 2 // Bounds the maximum gas limit an EIP-1559 block may have. - InitialBaseFee = 1000000000 // Initial base fee for EIP-1559 blocks. + TxGas uint64 = 21000 // Per transaction not creating a contract. NOTE: Not payable on data of calls between transactions. + TxGasContractCreation uint64 = 53000 // Per transaction that creates a contract. NOTE: Not payable on data of calls between transactions. + TxDataZeroGas uint64 = 4 // Per byte of data attached to a transaction that equals zero. NOTE: Not payable on data of calls between transactions. + TxDataNonZeroGasFrontier uint64 = 68 // Per byte of data attached to a transaction that is not equal to zero. NOTE: Not payable on data of calls between transactions. + TxDataNonZeroGasEIP2028 uint64 = 16 // Per byte of non zero data attached to a transaction after EIP 2028 (part in Istanbul) + TxAccessListAddressGas uint64 = 2400 // Per address specified in EIP 2930 access list + TxAccessListStorageKeyGas uint64 = 1900 // Per storage key specified in EIP 2930 access list MaxCodeSize = 24576 // Maximum bytecode to permit for a contract - // Precompiled contract gas prices - - EcrecoverGas uint64 = 3000 // Elliptic curve sender recovery gas price - Sha256BaseGas uint64 = 60 // Base price for a SHA256 operation - Sha256PerWordGas uint64 = 12 // Per-word price for a SHA256 operation - Ripemd160BaseGas uint64 = 600 // Base price for a RIPEMD160 operation - Ripemd160PerWordGas uint64 = 120 // Per-word price for a RIPEMD160 operation - IdentityBaseGas uint64 = 15 // Base price for a data copy operation - IdentityPerWordGas uint64 = 3 // Per-work price for a data copy operation - - Bn256AddGasByzantium uint64 = 500 // Byzantium gas needed for an elliptic curve addition - Bn256AddGasIstanbul uint64 = 150 // Gas needed for an elliptic curve addition - Bn256ScalarMulGasByzantium uint64 = 40000 // Byzantium gas needed for an elliptic curve scalar multiplication - Bn256ScalarMulGasIstanbul uint64 = 6000 // Gas needed for an elliptic curve scalar multiplication - Bn256PairingBaseGasByzantium uint64 = 100000 // Byzantium base price for an elliptic curve pairing check - Bn256PairingBaseGasIstanbul uint64 = 45000 // Base price for an elliptic curve pairing check - Bn256PairingPerPointGasByzantium uint64 = 80000 // Byzantium per-point price for an elliptic curve pairing check - Bn256PairingPerPointGasIstanbul uint64 = 34000 // Per-point price for an elliptic curve pairing check - - Bls12381G1AddGas uint64 = 600 // Price for BLS12-381 elliptic curve G1 point addition - Bls12381G1MulGas uint64 = 12000 // Price for BLS12-381 elliptic curve G1 point scalar multiplication - Bls12381G2AddGas uint64 = 4500 // Price for BLS12-381 elliptic curve G2 point addition - Bls12381G2MulGas uint64 = 55000 // Price for BLS12-381 elliptic curve G2 point scalar multiplication - Bls12381PairingBaseGas uint64 = 115000 // Base gas price for BLS12-381 elliptic curve pairing check - Bls12381PairingPerPairGas uint64 = 23000 // Per-point pair gas price for BLS12-381 elliptic curve pairing check - Bls12381MapG1Gas uint64 = 5500 // Gas price for BLS12-381 mapping field element to G1 operation - Bls12381MapG2Gas uint64 = 110000 // Gas price for BLS12-381 mapping field element to G2 operation - - // The Refund Quotient is the cap on how much of the used gas can be refunded. Before EIP-3529, - // up to half the consumed gas could be refunded. Redefined as 1/5th in EIP-3529 - RefundQuotient uint64 = 2 - RefundQuotientEIP3529 uint64 = 5 - // EIP-3860 to limit size of initcode MaxInitCodeSize = 2 * MaxCodeSize // Maximum initcode to permit in a creation transaction and create instructions InitCodeWordGas = 2 diff --git a/erigon-lib/diagnostics/entities.go b/erigon-lib/diagnostics/entities.go index 9b03d7813f2..d8b8172fa14 100644 --- a/erigon-lib/diagnostics/entities.go +++ b/erigon-lib/diagnostics/entities.go @@ -28,3 +28,22 @@ type PeerStatistics struct { TypeBytesIn map[string]uint64 TypeBytesOut map[string]uint64 } + +type DownloadStatistics struct { + Downloaded uint64 `json:"downloaded"` + Total uint64 `json:"total"` + TotalTime float64 `json:"totalTime"` + DownloadRate uint64 `json:"downloadRate"` + UploadRate uint64 `json:"uploadRate"` + Peers int32 `json:"peers"` + Files int32 `json:"files"` + Connections uint64 `json:"connections"` + Alloc uint64 `json:"alloc"` + Sys uint64 `json:"sys"` + DownloadFinished bool `json:"downloadFinished"` + StagePrefix string `json:"stagePrefix"` +} + +func (ti DownloadStatistics) Type() Type { + return TypeOf(ti) +} diff --git a/erigon-lib/diagnostics/network.go b/erigon-lib/diagnostics/network.go index 08bfaed8d31..7436a4b9166 100644 --- a/erigon-lib/diagnostics/network.go +++ b/erigon-lib/diagnostics/network.go @@ -16,8 +16,6 @@ package diagnostics -import "reflect" - func (p PeerStatistics) Type() Type { - return Type(reflect.TypeOf(p)) + return TypeOf(p) } diff --git a/erigon-lib/diagnostics/provider.go b/erigon-lib/diagnostics/provider.go index c1c2ae756c7..ef9b3f045f5 100644 --- a/erigon-lib/diagnostics/provider.go +++ b/erigon-lib/diagnostics/provider.go @@ -6,6 +6,7 @@ import ( "fmt" "reflect" "sync" + "sync/atomic" "github.com/ledgerwatch/erigon-lib/common/dbg" "github.com/ledgerwatch/log/v3" @@ -17,7 +18,35 @@ const ( ckChan ctxKey = iota ) -type Type reflect.Type +type Type interface { + reflect.Type + Context() context.Context + Err() error +} + +type diagType struct { + reflect.Type +} + +var cancelled = func() context.Context { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + return ctx +}() + +func (t diagType) Context() context.Context { + providerMutex.Lock() + defer providerMutex.Unlock() + if reg := providers[t]; reg != nil { + return reg.context + } + + return cancelled +} + +func (t diagType) Err() error { + return t.Context().Err() +} type Info interface { Type() Type @@ -25,7 +54,7 @@ type Info interface { func TypeOf(i Info) Type { t := reflect.TypeOf(i) - return Type(t) + return diagType{t} } type Provider interface { @@ -50,7 +79,7 @@ func RegisterProvider(provider Provider, infoType Type, logger log.Logger) { providerMutex.Lock() defer providerMutex.Unlock() - reg, _ := providers[infoType] + reg := providers[infoType] if reg != nil { for _, p := range reg.providers { @@ -73,13 +102,14 @@ func RegisterProvider(provider Provider, infoType Type, logger log.Logger) { func StartProviders(ctx context.Context, infoType Type, logger log.Logger) { providerMutex.Lock() - reg, _ := providers[infoType] + reg := providers[infoType] + if reg == nil { + reg = ®istry{} + providers[infoType] = reg + } toStart := make([]Provider, len(reg.providers)) - - for i, provider := range reg.providers { - toStart[i] = provider - } + copy(toStart, reg.providers) reg.context = ctx @@ -105,18 +135,29 @@ func startProvider(ctx context.Context, infoType Type, provider Provider, logger } } -func Send[I Info](ctx context.Context, info I) error { +func Send[I Info](info I) error { + ctx := info.Type().Context() + if ctx.Err() != nil { + if !errors.Is(ctx.Err(), context.Canceled) { + // drop the diagnostic message if there is + // no active diagnostic context for the type + return nil + } + return ctx.Err() } cval := ctx.Value(ckChan) - if c, ok := cval.(chan I); ok { - select { - case c <- info: - default: - // drop the diagnostic message if the receiver is busy - // so the sender is not blocked on non critcal actions + + if cp, ok := cval.(*atomic.Pointer[chan I]); ok { + if c := (*cp).Load(); c != nil { + select { + case *c <- info: + default: + // drop the diagnostic message if the receiver is busy + // so the sender is not blocked on non critcal actions + } } } else { return fmt.Errorf("unexpected channel type: %T", cval) @@ -126,16 +167,20 @@ func Send[I Info](ctx context.Context, info I) error { } func Context[I Info](ctx context.Context, buffer int) (context.Context, <-chan I, context.CancelFunc) { - ch := make(chan I, buffer) - ctx = context.WithValue(ctx, ckChan, ch) + c := make(chan I, buffer) + cp := atomic.Pointer[chan I]{} + cp.Store(&c) + + ctx = context.WithValue(ctx, ckChan, &cp) ctx, cancel := context.WithCancel(ctx) - return ctx, ch, func() { - if ch != nil { - toClose := ch - ch = nil - close(toClose) - } + return ctx, *cp.Load(), func() { cancel() + + if cp.CompareAndSwap(&c, nil) { + ch := c + c = nil + close(ch) + } } } diff --git a/erigon-lib/diagnostics/provider_test.go b/erigon-lib/diagnostics/provider_test.go index 7d8ea6b10ec..b5f2fefc7f4 100644 --- a/erigon-lib/diagnostics/provider_test.go +++ b/erigon-lib/diagnostics/provider_test.go @@ -31,7 +31,7 @@ func (t *testProvider) StartDiagnostics(ctx context.Context) error { case <-ctx.Done(): return nil case <-timer.C: - diagnostics.Send(ctx, testInfo{count}) + diagnostics.Send(testInfo{count}) count++ } } @@ -54,6 +54,25 @@ func TestProviderRegistration(t *testing.T) { } } +func TestDelayedProviderRegistration(t *testing.T) { + + time.AfterFunc(1*time.Second, func() { + // diagnostics provider + provider := &testProvider{} + diagnostics.RegisterProvider(provider, diagnostics.TypeOf(testInfo{}), log.Root()) + }) + + // diagnostics receiver + ctx, ch, cancel := diagnostics.Context[testInfo](context.Background(), 1) + diagnostics.StartProviders(ctx, diagnostics.TypeOf(testInfo{}), log.Root()) + + for info := range ch { + if info.count == 3 { + cancel() + } + } +} + func TestProviderFuncRegistration(t *testing.T) { // diagnostics provider @@ -68,7 +87,7 @@ func TestProviderFuncRegistration(t *testing.T) { case <-ctx.Done(): return nil case <-timer.C: - diagnostics.Send(ctx, testInfo{count}) + diagnostics.Send(testInfo{count}) count++ } } diff --git a/erigon-lib/downloader/downloadercfg/logger.go b/erigon-lib/downloader/downloadercfg/logger.go index b3a3178d101..88eb5dcabfa 100644 --- a/erigon-lib/downloader/downloadercfg/logger.go +++ b/erigon-lib/downloader/downloadercfg/logger.go @@ -127,7 +127,8 @@ func (b adapterHandler) Handle(r lg.Record) { log.Error(str) default: str := r.String() - skip := strings.Contains(str, "EOF") || strings.Contains(str, "unhandled response status") + skip := strings.Contains(str, "EOF") || strings.Contains(str, "unhandled response status") || + strings.Contains(str, "error doing webseed request") if skip { log.Trace(str, "lvl", lvl.LogString()) break diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index 8e0ee07090a..c2e35548ab5 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -4,7 +4,7 @@ go 1.20 require ( github.com/erigontech/mdbx-go v0.35.2-0.20231101074031-9f999220e9ed - github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231102060711-19219b948f46 + github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231106204511-f1e556dd5c50 github.com/ledgerwatch/interfaces v0.0.0-20231031050643-c86352e41520 github.com/ledgerwatch/log/v3 v3.9.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index 1214b33a4d6..83f40d9327c 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -293,8 +293,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= -github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231102060711-19219b948f46 h1:yt3/AcefMQOzY/P05jyeaKpqMQvrCbL6OJWALsjKp5U= -github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231102060711-19219b948f46/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= +github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231106204511-f1e556dd5c50 h1:RECb+fAC9doD1EhVxK2/b20JeLCAumLDjnysSQ3kWfs= +github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231106204511-f1e556dd5c50/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.9.0 h1:iDwrXe0PVwBC68Dd94YSsHbMgQ3ufsgjzXtFNFVZFRk= github.com/ledgerwatch/log/v3 v3.9.0/go.mod h1:EiAY6upmI/6LkNhOVxb4eVsmsP11HZCnZ3PlJMjYiqE= github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ= diff --git a/erigon-lib/kv/tables.go b/erigon-lib/kv/tables.go index 995603bfa8f..d67d43db6f3 100644 --- a/erigon-lib/kv/tables.go +++ b/erigon-lib/kv/tables.go @@ -438,6 +438,10 @@ const ( // [Block Root] => [State Root] BlockRootToStateRoot = "BlockRootToStateRoot" StateRootToBlockRoot = "StateRootToBlockRoot" + + BlockRootToBlockNumber = "BlockRootToBlockNumber" + BlockRootToBlockHash = "BlockRootToBlockHash" + // [Block Root] => [Parent Root] BlockRootToParentRoot = "BlockRootToParentRoot" @@ -608,6 +612,8 @@ var ChaindataTables = []string{ Attestetations, LightClient, LightClientUpdates, + BlockRootToBlockHash, + BlockRootToBlockNumber, } const ( diff --git a/erigon-lib/rlp2/commitment.go b/erigon-lib/rlp2/commitment.go new file mode 100644 index 00000000000..c554cfe6cea --- /dev/null +++ b/erigon-lib/rlp2/commitment.go @@ -0,0 +1,284 @@ +/* + Copyright 2022 Erigon contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package rlp + +import "io" + +// RLP-related utilities necessary for computing commitments for state root hash + +// generateRlpPrefixLenDouble calculates the length of RLP prefix to encode a string of bytes of length l "twice", +// meaning that it is the prefix for rlp(rlp(data)) +func generateRlpPrefixLenDouble(l int, firstByte byte) int { + if l < 2 { + // firstByte only matters when there is 1 byte to encode + if firstByte >= 0x80 { + return 2 + } + return 0 + } + if l < 55 { + return 2 + } + if l < 56 { // 2 + 1 + return 3 + } + if l < 254 { + return 4 + } + if l < 256 { + return 5 + } + if l < 65533 { + return 6 + } + if l < 65536 { + return 7 + } + return 8 +} + +func multiByteHeaderPrefixOfLen(l int) byte { + // > If a string is more than 55 bytes long, the + // > RLP encoding consists of a single byte with value 0xB7 plus the length + // > of the length of the string in binary form, followed by the length of + // > the string, followed by the string. For example, a length-1024 string + // > would be encoded as 0xB90400 followed by the string. The range of + // > the first byte is thus [0xB8, 0xBF]. + // + // see package rlp/decode.go:887 + return byte(0xB7 + l) +} + +func generateByteArrayLen(buffer []byte, pos int, l int) int { + if l < 56 { + buffer[pos] = byte(0x80 + l) + pos++ + } else if l < 256 { + // len(vn) can be encoded as 1 byte + buffer[pos] = multiByteHeaderPrefixOfLen(1) + pos++ + buffer[pos] = byte(l) + pos++ + } else if l < 65536 { + // len(vn) is encoded as two bytes + buffer[pos] = multiByteHeaderPrefixOfLen(2) + pos++ + buffer[pos] = byte(l >> 8) + pos++ + buffer[pos] = byte(l & 255) + pos++ + } else { + // len(vn) is encoded as three bytes + buffer[pos] = multiByteHeaderPrefixOfLen(3) + pos++ + buffer[pos] = byte(l >> 16) + pos++ + buffer[pos] = byte((l >> 8) & 255) + pos++ + buffer[pos] = byte(l & 255) + pos++ + } + return pos +} + +func generateByteArrayLenDouble(buffer []byte, pos int, l int) int { + if l < 55 { + // After first wrapping, the length will be l + 1 < 56 + buffer[pos] = byte(0x80 + l + 1) + pos++ + buffer[pos] = byte(0x80 + l) + pos++ + } else if l < 56 { + buffer[pos] = multiByteHeaderPrefixOfLen(1) + pos++ + buffer[pos] = byte(l + 1) + pos++ + buffer[pos] = byte(0x80 + l) + pos++ + } else if l < 254 { + // After first wrapping, the length will be l + 2 < 256 + buffer[pos] = multiByteHeaderPrefixOfLen(1) + pos++ + buffer[pos] = byte(l + 2) + pos++ + buffer[pos] = multiByteHeaderPrefixOfLen(1) + pos++ + buffer[pos] = byte(l) + pos++ + } else if l < 256 { + // First wrapping is 2 bytes, second wrapping 3 bytes + buffer[pos] = multiByteHeaderPrefixOfLen(2) + pos++ + buffer[pos] = byte((l + 2) >> 8) + pos++ + buffer[pos] = byte((l + 2) & 255) + pos++ + buffer[pos] = multiByteHeaderPrefixOfLen(1) + pos++ + buffer[pos] = byte(l) + pos++ + } else if l < 65533 { + // Both wrappings are 3 bytes + buffer[pos] = multiByteHeaderPrefixOfLen(2) + pos++ + buffer[pos] = byte((l + 3) >> 8) + pos++ + buffer[pos] = byte((l + 3) & 255) + pos++ + buffer[pos] = multiByteHeaderPrefixOfLen(2) + pos++ + buffer[pos] = byte(l >> 8) + pos++ + buffer[pos] = byte(l & 255) + pos++ + } else if l < 65536 { + // First wrapping is 3 bytes, second wrapping is 4 bytes + buffer[pos] = multiByteHeaderPrefixOfLen(3) + pos++ + buffer[pos] = byte((l + 3) >> 16) + pos++ + buffer[pos] = byte(((l + 3) >> 8) & 255) + pos++ + buffer[pos] = byte((l + 3) & 255) + pos++ + buffer[pos] = multiByteHeaderPrefixOfLen(2) + pos++ + buffer[pos] = byte((l >> 8) & 255) + pos++ + buffer[pos] = byte(l & 255) + pos++ + } else { + // Both wrappings are 4 bytes + buffer[pos] = multiByteHeaderPrefixOfLen(3) + pos++ + buffer[pos] = byte((l + 4) >> 16) + pos++ + buffer[pos] = byte(((l + 4) >> 8) & 255) + pos++ + buffer[pos] = byte((l + 4) & 255) + pos++ + buffer[pos] = multiByteHeaderPrefixOfLen(3) + pos++ + buffer[pos] = byte(l >> 16) + pos++ + buffer[pos] = byte((l >> 8) & 255) + pos++ + buffer[pos] = byte(l & 255) + pos++ + } + return pos +} + +func generateRlpPrefixLen(l int) int { + if l < 2 { + return 0 + } + if l < 56 { + return 1 + } + if l < 256 { + return 2 + } + if l < 65536 { + return 3 + } + return 4 +} + +// RlpSerializable is a value that can be double-RLP coded. +type RlpSerializable interface { + ToDoubleRLP(io.Writer, []byte) error + DoubleRLPLen() int + RawBytes() []byte +} + +type RlpSerializableBytes []byte + +func (b RlpSerializableBytes) ToDoubleRLP(w io.Writer, prefixBuf []byte) error { + return encodeBytesAsRlpToWriter(b, w, generateByteArrayLenDouble, prefixBuf) +} + +func (b RlpSerializableBytes) RawBytes() []byte { + return b +} + +func (b RlpSerializableBytes) DoubleRLPLen() int { + if len(b) < 1 { + return 0 + } + return generateRlpPrefixLenDouble(len(b), b[0]) + len(b) +} + +type RlpEncodedBytes []byte + +func (b RlpEncodedBytes) ToDoubleRLP(w io.Writer, prefixBuf []byte) error { + return encodeBytesAsRlpToWriter(b, w, generateByteArrayLen, prefixBuf) +} + +func (b RlpEncodedBytes) RawBytes() []byte { + return b +} + +func (b RlpEncodedBytes) DoubleRLPLen() int { + return generateRlpPrefixLen(len(b)) + len(b) +} + +func encodeBytesAsRlpToWriter(source []byte, w io.Writer, prefixGenFunc func([]byte, int, int) int, prefixBuf []byte) error { + // > 1 byte, write a prefix or prefixes first + if len(source) > 1 || (len(source) == 1 && source[0] >= 0x80) { + prefixLen := prefixGenFunc(prefixBuf, 0, len(source)) + + if _, err := w.Write(prefixBuf[:prefixLen]); err != nil { + return err + } + } + + _, err := w.Write(source) + return err +} + +func EncodeByteArrayAsRlp(raw []byte, w io.Writer, prefixBuf []byte) (int, error) { + err := encodeBytesAsRlpToWriter(raw, w, generateByteArrayLen, prefixBuf) + if err != nil { + return 0, err + } + return generateRlpPrefixLen(len(raw)) + len(raw), nil +} + +func GenerateStructLen(buffer []byte, l int) int { + if l < 56 { + buffer[0] = byte(192 + l) + return 1 + } + if l < 256 { + // l can be encoded as 1 byte + buffer[1] = byte(l) + buffer[0] = byte(247 + 1) + return 2 + } + if l < 65536 { + buffer[2] = byte(l & 255) + buffer[1] = byte(l >> 8) + buffer[0] = byte(247 + 2) + return 3 + } + buffer[3] = byte(l & 255) + buffer[2] = byte((l >> 8) & 255) + buffer[1] = byte(l >> 16) + buffer[0] = byte(247 + 3) + return 4 +} diff --git a/erigon-lib/rlp2/decoder.go b/erigon-lib/rlp2/decoder.go new file mode 100644 index 00000000000..4f41ecd7f9a --- /dev/null +++ b/erigon-lib/rlp2/decoder.go @@ -0,0 +1,277 @@ +package rlp + +import ( + "errors" + "fmt" + "io" +) + +type Decoder struct { + buf *buf +} + +func NewDecoder(buf []byte) *Decoder { + return &Decoder{ + buf: newBuf(buf, 0), + } +} + +func (d *Decoder) String() string { + return fmt.Sprintf(`left=%x pos=%d`, d.buf.Bytes(), d.buf.off) +} + +func (d *Decoder) Consumed() []byte { + return d.buf.u[:d.buf.off] +} + +func (d *Decoder) Underlying() []byte { + return d.buf.Underlying() +} + +func (d *Decoder) Empty() bool { + return d.buf.empty() +} + +func (d *Decoder) Offset() int { + return d.buf.Offset() +} + +func (d *Decoder) Bytes() []byte { + return d.buf.Bytes() +} + +func (d *Decoder) ReadByte() (n byte, err error) { + return d.buf.ReadByte() +} + +func (d *Decoder) PeekByte() (n byte, err error) { + return d.buf.PeekByte() +} + +func (d *Decoder) Rebase() { + d.buf.u = d.Bytes() + d.buf.off = 0 +} +func (d *Decoder) Fork() *Decoder { + return &Decoder{ + buf: newBuf(d.buf.u, d.buf.off), + } +} + +func (d *Decoder) PeekToken() (Token, error) { + prefix, err := d.PeekByte() + if err != nil { + return TokenUnknown, err + } + return identifyToken(prefix), nil +} + +func (d *Decoder) ElemDec() (*Decoder, Token, error) { + a, t, err := d.Elem() + return NewDecoder(a), t, err +} + +func (d *Decoder) RawElemDec() (*Decoder, Token, error) { + a, t, err := d.RawElem() + return NewDecoder(a), t, err +} + +func (d *Decoder) RawElem() ([]byte, Token, error) { + w := d.buf + start := w.Offset() + // figure out what we are reading + prefix, err := w.ReadByte() + if err != nil { + return nil, TokenUnknown, err + } + token := identifyToken(prefix) + + var ( + sz int + lenSz int + ) + // switch on the token + switch token { + case TokenDecimal: + // in this case, the value is just the byte itself + case TokenShortList: + sz = int(token.Diff(prefix)) + _, err = nextFull(w, sz) + case TokenLongList: + lenSz = int(token.Diff(prefix)) + sz, err = nextBeInt(w, lenSz) + if err != nil { + return nil, token, err + } + _, err = nextFull(w, sz) + case TokenShortBlob: + sz := int(token.Diff(prefix)) + _, err = nextFull(w, sz) + case TokenLongBlob: + lenSz := int(token.Diff(prefix)) + sz, err = nextBeInt(w, lenSz) + if err != nil { + return nil, token, err + } + _, err = nextFull(w, sz) + default: + return nil, token, fmt.Errorf("%w: unknown token", ErrDecode) + } + stop := w.Offset() + //log.Printf("%x %s\n", buf, token) + if err != nil { + return nil, token, err + } + return w.Underlying()[start:stop], token, nil +} + +func (d *Decoder) Elem() ([]byte, Token, error) { + w := d.buf + // figure out what we are reading + prefix, err := w.ReadByte() + if err != nil { + return nil, TokenUnknown, err + } + token := identifyToken(prefix) + + var ( + buf []byte + sz int + lenSz int + ) + // switch on the token + switch token { + case TokenDecimal: + // in this case, the value is just the byte itself + buf = []byte{prefix} + case TokenShortList: + sz = int(token.Diff(prefix)) + buf, err = nextFull(w, sz) + case TokenLongList: + lenSz = int(token.Diff(prefix)) + sz, err = nextBeInt(w, lenSz) + if err != nil { + return nil, token, err + } + buf, err = nextFull(w, sz) + case TokenShortBlob: + sz := int(token.Diff(prefix)) + buf, err = nextFull(w, sz) + case TokenLongBlob: + lenSz := int(token.Diff(prefix)) + sz, err = nextBeInt(w, lenSz) + if err != nil { + return nil, token, err + } + buf, err = nextFull(w, sz) + default: + return nil, token, fmt.Errorf("%w: unknown token", ErrDecode) + } + //log.Printf("%x %s\n", buf, token) + if err != nil { + return nil, token, fmt.Errorf("read data: %w", err) + } + return buf, token, nil +} + +func ReadElem[T any](d *Decoder, fn func(*T, []byte) error, receiver *T) error { + buf, token, err := d.Elem() + if err != nil { + return err + } + switch token { + case TokenDecimal, + TokenShortBlob, + TokenLongBlob, + TokenShortList, + TokenLongList: + return fn(receiver, buf) + default: + return fmt.Errorf("%w: ReadElem found unexpected token", ErrDecode) + } +} + +func (d *Decoder) ForList(fn func(*Decoder) error) error { + // grab the list bytes + buf, token, err := d.Elem() + if err != nil { + return err + } + switch token { + case TokenShortList, TokenLongList: + dec := NewDecoder(buf) + for { + if dec.buf.Len() == 0 { + return nil + } + err := fn(dec) + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return err + } + // reset the byte + dec = NewDecoder(dec.Bytes()) + } + default: + return fmt.Errorf("%w: ForList on non-list", ErrDecode) + } +} + +type buf struct { + u []byte + off int +} + +func newBuf(u []byte, off int) *buf { + return &buf{u: u, off: off} +} + +func (b *buf) empty() bool { return len(b.u) <= b.off } + +func (b *buf) PeekByte() (n byte, err error) { + if len(b.u) <= b.off { + return 0, io.EOF + } + return b.u[b.off], nil +} +func (b *buf) ReadByte() (n byte, err error) { + if len(b.u) <= b.off { + return 0, io.EOF + } + b.off++ + return b.u[b.off-1], nil +} + +func (b *buf) Next(n int) (xs []byte) { + m := b.Len() + if n > m { + n = m + } + data := b.u[b.off : b.off+n] + b.off += n + return data +} + +func (b *buf) Offset() int { + return b.off +} + +func (b *buf) Bytes() []byte { + return b.u[b.off:] +} + +func (b *buf) String() string { + if b == nil { + // Special case, useful in debugging. + return "" + } + return string(b.u[b.off:]) +} + +func (b *buf) Len() int { return len(b.u) - b.off } + +func (b *buf) Underlying() []byte { + return b.u +} diff --git a/erigon-lib/rlp2/encodel.go b/erigon-lib/rlp2/encodel.go new file mode 100644 index 00000000000..7e075a7b8c0 --- /dev/null +++ b/erigon-lib/rlp2/encodel.go @@ -0,0 +1,298 @@ +/* + Copyright 2021 The Erigon contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package rlp + +import ( + "encoding/binary" + "math/bits" + + "github.com/ledgerwatch/erigon-lib/common" +) + +// General design: +// - rlp package doesn't manage memory - and Caller must ensure buffers are big enough. +// - no io.Writer, because it's incompatible with binary.BigEndian functions and Writer can't be used as temporary buffer +// +// Composition: +// - each Encode method does write to given buffer and return written len +// - each Parse accept position in payload and return new position +// +// General rules: +// - functions to calculate prefix len are fast (and pure). it's ok to call them multiple times during encoding of large object for readability. +// - rlp has 2 data types: List and String (bytes array), and low-level funcs are operate with this types. +// - but for convenience and performance - provided higher-level functions (for example for EncodeHash - for []byte of len 32) +// - functions to Parse (Decode) data - using data type as name (without any prefix): rlp.String(), rlp.List, rlp.U64(), rlp.U256() +// + +func ListPrefixLen(dataLen int) int { + if dataLen >= 56 { + return 1 + common.BitLenToByteLen(bits.Len64(uint64(dataLen))) + } + return 1 +} +func EncodeListPrefix(dataLen int, to []byte) int { + if dataLen >= 56 { + _ = to[9] + beLen := common.BitLenToByteLen(bits.Len64(uint64(dataLen))) + binary.BigEndian.PutUint64(to[1:], uint64(dataLen)) + to[8-beLen] = 247 + byte(beLen) + copy(to, to[8-beLen:9]) + return 1 + beLen + } + to[0] = 192 + byte(dataLen) + return 1 +} + +func U32Len(i uint32) int { + if i < 128 { + return 1 + } + return 1 + common.BitLenToByteLen(bits.Len32(i)) +} + +func U64Len(i uint64) int { + if i < 128 { + return 1 + } + return 1 + common.BitLenToByteLen(bits.Len64(i)) +} + +func EncodeU32(i uint32, to []byte) int { + if i == 0 { + to[0] = 128 + return 1 + } + if i < 128 { + to[0] = byte(i) // fits single byte + return 1 + } + + b := to[1:] + var l int + + // writes i to b in big endian byte order, using the least number of bytes needed to represent i. + switch { + case i < (1 << 8): + b[0] = byte(i) + l = 1 + case i < (1 << 16): + b[0] = byte(i >> 8) + b[1] = byte(i) + l = 2 + case i < (1 << 24): + b[0] = byte(i >> 16) + b[1] = byte(i >> 8) + b[2] = byte(i) + l = 3 + default: + b[0] = byte(i >> 24) + b[1] = byte(i >> 16) + b[2] = byte(i >> 8) + b[3] = byte(i) + l = 4 + } + + to[0] = 128 + byte(l) + return 1 + l +} + +func EncodeU64(i uint64, to []byte) int { + if i == 0 { + to[0] = 128 + return 1 + } + if i < 128 { + to[0] = byte(i) // fits single byte + return 1 + } + + b := to[1:] + var l int + + // writes i to b in big endian byte order, using the least number of bytes needed to represent i. + switch { + case i < (1 << 8): + b[0] = byte(i) + l = 1 + case i < (1 << 16): + b[0] = byte(i >> 8) + b[1] = byte(i) + l = 2 + case i < (1 << 24): + b[0] = byte(i >> 16) + b[1] = byte(i >> 8) + b[2] = byte(i) + l = 3 + case i < (1 << 32): + b[0] = byte(i >> 24) + b[1] = byte(i >> 16) + b[2] = byte(i >> 8) + b[3] = byte(i) + l = 4 + case i < (1 << 40): + b[0] = byte(i >> 32) + b[1] = byte(i >> 24) + b[2] = byte(i >> 16) + b[3] = byte(i >> 8) + b[4] = byte(i) + l = 5 + case i < (1 << 48): + b[0] = byte(i >> 40) + b[1] = byte(i >> 32) + b[2] = byte(i >> 24) + b[3] = byte(i >> 16) + b[4] = byte(i >> 8) + b[5] = byte(i) + l = 6 + case i < (1 << 56): + b[0] = byte(i >> 48) + b[1] = byte(i >> 40) + b[2] = byte(i >> 32) + b[3] = byte(i >> 24) + b[4] = byte(i >> 16) + b[5] = byte(i >> 8) + b[6] = byte(i) + l = 7 + default: + b[0] = byte(i >> 56) + b[1] = byte(i >> 48) + b[2] = byte(i >> 40) + b[3] = byte(i >> 32) + b[4] = byte(i >> 24) + b[5] = byte(i >> 16) + b[6] = byte(i >> 8) + b[7] = byte(i) + l = 8 + } + + to[0] = 128 + byte(l) + return 1 + l +} + +func StringLen(s []byte) int { + sLen := len(s) + switch { + case sLen > 56: + beLen := common.BitLenToByteLen(bits.Len(uint(sLen))) + return 1 + beLen + sLen + case sLen == 0: + return 1 + case sLen == 1: + if s[0] < 128 { + return 1 + } + return 1 + sLen + default: // 1 56: + beLen := common.BitLenToByteLen(bits.Len(uint(len(s)))) + binary.BigEndian.PutUint64(to[1:], uint64(len(s))) + _ = to[beLen+len(s)] + + to[8-beLen] = byte(beLen) + 183 + copy(to, to[8-beLen:9]) + copy(to[1+beLen:], s) + return 1 + beLen + len(s) + case len(s) == 0: + to[0] = 128 + return 1 + case len(s) == 1: + if s[0] < 128 { + to[0] = s[0] + return 1 + } + to[0] = 129 + to[1] = s[0] + return 2 + default: // 1 55 { + return e.LongString(str) + } + return e.ShortString(str) +} + +// String will assume your string is less than 56 bytes long, and do no validation as such +func (e *Encoder) ShortString(str []byte) *Encoder { + return e.Byte(TokenShortBlob.Plus(byte(len(str)))).Bytes(str) +} + +// String will assume your string is greater than 55 bytes long, and do no validation as such +func (e *Encoder) LongString(str []byte) *Encoder { + // write the indicator token + e.Byte(byte(TokenLongBlob)) + // write the integer, knowing that we appended n bytes + n := putUint(e, len(str)) + // so we knw the indicator token was n+1 bytes ago. + e.buf[len(e.buf)-(int(n)+1)] += n + // and now add the actual length + e.buf = append(e.buf, str...) + return e +} + +// List will attempt to write the list of encoder funcs to the buf +func (e *Encoder) List(items ...EncoderFunc) *Encoder { + return e.writeList(true, items...) +} + +// ShortList actually calls List +func (e *Encoder) ShortList(items ...EncoderFunc) *Encoder { + return e.writeList(true, items...) +} + +// LongList will assume that your list payload is more than 55 bytes long, and do no validation as such +func (e *Encoder) LongList(items ...EncoderFunc) *Encoder { + return e.writeList(false, items...) +} + +// writeList will first attempt to write a long list with the dat +// if validate is false, it will just format it like the length is above 55 +// if validate is true, it will format it like it is a shrot list +func (e *Encoder) writeList(validate bool, items ...EncoderFunc) *Encoder { + // write the indicator token + e = e.Byte(byte(TokenLongList)) + // now pad 8 bytes + e = e.Bytes(make([]byte, 8)) + // record the length before encoding items + startLength := len(e.buf) + // now write all the items + for _, v := range items { + e = v(e) + } + // the size is the difference in the lengths now + dataSize := len(e.buf) - startLength + if dataSize <= 55 && validate { + // oh it's actually a short string! awkward. let's set that then. + e.buf[startLength-8-1] = TokenShortList.Plus(byte(dataSize)) + // and then copy the data over + copy(e.buf[startLength-8:], e.buf[startLength:startLength+dataSize]) + // and now set the new size + e.buf = e.buf[:startLength+dataSize-8] + // we are done, return + return e + } + // ok, so it's a long string. + // create a new encoder centered at startLength - 8 + enc := NewEncoder(e.buf[startLength-8:]) + // now write using that encoder the size + n := putUint(enc, dataSize) + // and update the token, which we know is at startLength-8-1 + e.buf[startLength-8-1] += n + // the shift to perform now is 8 - n. + shift := int(8 - n) + // if there is a positive shift, then we must perform the shift + if shift > 0 { + // copy the data + copy(e.buf[startLength-shift:], e.buf[startLength:startLength+dataSize]) + // set the new length + e.buf = e.buf[:startLength-shift+dataSize] + } + return e +} + +func putUint[T constraints.Integer](e *Encoder, t T) (size byte) { + i := uint64(t) + switch { + case i < (1 << 8): + e.buf = append(e.buf, byte(i)) + return 1 + case i < (1 << 16): + e.buf = append(e.buf, + byte(i>>8), + byte(i), + ) + return 2 + case i < (1 << 24): + + e.buf = append(e.buf, + byte(i>>16), + byte(i>>8), + byte(i), + ) + return 3 + case i < (1 << 32): + e.buf = append(e.buf, + byte(i>>24), + byte(i>>16), + byte(i>>8), + byte(i), + ) + return 4 + case i < (1 << 40): + e.buf = append(e.buf, + byte(i>>32), + byte(i>>24), + byte(i>>16), + byte(i>>8), + byte(i), + ) + return 5 + case i < (1 << 48): + e.buf = append(e.buf, + byte(i>>40), + byte(i>>32), + byte(i>>24), + byte(i>>16), + byte(i>>8), + byte(i), + ) + return 6 + case i < (1 << 56): + e.buf = append(e.buf, + byte(i>>48), + byte(i>>40), + byte(i>>32), + byte(i>>24), + byte(i>>16), + byte(i>>8), + byte(i), + ) + return 7 + default: + e.buf = append(e.buf, + byte(i>>56), + byte(i>>48), + byte(i>>40), + byte(i>>32), + byte(i>>24), + byte(i>>16), + byte(i>>8), + byte(i), + ) + return 8 + } +} diff --git a/erigon-lib/rlp2/parse.go b/erigon-lib/rlp2/parse.go new file mode 100644 index 00000000000..449277f0917 --- /dev/null +++ b/erigon-lib/rlp2/parse.go @@ -0,0 +1,289 @@ +/* + Copyright 2021 The Erigon contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package rlp + +import ( + "errors" + "fmt" + + "github.com/holiman/uint256" + + "github.com/ledgerwatch/erigon-lib/common" +) + +var ( + ErrBase = fmt.Errorf("rlp") + ErrParse = fmt.Errorf("%w parse", ErrBase) + ErrDecode = fmt.Errorf("%w decode", ErrBase) + ErrUnexpectedEOF = fmt.Errorf("%w EOF", ErrBase) +) + +func IsRLPError(err error) bool { return errors.Is(err, ErrBase) } + +// BeInt parses Big Endian representation of an integer from given payload at given position +func BeInt(payload []byte, pos, length int) (int, error) { + var r int + if pos+length > len(payload) { + return 0, ErrUnexpectedEOF + } + if length > 0 && payload[pos] == 0 { + return 0, fmt.Errorf("%w: integer encoding for RLP must not have leading zeros: %x", ErrParse, payload[pos:pos+length]) + } + for _, b := range payload[pos : pos+length] { + r = (r << 8) | int(b) + } + return r, nil +} + +// Prefix parses RLP Prefix from given payload at given position. It returns the offset and length of the RLP element +// as well as the indication of whether it is a list of string +func Prefix(payload []byte, pos int) (dataPos int, dataLen int, isList bool, err error) { + if pos < 0 { + return 0, 0, false, fmt.Errorf("%w: negative position not allowed", ErrParse) + } + if pos >= len(payload) { + return 0, 0, false, fmt.Errorf("%w: unexpected end of payload", ErrParse) + } + switch first := payload[pos]; { + case first < 128: + dataPos = pos + dataLen = 1 + isList = false + case first < 184: + // Otherwise, if a string is 0-55 bytes long, + // the RLP encoding consists of a single byte with value 0x80 plus the + // length of the string followed by the string. The range of the first + // byte is thus [0x80, 0xB7]. + dataPos = pos + 1 + dataLen = int(first) - 128 + isList = false + if dataLen == 1 && dataPos < len(payload) && payload[dataPos] < 128 { + err = fmt.Errorf("%w: non-canonical size information", ErrParse) + } + case first < 192: + // If a string is more than 55 bytes long, the + // RLP encoding consists of a single byte with value 0xB7 plus the length + // of the length of the string in binary form, followed by the length of + // the string, followed by the string. For example, a length-1024 string + // would be encoded as 0xB90400 followed by the string. The range of + // the first byte is thus [0xB8, 0xBF]. + beLen := int(first) - 183 + dataPos = pos + 1 + beLen + dataLen, err = BeInt(payload, pos+1, beLen) + isList = false + if dataLen < 56 { + err = fmt.Errorf("%w: non-canonical size information", ErrParse) + } + case first < 248: + // isList of len < 56 + // If the total payload of a list + // (i.e. the combined length of all its items) is 0-55 bytes long, the + // RLP encoding consists of a single byte with value 0xC0 plus the length + // of the list followed by the concatenation of the RLP encodings of the + // items. The range of the first byte is thus [0xC0, 0xF7]. + dataPos = pos + 1 + dataLen = int(first) - 192 + isList = true + default: + // If the total payload of a list is more than 55 bytes long, + // the RLP encoding consists of a single byte with value 0xF7 + // plus the length of the length of the payload in binary + // form, followed by the length of the payload, followed by + // the concatenation of the RLP encodings of the items. The + // range of the first byte is thus [0xF8, 0xFF]. + beLen := int(first) - 247 + dataPos = pos + 1 + beLen + dataLen, err = BeInt(payload, pos+1, beLen) + isList = true + if dataLen < 56 { + err = fmt.Errorf("%w: : non-canonical size information", ErrParse) + } + } + if err == nil { + if dataPos+dataLen > len(payload) { + err = fmt.Errorf("%w: unexpected end of payload", ErrParse) + } else if dataPos+dataLen < 0 { + err = fmt.Errorf("%w: found too big len", ErrParse) + } + } + return +} + +func List(payload []byte, pos int) (dataPos, dataLen int, err error) { + dataPos, dataLen, isList, err := Prefix(payload, pos) + if err != nil { + return 0, 0, err + } + if !isList { + return 0, 0, fmt.Errorf("%w: must be a list", ErrParse) + } + return +} + +func String(payload []byte, pos int) (dataPos, dataLen int, err error) { + dataPos, dataLen, isList, err := Prefix(payload, pos) + if err != nil { + return 0, 0, err + } + if isList { + return 0, 0, fmt.Errorf("%w: must be a string, instead of a list", ErrParse) + } + return +} +func StringOfLen(payload []byte, pos, expectedLen int) (dataPos int, err error) { + dataPos, dataLen, err := String(payload, pos) + if err != nil { + return 0, err + } + if dataLen != expectedLen { + return 0, fmt.Errorf("%w: expected string of len %d, got %d", ErrParse, expectedLen, dataLen) + } + return +} + +// U64 parses uint64 number from given payload at given position +func U64(payload []byte, pos int) (int, uint64, error) { + dataPos, dataLen, isList, err := Prefix(payload, pos) + if err != nil { + return 0, 0, err + } + if isList { + return 0, 0, fmt.Errorf("%w: uint64 must be a string, not isList", ErrParse) + } + if dataLen > 8 { + return 0, 0, fmt.Errorf("%w: uint64 must not be more than 8 bytes long, got %d", ErrParse, dataLen) + } + if dataLen > 0 && payload[dataPos] == 0 { + return 0, 0, fmt.Errorf("%w: integer encoding for RLP must not have leading zeros: %x", ErrParse, payload[dataPos:dataPos+dataLen]) + } + var r uint64 + for _, b := range payload[dataPos : dataPos+dataLen] { + r = (r << 8) | uint64(b) + } + return dataPos + dataLen, r, nil +} + +// U32 parses uint64 number from given payload at given position +func U32(payload []byte, pos int) (int, uint32, error) { + dataPos, dataLen, isList, err := Prefix(payload, pos) + if err != nil { + return 0, 0, err + } + if isList { + return 0, 0, fmt.Errorf("%w: uint32 must be a string, not isList", ErrParse) + } + if dataLen > 4 { + return 0, 0, fmt.Errorf("%w: uint32 must not be more than 4 bytes long, got %d", ErrParse, dataLen) + } + if dataLen > 0 && payload[dataPos] == 0 { + return 0, 0, fmt.Errorf("%w: integer encoding for RLP must not have leading zeros: %x", ErrParse, payload[dataPos:dataPos+dataLen]) + } + var r uint32 + for _, b := range payload[dataPos : dataPos+dataLen] { + r = (r << 8) | uint32(b) + } + return dataPos + dataLen, r, nil +} + +// U256 parses uint256 number from given payload at given position +func U256(payload []byte, pos int, x *uint256.Int) (int, error) { + dataPos, dataLen, err := String(payload, pos) + if err != nil { + return 0, err + } + if dataLen > 32 { + return 0, fmt.Errorf("%w: uint256 must not be more than 32 bytes long, got %d", ErrParse, dataLen) + } + if dataLen > 0 && payload[dataPos] == 0 { + return 0, fmt.Errorf("%w: integer encoding for RLP must not have leading zeros: %x", ErrParse, payload[dataPos:dataPos+dataLen]) + } + x.SetBytes(payload[dataPos : dataPos+dataLen]) + return dataPos + dataLen, nil +} + +func U256Len(z *uint256.Int) int { + if z == nil { + return 1 + } + nBits := z.BitLen() + if nBits == 0 { + return 1 + } + if nBits <= 7 { + return 1 + } + return 1 + common.BitLenToByteLen(nBits) +} + +func ParseHash(payload []byte, pos int, hashbuf []byte) (int, error) { + pos, err := StringOfLen(payload, pos, 32) + if err != nil { + return 0, fmt.Errorf("%s: hash len: %w", ParseHashErrorPrefix, err) + } + copy(hashbuf, payload[pos:pos+32]) + return pos + 32, nil +} + +const ParseHashErrorPrefix = "parse hash payload" + +const ParseAnnouncementsErrorPrefix = "parse announcement payload" + +func ParseAnnouncements(payload []byte, pos int) ([]byte, []uint32, []byte, int, error) { + pos, totalLen, err := List(payload, pos) + if err != nil { + return nil, nil, nil, pos, err + } + if pos+totalLen > len(payload) { + return nil, nil, nil, pos, fmt.Errorf("%s: totalLen %d is beyond the end of payload", ParseAnnouncementsErrorPrefix, totalLen) + } + pos, typesLen, err := String(payload, pos) + if err != nil { + return nil, nil, nil, pos, err + } + if pos+typesLen > len(payload) { + return nil, nil, nil, pos, fmt.Errorf("%s: typesLen %d is beyond the end of payload", ParseAnnouncementsErrorPrefix, typesLen) + } + types := payload[pos : pos+typesLen] + pos += typesLen + pos, sizesLen, err := List(payload, pos) + if err != nil { + return nil, nil, nil, pos, err + } + if pos+sizesLen > len(payload) { + return nil, nil, nil, pos, fmt.Errorf("%s: sizesLen %d is beyond the end of payload", ParseAnnouncementsErrorPrefix, sizesLen) + } + sizes := make([]uint32, typesLen) + for i := 0; i < len(sizes); i++ { + if pos, sizes[i], err = U32(payload, pos); err != nil { + return nil, nil, nil, pos, err + } + } + pos, hashesLen, err := List(payload, pos) + if err != nil { + return nil, nil, nil, pos, err + } + if pos+hashesLen > len(payload) { + return nil, nil, nil, pos, fmt.Errorf("%s: hashesLen %d is beyond the end of payload", ParseAnnouncementsErrorPrefix, hashesLen) + } + hashes := make([]byte, 32*(hashesLen/33)) + for i := 0; i < len(hashes); i += 32 { + if pos, err = ParseHash(payload, pos, hashes[i:]); err != nil { + return nil, nil, nil, pos, err + } + } + return types, sizes, hashes, pos, nil +} diff --git a/erigon-lib/rlp2/parse_test.go b/erigon-lib/rlp2/parse_test.go new file mode 100644 index 00000000000..00712776868 --- /dev/null +++ b/erigon-lib/rlp2/parse_test.go @@ -0,0 +1,92 @@ +package rlp + +import ( + "fmt" + "testing" + + "github.com/holiman/uint256" + "github.com/stretchr/testify/assert" + + "github.com/ledgerwatch/erigon-lib/common/hexutility" +) + +var parseU64Tests = []struct { + expectErr error + payload []byte + expectPos int + expectRes uint64 +}{ + {payload: hexutility.MustDecodeHex("820400"), expectPos: 3, expectRes: 1024}, + {payload: hexutility.MustDecodeHex("07"), expectPos: 1, expectRes: 7}, + {payload: hexutility.MustDecodeHex("8107"), expectErr: fmt.Errorf("%w: non-canonical size information", ErrParse)}, + {payload: hexutility.MustDecodeHex("B8020004"), expectErr: fmt.Errorf("%w: non-canonical size information", ErrParse)}, + {payload: hexutility.MustDecodeHex("C0"), expectErr: fmt.Errorf("%w: uint64 must be a string, not isList", ErrParse)}, + {payload: hexutility.MustDecodeHex("00"), expectErr: fmt.Errorf("%w: integer encoding for RLP must not have leading zeros: 00", ErrParse)}, + {payload: hexutility.MustDecodeHex("8AFFFFFFFFFFFFFFFFFF7C"), expectErr: fmt.Errorf("%w: uint64 must not be more than 8 bytes long, got 10", ErrParse)}, +} + +var parseU32Tests = []struct { + expectErr error + payload []byte + expectPos int + expectRes uint32 +}{ + {payload: hexutility.MustDecodeHex("820400"), expectPos: 3, expectRes: 1024}, + {payload: hexutility.MustDecodeHex("07"), expectPos: 1, expectRes: 7}, + {payload: hexutility.MustDecodeHex("8107"), expectErr: fmt.Errorf("%w: non-canonical size information", ErrParse)}, + {payload: hexutility.MustDecodeHex("B8020004"), expectErr: fmt.Errorf("%w: non-canonical size information", ErrParse)}, + {payload: hexutility.MustDecodeHex("C0"), expectErr: fmt.Errorf("%w: uint32 must be a string, not isList", ErrParse)}, + {payload: hexutility.MustDecodeHex("00"), expectErr: fmt.Errorf("%w: integer encoding for RLP must not have leading zeros: 00", ErrParse)}, + {payload: hexutility.MustDecodeHex("85FF6738FF7C"), expectErr: fmt.Errorf("%w: uint32 must not be more than 4 bytes long, got 5", ErrParse)}, +} + +var parseU256Tests = []struct { + expectErr error + expectRes *uint256.Int + payload []byte + expectPos int +}{ + {payload: hexutility.MustDecodeHex("8BFFFFFFFFFFFFFFFFFF7C"), expectErr: fmt.Errorf("%w: unexpected end of payload", ErrParse)}, + {payload: hexutility.MustDecodeHex("8AFFFFFFFFFFFFFFFFFF7C"), expectPos: 11, expectRes: new(uint256.Int).SetBytes(hexutility.MustDecodeHex("FFFFFFFFFFFFFFFFFF7C"))}, + {payload: hexutility.MustDecodeHex("85CE05050505"), expectPos: 6, expectRes: new(uint256.Int).SetUint64(0xCE05050505)}, + {payload: hexutility.MustDecodeHex("820400"), expectPos: 3, expectRes: new(uint256.Int).SetUint64(1024)}, + {payload: hexutility.MustDecodeHex("07"), expectPos: 1, expectRes: new(uint256.Int).SetUint64(7)}, + {payload: hexutility.MustDecodeHex("8107"), expectErr: fmt.Errorf("%w: non-canonical size information", ErrParse)}, + {payload: hexutility.MustDecodeHex("B8020004"), expectErr: fmt.Errorf("%w: non-canonical size information", ErrParse)}, + {payload: hexutility.MustDecodeHex("C0"), expectErr: fmt.Errorf("%w: must be a string, instead of a list", ErrParse)}, + {payload: hexutility.MustDecodeHex("00"), expectErr: fmt.Errorf("%w: integer encoding for RLP must not have leading zeros: 00", ErrParse)}, + {payload: hexutility.MustDecodeHex("A101000000000000000000000000000000000000008B000000000000000000000000"), expectErr: fmt.Errorf("%w: uint256 must not be more than 32 bytes long, got 33", ErrParse)}, +} + +func TestPrimitives(t *testing.T) { + for i, tt := range parseU64Tests { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + assert := assert.New(t) + pos, res, err := U64(tt.payload, 0) + assert.Equal(tt.expectErr, err) + assert.Equal(tt.expectPos, pos) + assert.Equal(tt.expectRes, res) + }) + } + for i, tt := range parseU32Tests { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + assert := assert.New(t) + pos, res, err := U32(tt.payload, 0) + assert.Equal(tt.expectErr, err) + assert.Equal(tt.expectPos, pos) + assert.Equal(tt.expectRes, res) + }) + } + for i, tt := range parseU256Tests { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + assert := assert.New(t) + res := new(uint256.Int) + pos, err := U256(tt.payload, 0, res) + assert.Equal(tt.expectErr, err) + assert.Equal(tt.expectPos, pos) + if err == nil { + assert.Equal(tt.expectRes, res) + } + }) + } +} diff --git a/erigon-lib/rlp2/readme.md b/erigon-lib/rlp2/readme.md new file mode 100644 index 00000000000..74e9f96eeb4 --- /dev/null +++ b/erigon-lib/rlp2/readme.md @@ -0,0 +1,11 @@ +## rlp + + +TERMINOLOGY: + +``` +RLP string = "Blob" // this is so we don't conflict with existing go name for String +RLP list = "List" // luckily we can keep using list name since go doesn't use it +RLP single byte number = "Decimal" // for numbers from 1-127. a special case +``` + diff --git a/erigon-lib/rlp2/types.go b/erigon-lib/rlp2/types.go new file mode 100644 index 00000000000..f33bdfdc25d --- /dev/null +++ b/erigon-lib/rlp2/types.go @@ -0,0 +1,59 @@ +package rlp + +import ( + "fmt" + + "github.com/holiman/uint256" +) + +func Bytes(dst *[]byte, src []byte) error { + if len(*dst) < len(src) { + (*dst) = make([]byte, len(src)) + } + copy(*dst, src) + return nil +} +func BytesExact(dst *[]byte, src []byte) error { + if len(*dst) != len(src) { + return fmt.Errorf("%w: BytesExact no match", ErrDecode) + } + copy(*dst, src) + return nil +} + +func Uint256(dst *uint256.Int, src []byte) error { + if len(src) > 32 { + return fmt.Errorf("%w: uint256 must not be more than 32 bytes long, got %d", ErrParse, len(src)) + } + if len(src) > 0 && src[0] == 0 { + return fmt.Errorf("%w: integer encoding for RLP must not have leading zeros: %x", ErrParse, src) + } + dst.SetBytes(src) + return nil +} + +func Uint64(dst *uint64, src []byte) error { + var r uint64 + for _, b := range src { + r = (r << 8) | uint64(b) + } + (*dst) = r + return nil +} + +func IsEmpty(dst *bool, src []byte) error { + if len(src) == 0 { + (*dst) = true + } else { + (*dst) = false + } + return nil +} +func BlobLength(dst *int, src []byte) error { + (*dst) = len(src) + return nil +} + +func Skip(dst *int, src []byte) error { + return nil +} diff --git a/erigon-lib/rlp2/unmarshaler.go b/erigon-lib/rlp2/unmarshaler.go new file mode 100644 index 00000000000..16c42a1f2f6 --- /dev/null +++ b/erigon-lib/rlp2/unmarshaler.go @@ -0,0 +1,191 @@ +package rlp + +import ( + "fmt" + "reflect" +) + +type Unmarshaler interface { + UnmarshalRLP(data []byte) error +} + +func Unmarshal(data []byte, val any) error { + buf := newBuf(data, 0) + return unmarshal(buf, val) +} + +func unmarshal(buf *buf, val any) error { + rv := reflect.ValueOf(val) + if rv.Kind() != reflect.Pointer || rv.IsNil() { + return fmt.Errorf("%w: v must be ptr", ErrDecode) + } + v := rv.Elem() + err := reflectAny(buf, v, rv) + if err != nil { + return fmt.Errorf("%w: %w", ErrDecode, err) + } + return nil +} + +func reflectAny(w *buf, v reflect.Value, rv reflect.Value) error { + if um, ok := rv.Interface().(Unmarshaler); ok { + return um.UnmarshalRLP(w.Bytes()) + } + // figure out what we are reading + prefix, err := w.ReadByte() + if err != nil { + return err + } + token := identifyToken(prefix) + // switch + switch token { + case TokenDecimal: + // in this case, the value is just the byte itself + switch v.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + v.SetInt(int64(prefix)) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + v.SetUint(uint64(prefix)) + case reflect.Invalid: + // do nothing + default: + return fmt.Errorf("%w: decimal must be unmarshal into integer type", ErrDecode) + } + case TokenShortBlob: + sz := int(token.Diff(prefix)) + str, err := nextFull(w, sz) + if err != nil { + return err + } + return putBlob(str, v, rv) + case TokenLongBlob: + lenSz := int(token.Diff(prefix)) + sz, err := nextBeInt(w, lenSz) + if err != nil { + return err + } + str, err := nextFull(w, sz) + if err != nil { + return err + } + return putBlob(str, v, rv) + case TokenShortList: + sz := int(token.Diff(prefix)) + buf, err := nextFull(w, sz) + if err != nil { + return err + } + return reflectList(newBuf(buf, 0), v, rv) + case TokenLongList: + lenSz := int(token.Diff(prefix)) + sz, err := nextBeInt(w, lenSz) + if err != nil { + return err + } + buf, err := nextFull(w, sz) + if err != nil { + return err + } + return reflectList(newBuf(buf, 0), v, rv) + case TokenUnknown: + return fmt.Errorf("%w: unknown token", ErrDecode) + } + return nil +} + +func putBlob(w []byte, v reflect.Value, rv reflect.Value) error { + switch v.Kind() { + case reflect.String: + v.SetString(string(w)) + case reflect.Slice: + if v.Type().Elem().Kind() != reflect.Uint8 { + return fmt.Errorf("%w: need to use uint8 as underlying if want slice output from longstring", ErrDecode) + } + v.SetBytes(w) + case reflect.Array: + if v.Type().Elem().Kind() != reflect.Uint8 { + return fmt.Errorf("%w: need to use uint8 as underlying if want array output from longstring", ErrDecode) + } + reflect.Copy(v, reflect.ValueOf(w)) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + val, err := BeInt(w, 0, len(w)) + if err != nil { + return err + } + v.SetInt(int64(val)) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + val, err := BeInt(w, 0, len(w)) + if err != nil { + return err + } + v.SetUint(uint64(val)) + case reflect.Invalid: + // do nothing + return nil + } + return nil +} + +func reflectList(w *buf, v reflect.Value, rv reflect.Value) error { + switch v.Kind() { + case reflect.Invalid: + // do nothing + return nil + case reflect.Map: + rv1 := reflect.New(v.Type().Key()) + v1 := rv1.Elem() + err := reflectAny(w, v1, rv1) + if err != nil { + return err + } + rv2 := reflect.New(v.Type().Elem()) + v2 := rv2.Elem() + err = reflectAny(w, v2, rv2) + if err != nil { + return err + } + v.SetMapIndex(rv1, rv2) + case reflect.Struct: + for idx := 0; idx < v.NumField(); idx++ { + // Decode into element. + rv1 := v.Field(idx).Addr() + rt1 := v.Type().Field(idx) + v1 := rv1.Elem() + shouldSet := rt1.IsExported() + if shouldSet { + err := reflectAny(w, v1, rv1) + if err != nil { + return err + } + } + } + case reflect.Array, reflect.Slice: + idx := 0 + for { + if idx >= v.Cap() { + v.Grow(1) + } + if idx >= v.Len() { + v.SetLen(idx + 1) + } + if idx < v.Len() { + // Decode into element. + rv1 := v.Index(idx) + v1 := rv1.Elem() + err := reflectAny(w, v1, rv1) + if err != nil { + return err + } + } else { + // Ran out of fixed array: skip. + rv1 := reflect.Value{} + err := reflectAny(w, rv1, rv1) + if err != nil { + return err + } + } + idx++ + } + } + return nil +} diff --git a/erigon-lib/rlp2/unmarshaler_test.go b/erigon-lib/rlp2/unmarshaler_test.go new file mode 100644 index 00000000000..07ad0dc015a --- /dev/null +++ b/erigon-lib/rlp2/unmarshaler_test.go @@ -0,0 +1,66 @@ +package rlp_test + +import ( + "testing" + + rlp "github.com/ledgerwatch/erigon-lib/rlp2" + "github.com/stretchr/testify/require" +) + +type plusOne int + +func (p *plusOne) UnmarshalRLP(data []byte) error { + var s int + err := rlp.Unmarshal(data, &s) + if err != nil { + return err + } + (*p) = plusOne(s + 1) + return nil +} + +func TestDecoder(t *testing.T) { + + type simple struct { + Key string + Value string + } + + t.Run("ShortString", func(t *testing.T) { + t.Run("ToString", func(t *testing.T) { + bts := []byte{0x83, 'd', 'o', 'g'} + var s string + err := rlp.Unmarshal(bts, &s) + require.NoError(t, err) + require.EqualValues(t, "dog", s) + }) + t.Run("ToBytes", func(t *testing.T) { + bts := []byte{0x83, 'd', 'o', 'g'} + var s []byte + err := rlp.Unmarshal(bts, &s) + require.NoError(t, err) + require.EqualValues(t, []byte("dog"), s) + }) + t.Run("ToInt", func(t *testing.T) { + bts := []byte{0x82, 0x04, 0x00} + var s int + err := rlp.Unmarshal(bts, &s) + require.NoError(t, err) + require.EqualValues(t, 1024, s) + }) + t.Run("ToIntUnmarshaler", func(t *testing.T) { + bts := []byte{0x82, 0x04, 0x00} + var s plusOne + err := rlp.Unmarshal(bts, &s) + require.NoError(t, err) + require.EqualValues(t, plusOne(1025), s) + }) + t.Run("ToSimpleStruct", func(t *testing.T) { + bts := []byte{0xc8, 0x83, 'c', 'a', 't', 0x83, 'd', 'o', 'g'} + var s simple + err := rlp.Unmarshal(bts, &s) + require.NoError(t, err) + require.EqualValues(t, simple{Key: "cat", Value: "dog"}, s) + }) + }) +} diff --git a/erigon-lib/rlp2/util.go b/erigon-lib/rlp2/util.go new file mode 100644 index 00000000000..0219e1d3953 --- /dev/null +++ b/erigon-lib/rlp2/util.go @@ -0,0 +1,84 @@ +package rlp + +type Token int32 + +func (T Token) String() string { + switch T { + case TokenDecimal: + return "decimal" + case TokenShortBlob: + return "short_blob" + case TokenLongBlob: + return "long_blob" + case TokenShortList: + return "short_list" + case TokenLongList: + return "long_list" + case TokenEOF: + return "eof" + case TokenUnknown: + return "unknown" + default: + return "nan" + } +} + +func (T Token) Plus(n byte) byte { + return byte(T) + n +} + +func (T Token) Diff(n byte) byte { + return n - byte(T) +} + +func (T Token) IsListType() bool { + return T == TokenLongList || T == TokenShortList +} + +func (T Token) IsBlobType() bool { + return T == TokenLongBlob || T == TokenShortBlob +} + +const ( + TokenDecimal Token = 0x00 + TokenShortBlob Token = 0x80 + TokenLongBlob Token = 0xb7 + TokenShortList Token = 0xc0 + TokenLongList Token = 0xf7 + + TokenUnknown Token = 0xff01 + TokenEOF Token = 0xdead +) + +func identifyToken(b byte) Token { + switch { + case b <= 127: + return TokenDecimal + case b >= 128 && b <= 183: + return TokenShortBlob + case b >= 184 && b <= 191: + return TokenLongBlob + case b >= 192 && b <= 247: + return TokenShortList + case b >= 248 && b <= 255: + return TokenLongList + } + return TokenUnknown +} + +// BeInt parses Big Endian representation of an integer from given payload at given position +func nextBeInt(w *buf, length int) (int, error) { + dat, err := nextFull(w, length) + if err != nil { + return 0, ErrUnexpectedEOF + } + return BeInt(dat, 0, length) +} + +func nextFull(dat *buf, size int) ([]byte, error) { + d := dat.Next(size) + if len(d) != size { + return nil, ErrUnexpectedEOF + } + return d, nil +} diff --git a/erigon-lib/tools/golangci_lint.sh b/erigon-lib/tools/golangci_lint.sh index f3fb6befce8..0b27a507523 100755 --- a/erigon-lib/tools/golangci_lint.sh +++ b/erigon-lib/tools/golangci_lint.sh @@ -13,7 +13,7 @@ fi if ! which golangci-lint > /dev/null then echo "golangci-lint tool is not found, install it with:" - echo " make lintci-deps" + echo " make lint-deps" echo "or follow https://golangci-lint.run/usage/install/" exit 2 fi diff --git a/eth/backend.go b/eth/backend.go index 227abeaf153..7ecf2fb6de7 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -31,6 +31,7 @@ import ( "sync" "time" + lru "github.com/hashicorp/golang-lru/arc/v2" "github.com/ledgerwatch/erigon-lib/chain/networkname" "github.com/ledgerwatch/erigon-lib/diagnostics" "github.com/ledgerwatch/erigon-lib/downloader/downloadergrpc" @@ -638,11 +639,21 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger backend.pendingBlocks = miner.PendingResultCh backend.minedBlocks = miner.MiningResultCh + var ( + snapDb kv.RwDB + recents *lru.ARCCache[libcommon.Hash, *bor.Snapshot] + signatures *lru.ARCCache[libcommon.Hash, libcommon.Address] + ) + if bor, ok := backend.engine.(*bor.Bor); ok { + snapDb = bor.DB + recents = bor.Recents + signatures = bor.Signatures + } // proof-of-work mining mining := stagedsync.New( stagedsync.MiningStages(backend.sentryCtx, stagedsync.StageMiningCreateBlockCfg(backend.chainDB, miner, *backend.chainConfig, backend.engine, backend.txPoolDB, nil, tmpdir, backend.blockReader), - stagedsync.StageBorHeimdallCfg(backend.chainDB, miner, *backend.chainConfig, heimdallClient, backend.blockReader, nil, nil), + stagedsync.StageBorHeimdallCfg(backend.chainDB, snapDb, miner, *backend.chainConfig, heimdallClient, backend.blockReader, nil, nil, recents, signatures), stagedsync.StageMiningExecCfg(backend.chainDB, miner, backend.notifications.Events, *backend.chainConfig, backend.engine, &vm.Config{}, tmpdir, nil, 0, backend.txPool, backend.txPoolDB, blockReader), stagedsync.StageHashStateCfg(backend.chainDB, dirs, config.HistoryV3), stagedsync.StageTrieCfg(backend.chainDB, false, true, true, tmpdir, blockReader, nil, config.HistoryV3, backend.agg), @@ -662,7 +673,7 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger proposingSync := stagedsync.New( stagedsync.MiningStages(backend.sentryCtx, stagedsync.StageMiningCreateBlockCfg(backend.chainDB, miningStatePos, *backend.chainConfig, backend.engine, backend.txPoolDB, param, tmpdir, backend.blockReader), - stagedsync.StageBorHeimdallCfg(backend.chainDB, miningStatePos, *backend.chainConfig, heimdallClient, backend.blockReader, nil, nil), + stagedsync.StageBorHeimdallCfg(backend.chainDB, snapDb, miningStatePos, *backend.chainConfig, heimdallClient, backend.blockReader, nil, nil, recents, signatures), stagedsync.StageMiningExecCfg(backend.chainDB, miningStatePos, backend.notifications.Events, *backend.chainConfig, backend.engine, &vm.Config{}, tmpdir, interrupt, param.PayloadId, backend.txPool, backend.txPoolDB, blockReader), stagedsync.StageHashStateCfg(backend.chainDB, dirs, config.HistoryV3), stagedsync.StageTrieCfg(backend.chainDB, false, true, true, tmpdir, blockReader, nil, config.HistoryV3, backend.agg), @@ -797,8 +808,8 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger backend.ethBackendRPC, backend.miningRPC, backend.stateChangesClient = ethBackendRPC, miningRPC, stateDiffClient - backend.syncStages = stages2.NewDefaultStages(backend.sentryCtx, backend.chainDB, stack.Config().P2P, config, backend.sentriesClient, backend.notifications, backend.downloaderClient, - blockReader, blockRetire, backend.agg, backend.silkworm, backend.forkValidator, heimdallClient, logger) + backend.syncStages = stages2.NewDefaultStages(backend.sentryCtx, backend.chainDB, snapDb, stack.Config().P2P, config, backend.sentriesClient, backend.notifications, backend.downloaderClient, + blockReader, blockRetire, backend.agg, backend.silkworm, backend.forkValidator, heimdallClient, recents, signatures, logger) backend.syncUnwindOrder = stagedsync.DefaultUnwindOrder backend.syncPruneOrder = stagedsync.DefaultPruneOrder backend.stagedSync = stagedsync.New(backend.syncStages, backend.syncUnwindOrder, backend.syncPruneOrder, logger) diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 9a49831aee6..80b4844eb00 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -74,7 +74,7 @@ var Defaults = Config{ ExecWorkerCount: estimate.ReconstituteState.WorkersHalf(), //only half of CPU, other half will spend for snapshots build/merge/prune ReconWorkerCount: estimate.ReconstituteState.Workers(), BodyCacheLimit: 256 * 1024 * 1024, - BodyDownloadTimeoutSeconds: 30, + BodyDownloadTimeoutSeconds: 2, }, Ethash: ethashcfg.Config{ CachesInMem: 2, diff --git a/eth/stagedsync/chain_reader.go b/eth/stagedsync/chain_reader.go index d86f7c3f2ff..862cae5710a 100644 --- a/eth/stagedsync/chain_reader.go +++ b/eth/stagedsync/chain_reader.go @@ -84,3 +84,7 @@ func (cr ChainReader) FrozenBlocks() uint64 { func (cr ChainReader) BorEventsByBlock(hash libcommon.Hash, number uint64) []rlp.RawValue { panic("") } + +func (cr ChainReader) BorSpan(spanId uint64) []byte { + panic("") +} diff --git a/eth/stagedsync/stage_bor_heimdall.go b/eth/stagedsync/stage_bor_heimdall.go index d6343f0c67b..9a259207c75 100644 --- a/eth/stagedsync/stage_bor_heimdall.go +++ b/eth/stagedsync/stage_bor_heimdall.go @@ -1,37 +1,54 @@ package stagedsync import ( + "bytes" "context" "encoding/binary" "encoding/json" "fmt" "math/big" + "sort" "strconv" "time" + lru "github.com/hashicorp/golang-lru/arc/v2" "github.com/ledgerwatch/erigon-lib/chain" + "github.com/ledgerwatch/erigon-lib/common" + libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon/accounts/abi" + "github.com/ledgerwatch/erigon/consensus" + "github.com/ledgerwatch/erigon/consensus/bor" "github.com/ledgerwatch/erigon/consensus/bor/contract" "github.com/ledgerwatch/erigon/consensus/bor/finality/generics" "github.com/ledgerwatch/erigon/consensus/bor/finality/whitelist" "github.com/ledgerwatch/erigon/consensus/bor/heimdall" + "github.com/ledgerwatch/erigon/consensus/bor/heimdall/span" + "github.com/ledgerwatch/erigon/consensus/bor/valset" "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/dataflow" + "github.com/ledgerwatch/erigon/eth/ethconfig/estimate" "github.com/ledgerwatch/erigon/eth/stagedsync/stages" "github.com/ledgerwatch/erigon/rlp" "github.com/ledgerwatch/erigon/turbo/services" "github.com/ledgerwatch/erigon/turbo/stages/headerdownload" "github.com/ledgerwatch/log/v3" + "golang.org/x/sync/errgroup" ) const ( - spanLength = 6400 // Number of blocks in a span - zerothSpanEnd = 255 // End block of 0th span + spanLength = 6400 // Number of blocks in a span + zerothSpanEnd = 255 // End block of 0th span + inmemorySnapshots = 128 // Number of recent vote snapshots to keep in memory + inmemorySignatures = 4096 // Number of recent block signatures to keep in memory + snapshotPersistInterval = 1024 // Number of blocks after which to persist the vote snapshot to the database + extraVanity = 32 // Fixed number of extra-data prefix bytes reserved for signer vanity + extraSeal = 65 // Fixed number of extra-data suffix bytes reserved for signer seal ) type BorHeimdallCfg struct { db kv.RwDB + snapDb kv.RwDB // Database to store and retrieve snapshot checkpoints miningState MiningState chainConfig chain.Config heimdallClient heimdall.IHeimdallClient @@ -39,19 +56,25 @@ type BorHeimdallCfg struct { hd *headerdownload.HeaderDownload penalize func(context.Context, []headerdownload.PenaltyItem) stateReceiverABI abi.ABI + recents *lru.ARCCache[libcommon.Hash, *bor.Snapshot] + signatures *lru.ARCCache[libcommon.Hash, libcommon.Address] } func StageBorHeimdallCfg( db kv.RwDB, + snapDb kv.RwDB, miningState MiningState, chainConfig chain.Config, heimdallClient heimdall.IHeimdallClient, blockReader services.FullBlockReader, hd *headerdownload.HeaderDownload, penalize func(context.Context, []headerdownload.PenaltyItem), + recents *lru.ARCCache[libcommon.Hash, *bor.Snapshot], + signatures *lru.ARCCache[libcommon.Hash, libcommon.Address], ) BorHeimdallCfg { return BorHeimdallCfg{ db: db, + snapDb: snapDb, miningState: miningState, chainConfig: chainConfig, heimdallClient: heimdallClient, @@ -59,6 +82,8 @@ func StageBorHeimdallCfg( hd: hd, penalize: penalize, stateReceiverABI: contract.StateReceiver(), + recents: recents, + signatures: signatures, } } @@ -159,21 +184,49 @@ func BorHeimdallForward( if k != nil { lastEventId = binary.BigEndian.Uint64(k) } - type LastFrozenEvent interface { + type LastFrozen interface { LastFrozenEventID() uint64 + LastFrozenSpanID() uint64 } - snapshotLastEventId := cfg.blockReader.(LastFrozenEvent).LastFrozenEventID() + snapshotLastEventId := cfg.blockReader.(LastFrozen).LastFrozenEventID() if snapshotLastEventId > lastEventId { lastEventId = snapshotLastEventId } + sCursor, err := tx.Cursor(kv.BorSpans) + if err != nil { + return err + } + defer sCursor.Close() + k, _, err = sCursor.Last() + if err != nil { + return err + } + var nextSpanId uint64 + if k != nil { + nextSpanId = binary.BigEndian.Uint64(k) + 1 + } + snapshotLastSpanId := cfg.blockReader.(LastFrozen).LastFrozenSpanID() + if snapshotLastSpanId+1 > nextSpanId { + nextSpanId = snapshotLastSpanId + 1 + } + var endSpanID uint64 + if headNumber > zerothSpanEnd { + endSpanID = 2 + (headNumber-zerothSpanEnd)/spanLength + } + lastBlockNum := s.BlockNumber if cfg.blockReader.FrozenBorBlocks() > lastBlockNum { lastBlockNum = cfg.blockReader.FrozenBorBlocks() } - - if !mine { - logger.Info("["+s.LogPrefix()+"] Processng sync events...", "from", lastBlockNum+1) + recents, err := lru.NewARC[libcommon.Hash, *bor.Snapshot](inmemorySnapshots) + if err != nil { + return err + } + signatures, err := lru.NewARC[libcommon.Hash, libcommon.Address](inmemorySignatures) + if err != nil { + return err } + chain := NewChainReaderImpl(&cfg.chainConfig, tx, cfg.blockReader, logger) var blockNum uint64 var fetchTime time.Duration @@ -183,6 +236,17 @@ func BorHeimdallForward( logTimer := time.NewTicker(30 * time.Second) defer logTimer.Stop() + if endSpanID >= nextSpanId { + logger.Info("["+s.LogPrefix()+"] Processing spans...", "from", nextSpanId, "to", endSpanID) + } + for spanID := nextSpanId; spanID <= endSpanID; spanID++ { + if lastSpanId, err = fetchAndWriteSpans(ctx, spanID, tx, cfg.heimdallClient, s.LogPrefix(), logger); err != nil { + return err + } + } + if !mine { + logger.Info("["+s.LogPrefix()+"] Processing sync events...", "from", lastBlockNum+1, "to", headNumber) + } for blockNum = lastBlockNum + 1; blockNum <= headNumber; blockNum++ { select { default: @@ -221,9 +285,15 @@ func BorHeimdallForward( fetchTime += callTime } - if blockNum == 1 || (blockNum > zerothSpanEnd && ((blockNum-zerothSpanEnd-1)%spanLength) == 0) { - if lastSpanId, err = fetchAndWriteSpans(ctx, blockNum, tx, cfg.heimdallClient, s.LogPrefix(), logger); err != nil { - return err + if err = PersistValidatorSets(u, ctx, tx, cfg.blockReader, cfg.chainConfig.Bor, chain, blockNum, header.Hash(), recents, signatures, cfg.snapDb, logger); err != nil { + return fmt.Errorf("persistValidatorSets: %w", err) + } + if !mine && header != nil { + sprintLength := cfg.chainConfig.Bor.CalculateSprint(blockNum) + if blockNum > zerothSpanEnd && ((blockNum+1)%sprintLength == 0) { + if err = checkHeaderExtraData(u, ctx, chain, blockNum, header); err != nil { + return err + } } } } @@ -243,6 +313,46 @@ func BorHeimdallForward( return } +func checkHeaderExtraData( + u Unwinder, + ctx context.Context, + chain consensus.ChainHeaderReader, + blockNum uint64, + header *types.Header, +) error { + var spanID uint64 + if blockNum+1 > zerothSpanEnd { + spanID = 1 + (blockNum+1-zerothSpanEnd-1)/spanLength + } + spanBytes := chain.BorSpan(spanID) + var sp span.HeimdallSpan + if err := json.Unmarshal(spanBytes, &sp); err != nil { + return err + } + producerSet := make([]*valset.Validator, len(sp.SelectedProducers)) + for i := range sp.SelectedProducers { + producerSet[i] = &sp.SelectedProducers[i] + } + + sort.Sort(valset.ValidatorsByAddress(producerSet)) + + headerVals, err := valset.ParseValidators(header.Extra[extraVanity : len(header.Extra)-extraSeal]) + if err != nil { + return err + } + + if len(producerSet) != len(headerVals) { + return bor.ErrInvalidSpanValidators + } + + for i, val := range producerSet { + if !bytes.Equal(val.HeaderBytes(), headerVals[i].HeaderBytes()) { + return bor.ErrInvalidSpanValidators + } + } + return nil +} + func fetchAndWriteBorEvents( ctx context.Context, blockReader services.FullBlockReader, @@ -348,17 +458,12 @@ func fetchAndWriteBorEvents( func fetchAndWriteSpans( ctx context.Context, - blockNum uint64, + spanId uint64, tx kv.RwTx, heimdallClient heimdall.IHeimdallClient, logPrefix string, logger log.Logger, ) (uint64, error) { - var spanId uint64 - if blockNum > zerothSpanEnd { - spanId = 1 + (blockNum-zerothSpanEnd-1)/spanLength - } - logger.Debug(fmt.Sprintf("[%s] Fetching span", logPrefix), "id", spanId) response, err := heimdallClient.Span(ctx, spanId) if err != nil { return 0, err @@ -372,9 +477,192 @@ func fetchAndWriteSpans( if err = tx.Put(kv.BorSpans, spanIDBytes[:], spanBytes); err != nil { return 0, err } + logger.Debug(fmt.Sprintf("[%s] Wrote span", logPrefix), "id", spanId) return spanId, nil } +// Not used currently +func PersistValidatorSets( + u Unwinder, + ctx context.Context, + tx kv.Tx, + blockReader services.FullBlockReader, + config *chain.BorConfig, + chain consensus.ChainHeaderReader, + blockNum uint64, + hash libcommon.Hash, + recents *lru.ARCCache[libcommon.Hash, *bor.Snapshot], + signatures *lru.ARCCache[libcommon.Hash, libcommon.Address], + snapDb kv.RwDB, + logger log.Logger) error { + + logEvery := time.NewTicker(logInterval) + defer logEvery.Stop() + // Search for a snapshot in memory or on disk for checkpoints + var snap *bor.Snapshot + + headers := make([]*types.Header, 0, 16) + var parent *types.Header + + if s, ok := recents.Get(hash); ok { + snap = s + } + + //nolint:govet + for snap == nil { + // If an on-disk snapshot can be found, use that + if blockNum%snapshotPersistInterval == 0 { + if s, err := bor.LoadSnapshot(config, signatures, snapDb, hash); err == nil { + logger.Trace("Loaded snapshot from disk", "number", blockNum, "hash", hash) + + snap = s + + break + } + } + + // No snapshot for this header, gather the header and move backward + var header *types.Header + // No explicit parents (or no more left), reach out to the database + if parent != nil { + header = parent + } else if chain != nil { + header = chain.GetHeader(hash, blockNum) + //logger.Info(fmt.Sprintf("header %d %x => %+v\n", header.Number.Uint64(), header.Hash(), header)) + } + + if header == nil { + return consensus.ErrUnknownAncestor + } + + if blockNum == 0 { + break + } + + headers = append(headers, header) + blockNum, hash = blockNum-1, header.ParentHash + if chain != nil { + parent = chain.GetHeader(hash, blockNum) + } + + // If an in-memory snapshot was found, use that + if s, ok := recents.Get(hash); ok { + snap = s + break + } + if chain != nil && blockNum < chain.FrozenBlocks() { + break + } + + select { + case <-logEvery.C: + logger.Info("Gathering headers for validator proposer prorities (backwards)", "blockNum", blockNum) + default: + } + } + if snap == nil && chain != nil && blockNum <= chain.FrozenBlocks() { + // Special handling of the headers in the snapshot + zeroHeader := chain.GetHeaderByNumber(0) + if zeroHeader != nil { + // get checkpoint data + hash := zeroHeader.Hash() + + // get validators and current span + zeroSpanBytes, err := blockReader.Span(ctx, tx, 0) + if err != nil { + return err + } + var zeroSpan span.HeimdallSpan + if err = json.Unmarshal(zeroSpanBytes, &zeroSpan); err != nil { + return err + } + + // new snap shot + snap = bor.NewSnapshot(config, signatures, 0, hash, zeroSpan.ValidatorSet.Validators, logger) + if err := snap.Store(snapDb); err != nil { + return fmt.Errorf("snap.Store (0): %w", err) + } + logger.Info("Stored proposer snapshot to disk", "number", 0, "hash", hash) + g := errgroup.Group{} + g.SetLimit(estimate.AlmostAllCPUs()) + defer g.Wait() + + batchSize := 128 // must be < inmemorySignatures + initialHeaders := make([]*types.Header, 0, batchSize) + parentHeader := zeroHeader + for i := uint64(1); i <= blockNum; i++ { + header := chain.GetHeaderByNumber(i) + { + // `snap.apply` bottleneck - is recover of signer. + // to speedup: recover signer in background goroutines and save in `sigcache` + // `batchSize` < `inmemorySignatures`: means all current batch will fit in cache - and `snap.apply` will find it there. + g.Go(func() error { + _, _ = bor.Ecrecover(header, signatures, config) + return nil + }) + } + initialHeaders = append(initialHeaders, header) + if len(initialHeaders) == cap(initialHeaders) { + if snap, err = snap.Apply(parentHeader, initialHeaders, logger); err != nil { + return fmt.Errorf("snap.Apply (inside loop): %w", err) + } + parentHeader = initialHeaders[len(initialHeaders)-1] + initialHeaders = initialHeaders[:0] + } + select { + case <-logEvery.C: + logger.Info("Computing validator proposer prorities (forward)", "blockNum", i) + default: + } + } + if snap, err = snap.Apply(parentHeader, initialHeaders, logger); err != nil { + return fmt.Errorf("snap.Apply (outside loop): %w", err) + } + } + } + + // check if snapshot is nil + if snap == nil { + return fmt.Errorf("unknown error while retrieving snapshot at block number %v", blockNum) + } + + // Previous snapshot found, apply any pending headers on top of it + for i := 0; i < len(headers)/2; i++ { + headers[i], headers[len(headers)-1-i] = headers[len(headers)-1-i], headers[i] + } + + if len(headers) > 0 { + var err error + if snap, err = snap.Apply(parent, headers, logger); err != nil { + if snap != nil { + var badHash common.Hash + for _, header := range headers { + if header.Number.Uint64() == snap.Number+1 { + badHash = header.Hash() + break + } + } + u.UnwindTo(snap.Number, BadBlock(badHash, err)) + } else { + return fmt.Errorf("snap.Apply %d, headers %d-%d: %w", blockNum, headers[0].Number.Uint64(), headers[len(headers)-1].Number.Uint64(), err) + } + } + } + + recents.Add(snap.Hash, snap) + + // If we've generated a new persistent snapshot, save to disk + if snap.Number%snapshotPersistInterval == 0 && len(headers) > 0 { + if err := snap.Store(snapDb); err != nil { + return fmt.Errorf("snap.Store: %w", err) + } + + logger.Info("Stored proposer snapshot to disk", "number", snap.Number, "hash", snap.Hash) + } + + return nil +} + func BorHeimdallUnwind(u *UnwindState, ctx context.Context, s *StageState, tx kv.RwTx, cfg BorHeimdallCfg) (err error) { if cfg.chainConfig.Bor == nil { return diff --git a/eth/stagedsync/stage_headers.go b/eth/stagedsync/stage_headers.go index 645cca61a92..e42b35a7058 100644 --- a/eth/stagedsync/stage_headers.go +++ b/eth/stagedsync/stage_headers.go @@ -203,6 +203,7 @@ Loop: if req != nil { peer, sentToPeer = cfg.headerReqSend(ctx, req) if sentToPeer { + logger.Debug(fmt.Sprintf("[%s] Requested header", logPrefix), "from", req.Number, "length", req.Length) cfg.hd.UpdateStats(req, false /* skeleton */, peer) cfg.hd.UpdateRetryTime(req, currentTime, 5*time.Second /* timeout */) } @@ -232,6 +233,7 @@ Loop: if req != nil { peer, sentToPeer = cfg.headerReqSend(ctx, req) if sentToPeer { + logger.Debug(fmt.Sprintf("[%s] Requested skeleton", logPrefix), "from", req.Number, "length", req.Length) cfg.hd.UpdateStats(req, true /* skeleton */, peer) lastSkeletonTime = time.Now() } @@ -567,3 +569,11 @@ func (cr ChainReaderImpl) BorEventsByBlock(hash libcommon.Hash, number uint64) [ } return events } +func (cr ChainReaderImpl) BorSpan(spanId uint64) []byte { + span, err := cr.blockReader.Span(context.Background(), cr.tx, spanId) + if err != nil { + cr.logger.Error("BorSpan failed", "err", err) + return nil + } + return span +} diff --git a/eth/stagedsync/stage_snapshots.go b/eth/stagedsync/stage_snapshots.go index 3da1d60ba74..1d36be12488 100644 --- a/eth/stagedsync/stage_snapshots.go +++ b/eth/stagedsync/stage_snapshots.go @@ -40,6 +40,7 @@ type SnapshotsCfg struct { dbEventNotifier services.DBEventNotifier historyV3 bool + caplin bool agg *state.AggregatorV3 silkworm *silkworm.Silkworm } @@ -53,6 +54,7 @@ func StageSnapshotsCfg(db kv.RwDB, dbEventNotifier services.DBEventNotifier, historyV3 bool, agg *state.AggregatorV3, + caplin bool, silkworm *silkworm.Silkworm, ) SnapshotsCfg { return SnapshotsCfg{ @@ -64,6 +66,7 @@ func StageSnapshotsCfg(db kv.RwDB, blockReader: blockReader, dbEventNotifier: dbEventNotifier, historyV3: historyV3, + caplin: caplin, agg: agg, silkworm: silkworm, } @@ -119,8 +122,12 @@ func DownloadAndIndexSnapshotsIfNeed(s *StageState, ctx context.Context, tx kv.R if !cfg.blockReader.FreezingCfg().Enabled { return nil } + cstate := snapshotsync.NoCaplin + // if cfg.caplin { //TODO(Giulio2002): uncomment + // cstate = snapshotsync.AlsoCaplin + // } - if err := snapshotsync.WaitForDownloader(s.LogPrefix(), ctx, cfg.historyV3, snapshotsync.NoCaplin, cfg.agg, tx, cfg.blockReader, cfg.dbEventNotifier, &cfg.chainConfig, cfg.snapshotDownloader); err != nil { + if err := snapshotsync.WaitForDownloader(s.LogPrefix(), ctx, cfg.historyV3, cstate, cfg.agg, tx, cfg.blockReader, cfg.dbEventNotifier, &cfg.chainConfig, cfg.snapshotDownloader); err != nil { return err } diff --git a/go.mod b/go.mod index 57900e44bc4..7564417565f 100644 --- a/go.mod +++ b/go.mod @@ -184,7 +184,7 @@ require ( github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231102060711-19219b948f46 // indirect + github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231106204511-f1e556dd5c50 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect diff --git a/go.sum b/go.sum index 933fa8abc70..1897147ff36 100644 --- a/go.sum +++ b/go.sum @@ -539,8 +539,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231102060711-19219b948f46 h1:yt3/AcefMQOzY/P05jyeaKpqMQvrCbL6OJWALsjKp5U= -github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231102060711-19219b948f46/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= +github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231106204511-f1e556dd5c50 h1:RECb+fAC9doD1EhVxK2/b20JeLCAumLDjnysSQ3kWfs= +github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231106204511-f1e556dd5c50/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.9.0 h1:iDwrXe0PVwBC68Dd94YSsHbMgQ3ufsgjzXtFNFVZFRk= github.com/ledgerwatch/log/v3 v3.9.0/go.mod h1:EiAY6upmI/6LkNhOVxb4eVsmsP11HZCnZ3PlJMjYiqE= github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ= diff --git a/p2p/dial.go b/p2p/dial.go index cadb821d5ef..8bb3934ebe4 100644 --- a/p2p/dial.go +++ b/p2p/dial.go @@ -43,7 +43,6 @@ const ( // Config for the "Looking for peers" message. dialStatsLogInterval = 60 * time.Second // printed at most this often - dialStatsPeerLimit = 20 // but not if more than this many dialed peers // Endpoint resolution is throttled with bounded backoff. initialResolveDelay = 60 * time.Second @@ -94,6 +93,7 @@ var ( // to create peer connections to nodes arriving through the iterator. type dialScheduler struct { dialConfig + mutex sync.Mutex setupFunc dialSetupFunc wg sync.WaitGroup cancel context.CancelFunc @@ -126,8 +126,8 @@ type dialScheduler struct { historyTimerTime mclock.AbsTime // for logStats - lastStatsLog mclock.AbsTime - doneSinceLastLog int + dialed int + errors map[string]uint } type dialSetupFunc func(net.Conn, connFlag, *enode.Node) error @@ -177,8 +177,9 @@ func newDialScheduler(config dialConfig, it enode.Iterator, setupFunc dialSetupF remPeerCh: make(chan *conn), subProtocolVersion: subProtocolVersion, + errors: map[string]uint{}, } - d.lastStatsLog = d.clock.Now() + d.ctx, d.cancel = context.WithCancel(context.Background()) d.wg.Add(2) go d.readNodes(it) @@ -232,6 +233,9 @@ func (d *dialScheduler) loop(it enode.Iterator) { historyExp = make(chan struct{}, 1) ) + logTimer := time.NewTicker(dialStatsLogInterval) + defer logTimer.Stop() + loop: for { // Launch new dials if slots are available. @@ -243,13 +247,15 @@ loop: nodesCh = nil } d.rearmHistoryTimer(historyExp) - //d.logStats() select { case <-d.ctx.Done(): it.Close() break loop + case <-logTimer.C: + d.logStats() + case node := <-nodesCh: if err := d.checkDial(node); err != nil { d.log.Trace("Discarding dial candidate", "id", node.ID(), "ip", node.IP(), "reason", err) @@ -261,7 +267,7 @@ loop: id := task.dest.ID() delete(d.dialing, id) d.updateStaticPool(id) - d.doneSinceLastLog++ + d.dialed++ case c := <-d.addPeerCh: if c.is(dynDialedConn) || c.is(staticDialedConn) { @@ -337,15 +343,16 @@ func (d *dialScheduler) readNodes(it enode.Iterator) { // or comes back online. // nolint func (d *dialScheduler) logStats() { - now := d.clock.Now() - if d.lastStatsLog.Add(dialStatsLogInterval) > now { - return - } - if d.dialPeers < dialStatsPeerLimit && d.dialPeers < d.maxDialPeers { - d.log.Info("[p2p] Looking for peers", "protocol", d.subProtocolVersion, "peers", fmt.Sprintf("%d/%d", len(d.peers), d.maxDialPeers), "tried", d.doneSinceLastLog, "static", len(d.static)) + vals := []interface{}{"protocol", d.subProtocolVersion, + "peers", fmt.Sprintf("%d/%d", len(d.peers), d.maxDialPeers), "tried", d.dialed, "static", len(d.static)} + + d.mutex.Lock() + for err, count := range d.errors { + vals = append(vals, err, count) } - d.doneSinceLastLog = 0 - d.lastStatsLog = now + d.mutex.Unlock() + + d.log.Debug("[p2p] Dial scheduler", vals...) } // rearmHistoryTimer configures d.historyTimer to fire when the @@ -543,7 +550,12 @@ func (t *dialTask) resolve(d *dialScheduler) bool { func (t *dialTask) dial(d *dialScheduler, dest *enode.Node) error { fd, err := d.dialer.Dial(d.ctx, t.dest) if err != nil { - d.log.Trace("Dial error", "id", t.dest.ID(), "addr", nodeAddr(t.dest), "conn", t.flags, "err", cleanupDialErr(err)) + cleanErr := cleanupDialErr(err) + d.log.Trace("Dial error", "id", t.dest.ID(), "addr", nodeAddr(t.dest), "conn", t.flags, "err", cleanErr) + + d.mutex.Lock() + d.errors[cleanErr.Error()] = d.errors[cleanErr.Error()] + 1 + d.mutex.Unlock() return &dialError{err} } mfd := newMeteredConn(fd, false, &net.TCPAddr{IP: dest.IP(), Port: dest.TCP()}) diff --git a/p2p/discover/common.go b/p2p/discover/common.go index 6ee5c4c0bd1..da45e7b6d0d 100644 --- a/p2p/discover/common.go +++ b/p2p/discover/common.go @@ -86,8 +86,8 @@ func (cfg Config) withDefaults(defaultReplyTimeout time.Duration) Config { } // ListenUDP starts listening for discovery packets on the given UDP socket. -func ListenUDP(ctx context.Context, c UDPConn, ln *enode.LocalNode, cfg Config) (*UDPv4, error) { - return ListenV4(ctx, c, ln, cfg) +func ListenUDP(ctx context.Context, protocol string, c UDPConn, ln *enode.LocalNode, cfg Config) (*UDPv4, error) { + return ListenV4(ctx, protocol, c, ln, cfg) } // ReadPacket is a packet that couldn't be handled. Those packets are sent to the unhandled @@ -96,3 +96,8 @@ type ReadPacket struct { Data []byte Addr *net.UDPAddr } + +type UnhandledPacket struct { + ReadPacket + Reason error +} diff --git a/p2p/discover/lookup.go b/p2p/discover/lookup.go index 0e03daa30f2..87ba2c2d55e 100644 --- a/p2p/discover/lookup.go +++ b/p2p/discover/lookup.go @@ -155,6 +155,7 @@ func (it *lookup) slowdown() { func (it *lookup) query(n *node, reply chan<- []*node) { fails := it.tab.db.FindFails(n.ID(), n.IP()) r, err := it.queryfunc(n) + if err == errClosed { // Avoid recording failures on shutdown. reply <- nil @@ -180,6 +181,7 @@ func (it *lookup) query(n *node, reply chan<- []*node) { for _, n := range r { it.tab.addSeenNode(n) } + reply <- r } diff --git a/p2p/discover/table.go b/p2p/discover/table.go index eaa79403447..feaf5d39788 100644 --- a/p2p/discover/table.go +++ b/p2p/discover/table.go @@ -55,12 +55,13 @@ const ( bucketIPLimit, bucketSubnet = 2, 24 // at most 2 addresses from the same /24 tableIPLimit, tableSubnet = 10, 24 - refreshInterval = 30 * time.Minute - revalidateInterval = 5 * time.Second - copyNodesInterval = 30 * time.Second - seedMinTableTime = 5 * time.Minute - seedCount = 30 - seedMaxAge = 5 * 24 * time.Hour + minRefreshInterval = 30 * time.Second + refreshInterval = 30 * time.Minute + revalidateInterval = 5 * time.Second + maintenanceInterval = 60 * time.Second + seedMinTableTime = 5 * time.Minute + seedCount = 30 + seedMaxAge = 5 * 24 * time.Hour ) // Table is the 'node table', a Kademlia-like index of neighbor nodes. The table keeps @@ -84,6 +85,12 @@ type Table struct { closed chan struct{} nodeAddedHook func(*node) // for testing + + // diagnostics + errors map[string]uint + dbseeds int + revalidates int + protocol string } // transport is implemented by the UDP transports. @@ -93,6 +100,9 @@ type transport interface { lookupRandom() []*enode.Node lookupSelf() []*enode.Node ping(*enode.Node) (seq uint64, err error) + Version() string + Errors() map[string]uint + LenUnsolicited() int } // bucket contains nodes, ordered by their last activity. the entry @@ -105,24 +115,25 @@ type bucket struct { func newTable( t transport, + protocol string, db *enode.DB, bootnodes []*enode.Node, revalidateInterval time.Duration, logger log.Logger, ) (*Table, error) { tab := &Table{ - net: t, - db: db, - refreshReq: make(chan chan struct{}), - initDone: make(chan struct{}), - closeReq: make(chan struct{}), - closed: make(chan struct{}), - rand: mrand.New(mrand.NewSource(0)), // nolint: gosec - ips: netutil.DistinctNetSet{Subnet: tableSubnet, Limit: tableIPLimit}, - + net: t, + db: db, + refreshReq: make(chan chan struct{}), + initDone: make(chan struct{}), + closeReq: make(chan struct{}), + closed: make(chan struct{}), + rand: mrand.New(mrand.NewSource(0)), // nolint: gosec + ips: netutil.DistinctNetSet{Subnet: tableSubnet, Limit: tableIPLimit}, + errors: map[string]uint{}, revalidateInterval: revalidateInterval, - - log: logger, + protocol: protocol, + log: logger, } if err := tab.setFallbackNodes(bootnodes); err != nil { return nil, err @@ -147,8 +158,8 @@ func (tab *Table) seedRand() { crand.Read(b[:]) tab.mutex.Lock() + defer tab.mutex.Unlock() tab.rand.Seed(int64(binary.BigEndian.Uint64(b[:]))) - tab.mutex.Unlock() } // ReadRandomNodes fills the given slice with random nodes from the table. The results @@ -157,6 +168,7 @@ func (tab *Table) ReadRandomNodes(buf []*enode.Node) (n int) { if !tab.isInitDone() { return 0 } + tab.mutex.Lock() defer tab.mutex.Unlock() @@ -230,21 +242,29 @@ func (tab *Table) refresh() <-chan struct{} { // loop schedules runs of doRefresh, doRevalidate and copyLiveNodes. func (tab *Table) loop() { var ( - revalidate = time.NewTimer(tab.revalidateInterval) - refresh = time.NewTicker(refreshInterval) - copyNodes = time.NewTicker(copyNodesInterval) - refreshDone = make(chan struct{}) // where doRefresh reports completion - revalidateDone chan struct{} // where doRevalidate reports completion - waiting = []chan struct{}{tab.initDone} // holds waiting callers while doRefresh runs + revalidate = time.NewTimer(tab.revalidateInterval) + refresh = time.NewTicker(refreshInterval) + tableMainenance = time.NewTicker(maintenanceInterval) + refreshDone = make(chan struct{}) // where doRefresh reports completion + revalidateDone chan struct{} // where doRevalidate reports completion + waiting = []chan struct{}{tab.initDone} // holds waiting callers while doRefresh runs ) defer debug.LogPanic() defer refresh.Stop() defer revalidate.Stop() - defer copyNodes.Stop() + defer tableMainenance.Stop() // Start initial refresh. go tab.doRefresh(refreshDone) + var minRefreshTimer *time.Timer + + defer func() { + if minRefreshTimer != nil { + minRefreshTimer.Stop() + } + }() + loop: for { select { @@ -266,13 +286,49 @@ loop: } waiting, refreshDone = nil, nil case <-revalidate.C: - revalidateDone = make(chan struct{}) - go tab.doRevalidate(revalidateDone) + if revalidateDone == nil { + revalidateDone = make(chan struct{}) + go tab.doRevalidate(revalidateDone) + } case <-revalidateDone: revalidate.Reset(tab.revalidateInterval) + if tab.live() == 0 && len(waiting) == 0 && minRefreshTimer == nil { + minRefreshTimer = time.AfterFunc(minRefreshInterval, func() { + minRefreshTimer = nil + tab.net.lookupRandom() + tab.refresh() + }) + } revalidateDone = nil - case <-copyNodes.C: - go tab.copyLiveNodes() + case <-tableMainenance.C: + live := tab.live() + + vals := []interface{}{"protocol", tab.protocol, "version", tab.net.Version(), + "len", tab.len(), "live", tab.live(), "unsol", tab.net.LenUnsolicited(), "ips", tab.ips.Len(), "db", tab.dbseeds, "reval", tab.revalidates} + + func() { + tab.mutex.Lock() + defer tab.mutex.Unlock() + + for err, count := range tab.errors { + vals = append(vals, err, count) + } + + for err, count := range tab.net.Errors() { + vals = append(vals, err, count) + } + }() + + tab.log.Debug("[p2p] Discovery table", vals...) + + if live != 0 { + if revalidateDone == nil { + revalidateDone = make(chan struct{}) + go tab.doRevalidate(revalidateDone) + } + } else { + go tab.copyLiveNodes() + } case <-tab.closeReq: break loop } @@ -316,7 +372,10 @@ func (tab *Table) doRefresh(done chan struct{}) { } func (tab *Table) loadSeedNodes() { - seeds := wrapNodes(tab.db.QuerySeeds(seedCount, seedMaxAge)) + dbseeds := tab.db.QuerySeeds(seedCount, seedMaxAge) + tab.dbseeds = len(dbseeds) + + seeds := wrapNodes(dbseeds) tab.log.Debug("QuerySeeds read nodes from the node DB", "count", len(seeds)) seeds = append(seeds, tab.nursery...) for i := range seeds { @@ -333,6 +392,8 @@ func (tab *Table) doRevalidate(done chan<- struct{}) { defer debug.LogPanic() defer func() { done <- struct{}{} }() + tab.revalidates++ + last, bi := tab.nodeToRevalidate() if last == nil { // No non-empty bucket found. @@ -343,11 +404,14 @@ func (tab *Table) doRevalidate(done chan<- struct{}) { remoteSeq, rErr := tab.net.ping(unwrapNode(last)) // Also fetch record if the node replied and returned a higher sequence number. - if last.Seq() < remoteSeq { - if n, err := tab.net.RequestENR(unwrapNode(last)); err != nil { - tab.log.Trace("ENR request failed", "id", last.ID(), "addr", last.addr(), "err", err) - } else { - last = &node{Node: *n, addedAt: last.addedAt, livenessChecks: last.livenessChecks} + if rErr == nil { + if last.Seq() < remoteSeq { + if n, err := tab.net.RequestENR(unwrapNode(last)); err != nil { + rErr = err + tab.log.Trace("ENR request failed", "id", last.ID(), "addr", last.addr(), "err", err) + } else { + last = &node{Node: *n, addedAt: last.addedAt, livenessChecks: last.livenessChecks} + } } } @@ -360,7 +424,10 @@ func (tab *Table) doRevalidate(done chan<- struct{}) { tab.log.Trace("Revalidated node", "b", bi, "id", last.ID(), "checks", last.livenessChecks) tab.bumpInBucket(b, last) return + } else { + tab.addError(rErr) } + // No reply received, pick a replacement or delete the node if there aren't // any replacements. if r := tab.replace(b, last); r != nil { @@ -444,6 +511,26 @@ func (tab *Table) len() (n int) { return n } +func (tab *Table) live() (n int) { + tab.mutex.Lock() + defer tab.mutex.Unlock() + + for _, b := range &tab.buckets { + for _, e := range b.entries { + if e.livenessChecks > 0 { + n++ + } + } + } + + return n +} + +func (tab *Table) addError(err error) { + str := err.Error() + tab.errors[str] = tab.errors[str] + 1 +} + // bucketLen returns the number of nodes in the bucket for the given ID. func (tab *Table) bucketLen(id enode.ID) int { tab.mutex.Lock() @@ -477,6 +564,7 @@ func (tab *Table) addSeenNode(n *node) { tab.mutex.Lock() defer tab.mutex.Unlock() + b := tab.bucket(n.ID()) if contains(b.entries, n.ID()) { // Already in bucket, don't add. @@ -519,6 +607,7 @@ func (tab *Table) addVerifiedNode(n *node) { tab.mutex.Lock() defer tab.mutex.Unlock() + b := tab.bucket(n.ID()) if tab.bumpInBucket(b, n) { // Already in bucket, moved to front. diff --git a/p2p/discover/table_util_test.go b/p2p/discover/table_util_test.go index 50cff8aebe4..e4613192884 100644 --- a/p2p/discover/table_util_test.go +++ b/p2p/discover/table_util_test.go @@ -48,7 +48,7 @@ func newTestTable(t transport, tmpDir string) (*Table, *enode.DB) { if err != nil { panic(err) } - tab, _ := newTable(t, db, nil, time.Hour, log.Root()) + tab, _ := newTable(t, "test", db, nil, time.Hour, log.Root()) go tab.loop() return tab, db } @@ -156,6 +156,9 @@ func (t *pingRecorder) updateRecord(n *enode.Node) { // Stubs to satisfy the transport interface. func (t *pingRecorder) Self() *enode.Node { return nullNode } +func (t *pingRecorder) Version() string { return "none" } +func (t *pingRecorder) Errors() map[string]uint { return nil } +func (t *pingRecorder) LenUnsolicited() int { return 0 } func (t *pingRecorder) lookupSelf() []*enode.Node { return nil } func (t *pingRecorder) lookupRandom() []*enode.Node { return nil } diff --git a/p2p/discover/v4_udp.go b/p2p/discover/v4_udp.go index 38687292d3f..9d962df0414 100644 --- a/p2p/discover/v4_udp.go +++ b/p2p/discover/v4_udp.go @@ -28,6 +28,7 @@ import ( "sync" "time" + lru "github.com/hashicorp/golang-lru/v2" "github.com/ledgerwatch/erigon/common/debug" "github.com/ledgerwatch/erigon/crypto" "github.com/ledgerwatch/erigon/p2p/discover/v4wire" @@ -47,8 +48,14 @@ var ( errLowPort = errors.New("low port") ) +var ( + errExpiredStr = errExpired.Error() + errUnsolicitedReplyStr = errUnsolicitedReply.Error() + errUnknownNodeStr = errUnknownNode.Error() +) + const ( - respTimeout = 500 * time.Millisecond + respTimeout = 750 * time.Millisecond expiration = 20 * time.Second bondExpiration = 24 * time.Hour @@ -65,6 +72,7 @@ const ( // UDPv4 implements the v4 wire protocol. type UDPv4 struct { + mutex sync.Mutex conn UDPConn log log.Logger netrestrict *netutil.Netlist @@ -75,13 +83,16 @@ type UDPv4 struct { closeOnce sync.Once wg sync.WaitGroup - addReplyMatcher chan *replyMatcher - gotreply chan reply - replyTimeout time.Duration - pingBackDelay time.Duration - closeCtx context.Context - cancelCloseCtx context.CancelFunc - + addReplyMatcher chan *replyMatcher + gotreply chan reply + gotkey chan v4wire.Pubkey + gotnodes chan nodes + replyTimeout time.Duration + pingBackDelay time.Duration + closeCtx context.Context + cancelCloseCtx context.CancelFunc + errors map[string]uint + unsolicitedNodes *lru.Cache[enode.ID, *enode.Node] privateKeyGenerator func() (*ecdsa.PrivateKey, error) } @@ -98,6 +109,7 @@ type replyMatcher struct { // these fields must match in the reply. from enode.ID ip net.IP + port int ptype byte // time when the request must complete @@ -124,33 +136,44 @@ type replyMatchFunc func(v4wire.Packet) (matched bool, requestDone bool) type reply struct { from enode.ID ip net.IP + port int data v4wire.Packet // loop indicates whether there was // a matching request by sending on this channel. matched chan<- bool } -func ListenV4(ctx context.Context, c UDPConn, ln *enode.LocalNode, cfg Config) (*UDPv4, error) { +type nodes struct { + addr *net.UDPAddr + nodes []v4wire.Node +} + +func ListenV4(ctx context.Context, protocol string, c UDPConn, ln *enode.LocalNode, cfg Config) (*UDPv4, error) { cfg = cfg.withDefaults(respTimeout) closeCtx, cancel := context.WithCancel(ctx) - t := &UDPv4{ - conn: c, - priv: cfg.PrivateKey, - netrestrict: cfg.NetRestrict, - localNode: ln, - db: ln.Database(), - gotreply: make(chan reply), - addReplyMatcher: make(chan *replyMatcher), - replyTimeout: cfg.ReplyTimeout, - pingBackDelay: cfg.PingBackDelay, - closeCtx: closeCtx, - cancelCloseCtx: cancel, - log: cfg.Log, + unsolicitedNodes, _ := lru.New[enode.ID, *enode.Node](500) + t := &UDPv4{ + conn: c, + priv: cfg.PrivateKey, + netrestrict: cfg.NetRestrict, + localNode: ln, + db: ln.Database(), + gotreply: make(chan reply, 10), + addReplyMatcher: make(chan *replyMatcher, 10), + gotkey: make(chan v4wire.Pubkey, 10), + gotnodes: make(chan nodes, 10), + replyTimeout: cfg.ReplyTimeout, + pingBackDelay: cfg.PingBackDelay, + closeCtx: closeCtx, + cancelCloseCtx: cancel, + log: cfg.Log, + errors: map[string]uint{}, + unsolicitedNodes: unsolicitedNodes, privateKeyGenerator: cfg.PrivateKeyGenerator, } - tab, err := newTable(t, ln.Database(), cfg.Bootnodes, cfg.TableRevalidateInterval, cfg.Log) + tab, err := newTable(t, protocol, ln.Database(), cfg.Bootnodes, cfg.TableRevalidateInterval, cfg.Log) if err != nil { return nil, err } @@ -168,6 +191,28 @@ func (t *UDPv4) Self() *enode.Node { return t.localNode.Node() } +func (t *UDPv4) Version() string { + return "v4" +} + +func (t *UDPv4) Errors() map[string]uint { + errors := map[string]uint{} + + t.mutex.Lock() + for key, value := range t.errors { + errors[key] = value + } + t.mutex.Unlock() + + return errors +} + +func (t *UDPv4) LenUnsolicited() int { + t.mutex.Lock() + defer t.mutex.Unlock() + return t.unsolicitedNodes.Len() +} + // Close shuts down the socket and aborts any running queries. func (t *UDPv4) Close() { t.closeOnce.Do(func() { @@ -241,7 +286,7 @@ func (t *UDPv4) sendPing(toid enode.ID, toaddr *net.UDPAddr, callback func()) *r } // Add a matcher for the reply to the pending reply queue. Pongs are matched if they // reference the ping we're about to send. - rm := t.pending(toid, toaddr.IP, v4wire.PongPacket, func(p v4wire.Packet) (matched bool, requestDone bool) { + rm := t.pending(toid, toaddr.IP, toaddr.Port, v4wire.PongPacket, func(p v4wire.Packet) (matched bool, requestDone bool) { matched = bytes.Equal(p.(*v4wire.Pong).ReplyTok, hash) if matched && callback != nil { callback() @@ -301,6 +346,7 @@ func (t *UDPv4) newRandomLookup(ctx context.Context) *lookup { func (t *UDPv4) newLookup(ctx context.Context, targetKey *ecdsa.PublicKey) *lookup { targetKeyEnc := v4wire.EncodePubkey(targetKey) target := enode.PubkeyEncoded(targetKeyEnc).ID() + it := newLookup(ctx, t.tab, target, func(n *node) ([]*node, error) { return t.findnode(n.ID(), n.addr(), targetKeyEnc) }) @@ -322,7 +368,7 @@ func (t *UDPv4) findnode(toid enode.ID, toaddr *net.UDPAddr, target v4wire.Pubke // active until enough nodes have been received. nodes := make([]*node, 0, bucketSize) nreceived := 0 - rm := t.pending(toid, toaddr.IP, v4wire.NeighborsPacket, func(r v4wire.Packet) (matched bool, requestDone bool) { + rm := t.pending(toid, toaddr.IP, toaddr.Port, v4wire.NeighborsPacket, func(r v4wire.Packet) (matched bool, requestDone bool) { reply := r.(*v4wire.Neighbors) for _, rn := range reply.Nodes { nreceived++ @@ -374,7 +420,7 @@ func (t *UDPv4) RequestENR(n *enode.Node) (*enode.Node, error) { // Add a matcher for the reply to the pending reply queue. Responses are matched if // they reference the request we're about to send. - rm := t.pending(n.ID(), addr.IP, v4wire.ENRResponsePacket, func(r v4wire.Packet) (matched bool, requestDone bool) { + rm := t.pending(n.ID(), addr.IP, addr.Port, v4wire.ENRResponsePacket, func(r v4wire.Packet) (matched bool, requestDone bool) { matched = bytes.Equal(r.(*v4wire.ENRResponse).ReplyTok, hash) return matched, matched }) @@ -406,9 +452,10 @@ func (t *UDPv4) RequestENR(n *enode.Node) (*enode.Node, error) { // pending adds a reply matcher to the pending reply queue. // see the documentation of type replyMatcher for a detailed explanation. -func (t *UDPv4) pending(id enode.ID, ip net.IP, ptype byte, callback replyMatchFunc) *replyMatcher { +func (t *UDPv4) pending(id enode.ID, ip net.IP, port int, ptype byte, callback replyMatchFunc) *replyMatcher { ch := make(chan error, 1) - p := &replyMatcher{from: id, ip: ip, ptype: ptype, callback: callback, errc: ch} + p := &replyMatcher{from: id, ip: ip, port: port, ptype: ptype, callback: callback, errc: ch} + select { case t.addReplyMatcher <- p: // loop will handle it @@ -420,10 +467,10 @@ func (t *UDPv4) pending(id enode.ID, ip net.IP, ptype byte, callback replyMatchF // handleReply dispatches a reply packet, invoking reply matchers. It returns // whether any matcher considered the packet acceptable. -func (t *UDPv4) handleReply(from enode.ID, fromIP net.IP, req v4wire.Packet) bool { +func (t *UDPv4) handleReply(from enode.ID, fromIP net.IP, port int, req v4wire.Packet) bool { matched := make(chan bool, 1) select { - case t.gotreply <- reply{from, fromIP, req, matched}: + case t.gotreply <- reply{from, fromIP, port, req, matched}: // loop will handle it return <-matched case <-t.closeCtx.Done(): @@ -439,89 +486,208 @@ func (t *UDPv4) loop() { var ( plist = list.New() - timeout = time.NewTimer(0) - nextTimeout *replyMatcher // head of plist when timeout was last reset - contTimeouts = 0 // number of continuous timeouts to do NTP checks + mutex = sync.Mutex{} + contTimeouts = 0 // number of continuous timeouts to do NTP checks ntpWarnTime = time.Unix(0, 0) ) - <-timeout.C // ignore first timeout - defer timeout.Stop() - resetTimeout := func() { - if plist.Front() == nil || nextTimeout == plist.Front().Value { - return + listUpdate := make(chan *list.Element, 10) + + go func() { + var ( + timeout = time.NewTimer(0) + nextTimeout *replyMatcher // head of plist when timeout was last reset + ) + + <-timeout.C // ignore first timeout + defer timeout.Stop() + + resetTimeout := func() { + mutex.Lock() + defer mutex.Unlock() + + if plist.Front() == nil || nextTimeout == plist.Front().Value { + return + } + + // Start the timer so it fires when the next pending reply has expired. + now := time.Now() + for el := plist.Front(); el != nil; el = el.Next() { + nextTimeout = el.Value.(*replyMatcher) + if dist := nextTimeout.deadline.Sub(now); dist < 2*t.replyTimeout { + timeout.Reset(dist) + return + } + // Remove pending replies whose deadline is too far in the + // future. These can occur if the system clock jumped + // backwards after the deadline was assigned. + nextTimeout.errc <- errClockWarp + plist.Remove(el) + } + + nextTimeout = nil + timeout.Stop() } - // Start the timer so it fires when the next pending reply has expired. - now := time.Now() - for el := plist.Front(); el != nil; el = el.Next() { - nextTimeout = el.Value.(*replyMatcher) - if dist := nextTimeout.deadline.Sub(now); dist < 2*t.replyTimeout { - timeout.Reset(dist) + + for { + select { + case <-t.closeCtx.Done(): return + + case now := <-timeout.C: + func() { + mutex.Lock() + defer mutex.Unlock() + + nextTimeout = nil + // Notify and remove callbacks whose deadline is in the past. + for el := plist.Front(); el != nil; el = el.Next() { + p := el.Value.(*replyMatcher) + if !now.Before(p.deadline) { + p.errc <- errTimeout + plist.Remove(el) + contTimeouts++ + } + } + // If we've accumulated too many timeouts, do an NTP time sync check + if contTimeouts > ntpFailureThreshold { + if time.Since(ntpWarnTime) >= ntpWarningCooldown { + ntpWarnTime = time.Now() + go checkClockDrift() + } + contTimeouts = 0 + } + }() + + resetTimeout() + + case el := <-listUpdate: + if el == nil { + return + } + + resetTimeout() } - // Remove pending replies whose deadline is too far in the - // future. These can occur if the system clock jumped - // backwards after the deadline was assigned. - nextTimeout.errc <- errClockWarp - plist.Remove(el) } - nextTimeout = nil - timeout.Stop() - } + }() for { - resetTimeout() - select { case <-t.closeCtx.Done(): - for el := plist.Front(); el != nil; el = el.Next() { - el.Value.(*replyMatcher).errc <- errClosed - } + listUpdate <- nil + func() { + mutex.Lock() + defer mutex.Unlock() + for el := plist.Front(); el != nil; el = el.Next() { + el.Value.(*replyMatcher).errc <- errClosed + } + }() return case p := <-t.addReplyMatcher: - p.deadline = time.Now().Add(t.replyTimeout) - plist.PushBack(p) + func() { + mutex.Lock() + defer mutex.Unlock() + p.deadline = time.Now().Add(t.replyTimeout) + listUpdate <- plist.PushBack(p) + }() case r := <-t.gotreply: - var matched bool // whether any replyMatcher considered the reply acceptable. + + type matchCandidate struct { + el *list.Element + errc chan error + } + + var matchCandidates []matchCandidate + + mutex.Lock() for el := plist.Front(); el != nil; el = el.Next() { p := el.Value.(*replyMatcher) if p.from == r.from && p.ptype == r.data.Kind() && p.ip.Equal(r.ip) { + candidate := matchCandidate{el, p.errc} + p.errc = make(chan error, 1) + matchCandidates = append(matchCandidates, candidate) + } + } + mutex.Unlock() + + if len(matchCandidates) == 0 { + // if there are no matched candidates try again matching against + // ip & port to handle node key changes + mutex.Lock() + for el := plist.Front(); el != nil; el = el.Next() { + p := el.Value.(*replyMatcher) + if p.ptype == r.data.Kind() && p.ip.Equal(r.ip) && p.port == r.port { + candidate := matchCandidate{el, p.errc} + p.errc = make(chan error, 1) + matchCandidates = append(matchCandidates, candidate) + } + } + mutex.Unlock() + + if len(matchCandidates) == 0 { + r.matched <- false + } + } + + go func(r reply) { + var matched bool // whether any replyMatcher considered the reply acceptable. + for _, candidate := range matchCandidates { + p := candidate.el.Value.(*replyMatcher) ok, requestDone := p.callback(r.data) matched = matched || ok p.reply = r.data + // Remove the matcher if callback indicates that all replies have been received. if requestDone { - p.errc <- nil - plist.Remove(el) + mutex.Lock() + plist.Remove(candidate.el) + mutex.Unlock() + candidate.errc <- nil + listUpdate <- candidate.el + } else { + select { + case err := <-p.errc: + candidate.errc <- err + default: + p.errc = candidate.errc + } } - // Reset the continuous timeout counter (time drift detection) - contTimeouts = 0 } - } - r.matched <- matched - case now := <-timeout.C: - nextTimeout = nil + r.matched <- matched + }(r) - // Notify and remove callbacks whose deadline is in the past. - for el := plist.Front(); el != nil; el = el.Next() { - p := el.Value.(*replyMatcher) - if now.After(p.deadline) || now.Equal(p.deadline) { - p.errc <- errTimeout - plist.Remove(el) - contTimeouts++ + // Reset the continuous timeout counter (time drift detection) + contTimeouts = 0 + case key := <-t.gotkey: + go func() { + if key, err := v4wire.DecodePubkey(crypto.S256(), key); err == nil { + nodes := t.LookupPubkey(key) + mutex.Lock() + defer mutex.Unlock() + + for _, n := range nodes { + t.unsolicitedNodes.Add(n.ID(), n) + } } - } - // If we've accumulated too many timeouts, do an NTP time sync check - if contTimeouts > ntpFailureThreshold { - if time.Since(ntpWarnTime) >= ntpWarningCooldown { - ntpWarnTime = time.Now() - go checkClockDrift() + }() + + case nodes := <-t.gotnodes: + + func() { + mutex.Lock() + defer mutex.Unlock() + for _, rn := range nodes.nodes { + n, err := t.nodeFromRPC(nodes.addr, rn) + if err != nil { + t.log.Trace("Invalid neighbor node received", "ip", rn.IP, "addr", nodes.addr, "err", err) + continue + } + t.unsolicitedNodes.Add(n.ID(), &n.Node) } - contTimeouts = 0 - } + }() } } } @@ -545,10 +711,13 @@ func (t *UDPv4) write(toaddr *net.UDPAddr, toid enode.ID, what string, packet [] func (t *UDPv4) readLoop(unhandled chan<- ReadPacket) { defer t.wg.Done() defer debug.LogPanic() + if unhandled != nil { defer close(unhandled) } + unknownKeys, _ := lru.New[v4wire.Pubkey, any](100) + buf := make([]byte, maxPacketSize) for { nbytes, from, err := t.conn.ReadFromUDP(buf) @@ -563,11 +732,35 @@ func (t *UDPv4) readLoop(unhandled chan<- ReadPacket) { } return } - if t.handlePacket(from, buf[:nbytes]) != nil && unhandled != nil { - select { - case unhandled <- ReadPacket{buf[:nbytes], from}: - default: - } + if err := t.handlePacket(from, buf[:nbytes]); err != nil { + func() { + switch { + case errors.Is(err, errUnsolicitedReply): + if packet, fromKey, _, err := v4wire.Decode(buf[:nbytes]); err == nil { + switch packet.Kind() { + case v4wire.PongPacket: + if _, ok := unknownKeys.Get(fromKey); !ok { + fromId := enode.PubkeyEncoded(fromKey).ID() + t.log.Trace("Unsolicited packet", "type", packet.Name(), "from", fromId, "addr", from) + unknownKeys.Add(fromKey, nil) + t.gotkey <- fromKey + } + case v4wire.NeighborsPacket: + neighbors := packet.(*v4wire.Neighbors) + t.gotnodes <- nodes{from, neighbors.Nodes} + default: + fromId := enode.PubkeyEncoded(fromKey).ID() + t.log.Trace("Unsolicited packet", "type", packet.Name(), "from", fromId, "addr", from) + } + } else { + t.log.Trace("Unsolicited packet handling failed", "addr", from, "err", err) + } + default: + if unhandled != nil { + unhandled <- ReadPacket{buf[:nbytes], from} + } + } + }() } } } @@ -580,6 +773,7 @@ func (t *UDPv4) handlePacket(from *net.UDPAddr, buf []byte) error { } packet := t.wrapPacket(rawpacket) fromID := enode.PubkeyEncoded(fromKey).ID() + if packet.preverify != nil { err = packet.preverify(packet, from, fromID, fromKey) } @@ -677,9 +871,15 @@ func (t *UDPv4) verifyPing(h *packetHandlerV4, from *net.UDPAddr, fromID enode.I senderKey, err := v4wire.DecodePubkey(crypto.S256(), fromKey) if err != nil { + t.mutex.Lock() + t.errors[err.Error()] = t.errors[err.Error()] + 1 + t.mutex.Unlock() return err } if v4wire.Expired(req.Expiration) { + t.mutex.Lock() + t.errors[errExpiredStr] = t.errors[errExpiredStr] + 1 + t.mutex.Unlock() return errExpired } h.senderKey = senderKey @@ -719,9 +919,15 @@ func (t *UDPv4) verifyPong(h *packetHandlerV4, from *net.UDPAddr, fromID enode.I req := h.Packet.(*v4wire.Pong) if v4wire.Expired(req.Expiration) { + t.mutex.Lock() + t.errors[errExpiredStr] = t.errors[errExpiredStr] + 1 + t.mutex.Unlock() return errExpired } - if !t.handleReply(fromID, from.IP, req) { + if !t.handleReply(fromID, from.IP, from.Port, req) { + t.mutex.Lock() + t.errors[errUnsolicitedReplyStr] = t.errors[errUnsolicitedReplyStr] + 1 + t.mutex.Unlock() return errUnsolicitedReply } t.localNode.UDPEndpointStatement(from, &net.UDPAddr{IP: req.To.IP, Port: int(req.To.UDP)}) @@ -735,6 +941,9 @@ func (t *UDPv4) verifyFindnode(h *packetHandlerV4, from *net.UDPAddr, fromID eno req := h.Packet.(*v4wire.Findnode) if v4wire.Expired(req.Expiration) { + t.mutex.Lock() + t.errors[errExpiredStr] = t.errors[errExpiredStr] + 1 + t.mutex.Unlock() return errExpired } if !t.checkBond(fromID, from.IP) { @@ -744,6 +953,9 @@ func (t *UDPv4) verifyFindnode(h *packetHandlerV4, from *net.UDPAddr, fromID eno // and UDP port of the target as the source address. The recipient of the findnode // packet would then send a neighbors packet (which is a much bigger packet than // findnode) to the victim. + t.mutex.Lock() + t.errors[errUnknownNodeStr] = t.errors[errUnknownNodeStr] + 1 + t.mutex.Unlock() return errUnknownNode } return nil @@ -781,9 +993,15 @@ func (t *UDPv4) verifyNeighbors(h *packetHandlerV4, from *net.UDPAddr, fromID en req := h.Packet.(*v4wire.Neighbors) if v4wire.Expired(req.Expiration) { + t.mutex.Lock() + t.errors[errExpiredStr] = t.errors[errExpiredStr] + 1 + t.mutex.Unlock() return errExpired } - if !t.handleReply(fromID, from.IP, h.Packet) { + if !t.handleReply(fromID, from.IP, from.Port, h.Packet) { + t.mutex.Lock() + t.errors[errUnsolicitedReplyStr] = t.errors[errUnsolicitedReplyStr] + 1 + t.mutex.Unlock() return errUnsolicitedReply } return nil @@ -795,26 +1013,40 @@ func (t *UDPv4) verifyENRRequest(h *packetHandlerV4, from *net.UDPAddr, fromID e req := h.Packet.(*v4wire.ENRRequest) if v4wire.Expired(req.Expiration) { + t.mutex.Lock() + t.errors[errExpiredStr] = t.errors[errExpiredStr] + 1 + t.mutex.Unlock() return errExpired } if !t.checkBond(fromID, from.IP) { + t.mutex.Lock() + t.errors[errUnknownNodeStr] = t.errors[errUnknownNodeStr] + 1 + t.mutex.Unlock() return errUnknownNode } return nil } func (t *UDPv4) handleENRRequest(h *packetHandlerV4, from *net.UDPAddr, fromID enode.ID, mac []byte) { - //nolint:errcheck - t.send(from, fromID, &v4wire.ENRResponse{ + _, err := t.send(from, fromID, &v4wire.ENRResponse{ ReplyTok: mac, Record: *t.localNode.Node().Record(), }) + + if err != nil { + t.mutex.Lock() + t.errors[err.Error()] = t.errors[err.Error()] + 1 + t.mutex.Unlock() + } } // ENRRESPONSE/v4 func (t *UDPv4) verifyENRResponse(h *packetHandlerV4, from *net.UDPAddr, fromID enode.ID, fromKey v4wire.Pubkey) error { - if !t.handleReply(fromID, from.IP, h.Packet) { + if !t.handleReply(fromID, from.IP, from.Port, h.Packet) { + t.mutex.Lock() + t.errors[errUnsolicitedReplyStr] = t.errors[errUnsolicitedReplyStr] + 1 + t.mutex.Unlock() return errUnsolicitedReply } return nil diff --git a/p2p/discover/v4_udp_test.go b/p2p/discover/v4_udp_test.go index 289bd2715e0..5e2a9df92b6 100644 --- a/p2p/discover/v4_udp_test.go +++ b/p2p/discover/v4_udp_test.go @@ -87,7 +87,7 @@ func newUDPTestContext(ctx context.Context, t *testing.T, logger log.Logger) *ud panic(err) } ln := enode.NewLocalNode(test.db, test.localkey, logger) - test.udp, err = ListenV4(ctx, test.pipe, ln, Config{ + test.udp, err = ListenV4(ctx, "test", test.pipe, ln, Config{ PrivateKey: test.localkey, Log: testlog.Logger(t, log.LvlError), @@ -237,7 +237,7 @@ func TestUDPv4_responseTimeouts(t *testing.T) { p.errc = nilErr test.udp.addReplyMatcher <- p time.AfterFunc(randomDuration(60*time.Millisecond), func() { - if !test.udp.handleReply(p.from, p.ip, testPacket(p.ptype)) { + if !test.udp.handleReply(p.from, p.ip, p.port, testPacket(p.ptype)) { t.Logf("not matched: %v", p) } }) @@ -643,7 +643,7 @@ func startLocalhostV4(ctx context.Context, t *testing.T, cfg Config, logger log. realaddr := socket.LocalAddr().(*net.UDPAddr) ln.SetStaticIP(realaddr.IP) ln.SetFallbackUDP(realaddr.Port) - udp, err := ListenV4(ctx, socket, ln, cfg) + udp, err := ListenV4(ctx, "test", socket, ln, cfg) if err != nil { t.Fatal(err) } diff --git a/p2p/discover/v5_udp.go b/p2p/discover/v5_udp.go index 686bd267879..d66d44e36f0 100644 --- a/p2p/discover/v5_udp.go +++ b/p2p/discover/v5_udp.go @@ -97,6 +97,7 @@ type UDPv5 struct { closeCtx context.Context cancelCloseCtx context.CancelFunc wg sync.WaitGroup + errors map[string]uint } // TalkRequestHandler callback processes a talk request and optionally returns a reply @@ -125,8 +126,8 @@ type callTimeout struct { } // ListenV5 listens on the given connection. -func ListenV5(ctx context.Context, conn UDPConn, ln *enode.LocalNode, cfg Config) (*UDPv5, error) { - t, err := newUDPv5(ctx, conn, ln, cfg) +func ListenV5(ctx context.Context, protocol string, conn UDPConn, ln *enode.LocalNode, cfg Config) (*UDPv5, error) { + t, err := newUDPv5(ctx, protocol, conn, ln, cfg) if err != nil { return nil, err } @@ -138,7 +139,7 @@ func ListenV5(ctx context.Context, conn UDPConn, ln *enode.LocalNode, cfg Config } // newUDPv5 creates a UDPv5 transport, but doesn't start any goroutines. -func newUDPv5(ctx context.Context, conn UDPConn, ln *enode.LocalNode, cfg Config) (*UDPv5, error) { +func newUDPv5(ctx context.Context, protocol string, conn UDPConn, ln *enode.LocalNode, cfg Config) (*UDPv5, error) { closeCtx, cancelCloseCtx := context.WithCancel(ctx) cfg = cfg.withDefaults(respTimeoutV5) t := &UDPv5{ @@ -167,8 +168,9 @@ func newUDPv5(ctx context.Context, conn UDPConn, ln *enode.LocalNode, cfg Config // shutdown closeCtx: closeCtx, cancelCloseCtx: cancelCloseCtx, + errors: map[string]uint{}, } - tab, err := newTable(t, t.db, cfg.Bootnodes, cfg.TableRevalidateInterval, cfg.Log) + tab, err := newTable(t, protocol, t.db, cfg.Bootnodes, cfg.TableRevalidateInterval, cfg.Log) if err != nil { return nil, err } @@ -181,6 +183,18 @@ func (t *UDPv5) Self() *enode.Node { return t.localNode.Node() } +func (t *UDPv5) Version() string { + return "v5" +} + +func (t *UDPv5) Errors() map[string]uint { + return t.errors +} + +func (t *UDPv5) LenUnsolicited() int { + return 0 +} + // Close shuts down packet processing. func (t *UDPv5) Close() { t.closeOnce.Do(func() { diff --git a/p2p/discover/v5_udp_test.go b/p2p/discover/v5_udp_test.go index 09c8a21107c..5ca080e0435 100644 --- a/p2p/discover/v5_udp_test.go +++ b/p2p/discover/v5_udp_test.go @@ -67,7 +67,7 @@ func startLocalhostV5(t *testing.T, cfg Config, logger log.Logger) *UDPv5 { ln.SetFallbackUDP(realaddr.Port) ctx := context.Background() ctx = disableLookupSlowdown(ctx) - udp, err := ListenV5(ctx, socket, ln, cfg) + udp, err := ListenV5(ctx, "test", socket, ln, cfg) if err != nil { t.Fatal(err) } @@ -581,7 +581,7 @@ func newUDPV5TestContext(ctx context.Context, t *testing.T, logger log.Logger) * ln := enode.NewLocalNode(test.db, test.localkey, logger) ln.SetStaticIP(net.IP{10, 0, 0, 1}) ln.Set(enr.UDP(30303)) - test.udp, err = ListenV5(ctx, test.pipe, ln, Config{ + test.udp, err = ListenV5(ctx, "test", test.pipe, ln, Config{ PrivateKey: test.localkey, Log: testlog.Logger(t, log.LvlError), ValidSchemes: enode.ValidSchemesForTesting, diff --git a/p2p/peer.go b/p2p/peer.go index 0adf711d765..43767f42786 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -223,7 +223,9 @@ func (p *Peer) Inbound() bool { } func newPeer(logger log.Logger, conn *conn, protocols []Protocol, pubkey [64]byte, metricsEnabled bool) *Peer { - protomap := matchProtocols(protocols, conn.caps, conn) + log := logger.New("id", conn.node.ID(), "conn", conn.flags) + + protomap := matchProtocols(protocols, conn.caps, conn, log) p := &Peer{ rw: conn, running: protomap, @@ -232,7 +234,7 @@ func newPeer(logger log.Logger, conn *conn, protocols []Protocol, pubkey [64]byt protoErr: make(chan *PeerError, len(protomap)+1), // protocols + pingLoop closed: make(chan struct{}), pingRecv: make(chan struct{}, 16), - log: logger.New("id", conn.node.ID(), "conn", conn.flags), + log: log, pubkey: pubkey, metricsEnabled: metricsEnabled, CapBytesIn: make(map[string]uint64), @@ -438,7 +440,7 @@ func countMatchingProtocols(protocols []Protocol, caps []Cap) int { } // matchProtocols creates structures for matching named subprotocols. -func matchProtocols(protocols []Protocol, caps []Cap, rw MsgReadWriter) map[string]*protoRW { +func matchProtocols(protocols []Protocol, caps []Cap, rw MsgReadWriter, logger log.Logger) map[string]*protoRW { sort.Sort(capsByNameAndVersion(caps)) offset := baseProtocolLength result := make(map[string]*protoRW) @@ -452,7 +454,7 @@ outer: offset -= old.Length } // Assign the new match - result[cap.Name] = &protoRW{Protocol: proto, offset: offset, in: make(chan Msg), w: rw} + result[cap.Name] = &protoRW{Protocol: proto, offset: offset, in: make(chan Msg), w: rw, logger: logger} offset += proto.Length continue outer @@ -506,8 +508,11 @@ type protoRW struct { werr chan<- error // for write results offset uint64 w MsgWriter + logger log.Logger } +var traceMsg = false + func (rw *protoRW) WriteMsg(msg Msg) (err error) { if msg.Code >= rw.Length { return NewPeerError(PeerErrorInvalidMessageCode, DiscProtocolError, nil, fmt.Sprintf("not handled code=%d", msg.Code)) @@ -520,6 +525,15 @@ func (rw *protoRW) WriteMsg(msg Msg) (err error) { select { case <-rw.wstart: err = rw.w.WriteMsg(msg) + + if traceMsg { + if err != nil { + rw.logger.Trace("Write failed", "cap", rw.cap(), "msg", msg.Code-rw.offset, "size", msg.Size, "err", err) + } else { + rw.logger.Trace("Wrote", "cap", rw.cap(), "msg", msg.Code-rw.offset, "size", msg.Size) + } + } + // Report write status back to Peer.run. It will initiate // shutdown if the error is non-nil and unblock the next write // otherwise. The calling protocol code should exit for errors @@ -536,6 +550,9 @@ func (rw *protoRW) ReadMsg() (Msg, error) { select { case msg := <-rw.in: msg.Code -= rw.offset + if traceMsg { + rw.logger.Trace("Read", "cap", rw.cap(), "msg", msg.Code, "size", msg.Size) + } return msg, nil case <-rw.closed: return Msg{}, io.EOF diff --git a/p2p/peer_test.go b/p2p/peer_test.go index 45b0e89f655..c8409764592 100644 --- a/p2p/peer_test.go +++ b/p2p/peer_test.go @@ -331,7 +331,7 @@ func TestMatchProtocols(t *testing.T) { } for i, tt := range tests { - result := matchProtocols(tt.Local, tt.Remote, nil) + result := matchProtocols(tt.Local, tt.Remote, nil, log.Root()) if len(result) != len(tt.Match) { t.Errorf("test %d: negotiation mismatch: have %v, want %v", i, len(result), len(tt.Match)) continue diff --git a/p2p/server.go b/p2p/server.go index 4a32e45b35f..7ba83014a3e 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -27,6 +27,7 @@ import ( "net" "sort" "strconv" + "strings" "sync" "sync/atomic" "time" @@ -68,6 +69,8 @@ const ( // Maximum amount of time allowed for writing a complete message. frameWriteTimeout = 20 * time.Second + + serverStatsLogInterval = 60 * time.Second ) var errServerStopped = errors.New("server stopped") @@ -232,6 +235,7 @@ type Server struct { // State of run loop and listenLoop. inboundHistory expHeap + errors map[string]uint } type peerOpFunc func(map[enode.ID]*Peer) @@ -654,7 +658,7 @@ func (srv *Server) setupDiscovery(ctx context.Context) error { Unhandled: unhandled, Log: srv.logger, } - ntab, err := discover.ListenV4(ctx, conn, srv.localnode, cfg) + ntab, err := discover.ListenV4(ctx, fmt.Sprint(srv.Config.Protocols[0].Version), conn, srv.localnode, cfg) if err != nil { return err } @@ -672,9 +676,9 @@ func (srv *Server) setupDiscovery(ctx context.Context) error { } var err error if sconn != nil { - srv.DiscV5, err = discover.ListenV5(ctx, sconn, srv.localnode, cfg) + srv.DiscV5, err = discover.ListenV5(ctx, fmt.Sprint(srv.Config.Protocols[0].Version), sconn, srv.localnode, cfg) } else { - srv.DiscV5, err = discover.ListenV5(ctx, conn, srv.localnode, cfg) + srv.DiscV5, err = discover.ListenV5(ctx, fmt.Sprint(srv.Config.Protocols[0].Version), conn, srv.localnode, cfg) } if err != nil { return err @@ -792,6 +796,9 @@ func (srv *Server) run() { trusted[n.ID()] = true } + logTimer := time.NewTicker(serverStatsLogInterval) + defer logTimer.Stop() + running: for { select { @@ -855,6 +862,18 @@ running: if pd.Inbound() { inboundCount-- } + case <-logTimer.C: + vals := []interface{}{"protocol", srv.Config.Protocols[0].Version, "peers", len(peers), "trusted", len(trusted), "inbound", inboundCount} + + func() { + srv.lock.Lock() + defer srv.lock.Unlock() + for err, count := range srv.errors { + vals = append(vals, err, count) + } + }() + + srv.logger.Debug("[p2p] Server", vals...) } } @@ -906,6 +925,8 @@ func (srv *Server) listenLoop(ctx context.Context) { // The slots limit accepts of new connections. slots := semaphore.NewWeighted(int64(srv.MaxPendingPeers)) + srv.errors = map[string]uint{} + // Wait for slots to be returned on exit. This ensures all connection goroutines // are down before listenLoop returns. defer func() { @@ -1008,10 +1029,25 @@ func (srv *Server) SetupConn(fd net.Conn, flags connFlag, dialDest *enode.Node) return err } +func cleanError(err string) string { + switch { + case strings.HasSuffix(err, "i/o timeout"): + return "i/o timeout" + case strings.HasSuffix(err, "closed by the remote host."): + return "closed by remote" + case strings.HasSuffix(err, "connection reset by peer"): + return "closed by remote" + default: + return err + } +} + func (srv *Server) setupConn(c *conn, flags connFlag, dialDest *enode.Node) error { // Prevent leftover pending conns from entering the handshake. srv.lock.Lock() running := srv.running + // reset error counts + srv.errors = map[string]uint{} srv.lock.Unlock() if !running { return errServerStopped @@ -1031,6 +1067,10 @@ func (srv *Server) setupConn(c *conn, flags connFlag, dialDest *enode.Node) erro // Run the RLPx handshake. remotePubkey, err := c.doEncHandshake(srv.PrivateKey) if err != nil { + errStr := cleanError(err.Error()) + srv.lock.Lock() + srv.errors[errStr] = srv.errors[errStr] + 1 + srv.lock.Unlock() srv.logger.Trace("Failed RLPx handshake", "addr", c.fd.RemoteAddr(), "conn", c.flags, "err", err) return err } @@ -1050,6 +1090,10 @@ func (srv *Server) setupConn(c *conn, flags connFlag, dialDest *enode.Node) erro // Run the capability negotiation handshake. phs, err := c.doProtoHandshake(srv.ourHandshake) if err != nil { + errStr := cleanError(err.Error()) + srv.lock.Lock() + srv.errors[errStr] = srv.errors[errStr] + 1 + srv.lock.Unlock() clog.Trace("Failed p2p handshake", "err", err) return err } diff --git a/turbo/jsonrpc/bor_snapshot.go b/turbo/jsonrpc/bor_snapshot.go index a6793ceeead..7a6ef67f4c8 100644 --- a/turbo/jsonrpc/bor_snapshot.go +++ b/turbo/jsonrpc/bor_snapshot.go @@ -241,6 +241,7 @@ func (api *BorImpl) GetVoteOnHash(ctx context.Context, starBlockNr uint64, endBl if err != nil { return false, err } + defer tx.Rollback() service := whitelist.GetWhitelistingService() diff --git a/turbo/services/interfaces.go b/turbo/services/interfaces.go index 2b86e736f16..4fbe8a7c3a5 100644 --- a/turbo/services/interfaces.go +++ b/turbo/services/interfaces.go @@ -38,6 +38,10 @@ type BorEventReader interface { EventsByBlock(ctx context.Context, tx kv.Tx, hash common.Hash, blockNum uint64) ([]rlp.RawValue, error) } +type BorSpanReader interface { + Span(ctx context.Context, tx kv.Getter, spanNum uint64) ([]byte, error) +} + type CanonicalReader interface { CanonicalHash(ctx context.Context, tx kv.Getter, blockNum uint64) (common.Hash, error) BadHeaderNumber(ctx context.Context, tx kv.Getter, hash common.Hash) (blockHeight *uint64, err error) @@ -71,6 +75,7 @@ type FullBlockReader interface { BodyReader HeaderReader BorEventReader + BorSpanReader TxnReader CanonicalReader diff --git a/turbo/snapshotsync/freezeblocks/block_reader.go b/turbo/snapshotsync/freezeblocks/block_reader.go index 430e9f5ab95..d1b67cc6404 100644 --- a/turbo/snapshotsync/freezeblocks/block_reader.go +++ b/turbo/snapshotsync/freezeblocks/block_reader.go @@ -235,6 +235,10 @@ func (r *RemoteBlockReader) EventsByBlock(ctx context.Context, tx kv.Tx, hash co return result, nil } +func (r *RemoteBlockReader) Span(ctx context.Context, tx kv.Getter, spanId uint64) ([]byte, error) { + return nil, nil +} + // BlockReader can read blocks from db and snapshots type BlockReader struct { sn *RoSnapshots @@ -1078,6 +1082,73 @@ func (r *BlockReader) LastFrozenEventID() uint64 { return lastEventID } +func (r *BlockReader) LastFrozenSpanID() uint64 { + view := r.borSn.View() + defer view.Close() + segments := view.Spans() + if len(segments) == 0 { + return 0 + } + lastSegment := segments[len(segments)-1] + var lastSpanID uint64 + if lastSegment.ranges.to > zerothSpanEnd { + lastSpanID = (lastSegment.ranges.to - zerothSpanEnd - 1) / spanLength + } + return lastSpanID +} + +func (r *BlockReader) Span(ctx context.Context, tx kv.Getter, spanId uint64) ([]byte, error) { + // Compute starting block of the span + var endBlock uint64 + if spanId > 0 { + endBlock = (spanId)*spanLength + zerothSpanEnd + } + var buf [8]byte + binary.BigEndian.PutUint64(buf[:], spanId) + if endBlock >= r.FrozenBorBlocks() { + v, err := tx.GetOne(kv.BorSpans, buf[:]) + if err != nil { + return nil, err + } + if v == nil { + return nil, fmt.Errorf("span %d not found (db)", spanId) + } + return common.Copy(v), nil + } + view := r.borSn.View() + defer view.Close() + segments := view.Spans() + for i := len(segments) - 1; i >= 0; i-- { + sn := segments[i] + if sn.idx == nil { + continue + } + var spanFrom uint64 + if sn.ranges.from > zerothSpanEnd { + spanFrom = 1 + (sn.ranges.from-zerothSpanEnd-1)/spanLength + } + if spanId < spanFrom { + continue + } + var spanTo uint64 + if sn.ranges.to > zerothSpanEnd { + spanTo = 1 + (sn.ranges.to-zerothSpanEnd-1)/spanLength + } + if spanId >= spanTo { + continue + } + if sn.idx.KeyCount() == 0 { + continue + } + offset := sn.idx.OrdinalLookup(spanId - sn.idx.BaseDataID()) + gg := sn.seg.MakeGetter() + gg.Reset(offset) + result, _ := gg.Next(nil) + return common.Copy(result), nil + } + return nil, fmt.Errorf("span %d not found (snapshots)", spanId) +} + // ---- Data Integrity part ---- func (r *BlockReader) ensureHeaderNumber(n uint64, seg *HeaderSegment) error { diff --git a/turbo/snapshotsync/freezeblocks/bor_snapshots.go b/turbo/snapshotsync/freezeblocks/bor_snapshots.go index a1240ccb52f..11899abc9ac 100644 --- a/turbo/snapshotsync/freezeblocks/bor_snapshots.go +++ b/turbo/snapshotsync/freezeblocks/bor_snapshots.go @@ -374,7 +374,7 @@ func DumpBorEvents(ctx context.Context, db kv.RoDB, blockFrom, blockTo uint64, w return nil } -// DumpBorEvents - [from, to) +// DumpBorSpans - [from, to) func DumpBorSpans(ctx context.Context, db kv.RoDB, blockFrom, blockTo uint64, workers int, lvl log.Lvl, logger log.Logger, collect func([]byte) error) error { logEvery := time.NewTicker(20 * time.Second) defer logEvery.Stop() @@ -1163,8 +1163,7 @@ func (m *BorMerger) Merge(ctx context.Context, snapshots *BorRoSnapshots, mergeR return err } } - time.Sleep(1 * time.Second) // i working on blocking API - to ensure client does not use old snapsthos - and then delete them - for _, t := range snaptype.BlockSnapshotTypes { + for _, t := range []snaptype.Type{snaptype.BorEvents, snaptype.BorSpans} { m.removeOldFiles(toMerge[t], snapDir) } } diff --git a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go index 064de66a85d..744321df0ae 100644 --- a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go +++ b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go @@ -342,6 +342,8 @@ func dumpBeaconBlocksRange(ctx context.Context, db kv.RoDB, b persistence.BlockS var w bytes.Buffer lzWriter := lz4.NewWriter(&w) defer lzWriter.Close() + // Just make a reusable buffer + buf := make([]byte, 2048) // Generate .seg file, which is just the list of beacon blocks. for i := fromSlot; i < toSlot; i++ { obj, err := b.GetBlock(ctx, tx, i) @@ -360,7 +362,7 @@ func dumpBeaconBlocksRange(ctx context.Context, db kv.RoDB, b persistence.BlockS } lzWriter.Reset(&w) lzWriter.CompressionLevel = 1 - if err := snapshot_format.WriteBlockForSnapshot(obj.Data, lzWriter); err != nil { + if buf, err = snapshot_format.WriteBlockForSnapshot(lzWriter, obj.Data, buf); err != nil { return err } if err := lzWriter.Flush(); err != nil { diff --git a/turbo/snapshotsync/snapshotsync.go b/turbo/snapshotsync/snapshotsync.go index 2298ba8ed79..a071d4d962d 100644 --- a/turbo/snapshotsync/snapshotsync.go +++ b/turbo/snapshotsync/snapshotsync.go @@ -12,6 +12,7 @@ import ( "github.com/ledgerwatch/erigon-lib/chain/snapcfg" "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/dbg" + "github.com/ledgerwatch/erigon-lib/diagnostics" "github.com/ledgerwatch/erigon-lib/downloader/downloadergrpc" "github.com/ledgerwatch/erigon-lib/downloader/snaptype" proto_downloader "github.com/ledgerwatch/erigon-lib/gointerfaces/downloader" @@ -130,6 +131,7 @@ func WaitForDownloader(logPrefix string, ctx context.Context, histV3 bool, capli // send all hashes to the Downloader service preverifiedBlockSnapshots := snapcfg.KnownCfg(cc.ChainName, []string{} /* whitelist */, snHistInDB).Preverified downloadRequest := make([]services.DownloadRequest, 0, len(preverifiedBlockSnapshots)+len(missingSnapshots)) + // build all download requests // builds preverified snapshots request for _, p := range preverifiedBlockSnapshots { @@ -205,6 +207,22 @@ Loop: } } */ + + diagnostics.Send(diagnostics.DownloadStatistics{ + Downloaded: stats.BytesCompleted, + Total: stats.BytesTotal, + TotalTime: time.Since(downloadStartTime).Round(time.Second).Seconds(), + DownloadRate: stats.DownloadRate, + UploadRate: stats.UploadRate, + Peers: stats.PeersUnique, + Files: stats.FilesTotal, + Connections: stats.ConnectionsTotal, + Alloc: m.Alloc, + Sys: m.Sys, + DownloadFinished: stats.Completed, + StagePrefix: logPrefix, + }) + log.Info(fmt.Sprintf("[%s] download finished", logPrefix), "time", time.Since(downloadStartTime).String()) break Loop } else { @@ -218,6 +236,22 @@ Loop: if stats.Progress > 0 && stats.DownloadRate == 0 { suffix += " (or verifying)" } + + diagnostics.Send(diagnostics.DownloadStatistics{ + Downloaded: stats.BytesCompleted, + Total: stats.BytesTotal, + TotalTime: time.Since(downloadStartTime).Round(time.Second).Seconds(), + DownloadRate: stats.DownloadRate, + UploadRate: stats.UploadRate, + Peers: stats.PeersUnique, + Files: stats.FilesTotal, + Connections: stats.ConnectionsTotal, + Alloc: m.Alloc, + Sys: m.Sys, + DownloadFinished: stats.Completed, + StagePrefix: logPrefix, + }) + log.Info(fmt.Sprintf("[%s] %s", logPrefix, suffix), "progress", fmt.Sprintf("%.2f%% %s/%s", stats.Progress, common.ByteCount(stats.BytesCompleted), common.ByteCount(stats.BytesTotal)), "time-left", downloadTimeLeft, diff --git a/turbo/stages/headerdownload/header_algo_test.go b/turbo/stages/headerdownload/header_algo_test.go index 23a17fedf15..3e6d76d47ac 100644 --- a/turbo/stages/headerdownload/header_algo_test.go +++ b/turbo/stages/headerdownload/header_algo_test.go @@ -1,10 +1,12 @@ package headerdownload_test import ( + "bytes" "context" "math/big" "testing" + "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon/core" @@ -16,7 +18,7 @@ import ( "github.com/ledgerwatch/erigon/turbo/stages/mock" ) -func TestInserter1(t *testing.T) { +func TestSideChainInsert(t *testing.T) { funds := big.NewInt(1000000000) key, _ := crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") address := crypto.PubkeyToAddress(key.PublicKey) @@ -40,24 +42,83 @@ func TestInserter1(t *testing.T) { defer tx.Rollback() br := m.BlockReader hi := headerdownload.NewHeaderInserter("headers", big.NewInt(0), 0, br) - h1 := types.Header{ - Number: big.NewInt(1), - Difficulty: big.NewInt(10), - ParentHash: genesis.Hash(), + + // Chain with higher initial difficulty + chain1 := createTestChain(3, genesis.Hash(), 2, []byte("")) + + // Smaller side chain (non-canonical) + chain2 := createTestChain(5, genesis.Hash(), 1, []byte("side1")) + + // Bigger side chain (canonical) + chain3 := createTestChain(7, genesis.Hash(), 1, []byte("side2")) + + // Again smaller side chain but with high difficulty (canonical) + chain4 := createTestChain(5, genesis.Hash(), 2, []byte("side3")) + + // More smaller side chain with same difficulty (canonical) + chain5 := createTestChain(2, genesis.Hash(), 5, []byte("side5")) + + // Bigger side chain with same difficulty (non-canonical) + chain6 := createTestChain(10, genesis.Hash(), 1, []byte("side6")) + + // Same side chain (in terms of number and difficulty) but different hash + chain7 := createTestChain(2, genesis.Hash(), 5, []byte("side7")) + + finalExpectedHash := chain5[len(chain5)-1].Hash() + if bytes.Compare(chain5[len(chain5)-1].Hash().Bytes(), chain7[len(chain7)-1].Hash().Bytes()) < 0 { + finalExpectedHash = chain7[len(chain7)-1].Hash() } - h1Hash := h1.Hash() - h2 := types.Header{ - Number: big.NewInt(2), - Difficulty: big.NewInt(1010), - ParentHash: h1Hash, + + testCases := []struct { + name string + chain []types.Header + expectedHash common.Hash + expectedDiff int64 + }{ + {"normal initial insert", chain1, chain1[len(chain1)-1].Hash(), 6}, + {"td(current) > td(incoming)", chain2, chain1[len(chain1)-1].Hash(), 6}, + {"td(incoming) > td(current), number(incoming) > number(current)", chain3, chain3[len(chain3)-1].Hash(), 7}, + {"td(incoming) > td(current), number(current) > number(incoming)", chain4, chain4[len(chain4)-1].Hash(), 10}, + {"td(incoming) = td(current), number(current) > number(current)", chain5, chain5[len(chain5)-1].Hash(), 10}, + {"td(incoming) = td(current), number(incoming) > number(current)", chain6, chain5[len(chain5)-1].Hash(), 10}, + {"td(incoming) = td(current), number(incoming) = number(current), hash different", chain7, finalExpectedHash, 10}, } - h2Hash := h2.Hash() - data1, _ := rlp.EncodeToBytes(&h1) - if _, err = hi.FeedHeaderPoW(tx, br, &h1, data1, h1Hash, 1); err != nil { - t.Errorf("feed empty header 1: %v", err) + + for _, tc := range testCases { + tc := tc + for i, h := range tc.chain { + h := h + data, _ := rlp.EncodeToBytes(&h) + if _, err = hi.FeedHeaderPoW(tx, br, &h, data, h.Hash(), uint64(i+1)); err != nil { + t.Errorf("feed empty header for %s, err: %v", tc.name, err) + } + } + + if hi.GetHighestHash() != tc.expectedHash { + t.Errorf("incorrect highest hash for %s, expected %s, got %s", tc.name, tc.expectedHash, hi.GetHighestHash()) + } + if hi.GetLocalTd().Int64() != tc.expectedDiff { + t.Errorf("incorrect difficulty for %s, expected %d, got %d", tc.name, tc.expectedDiff, hi.GetLocalTd().Int64()) + } } - data2, _ := rlp.EncodeToBytes(&h2) - if _, err = hi.FeedHeaderPoW(tx, br, &h2, data2, h2Hash, 2); err != nil { - t.Errorf("feed empty header 2: %v", err) +} + +func createTestChain(length int64, parent common.Hash, diff int64, extra []byte) []types.Header { + var ( + i int64 + headers []types.Header + ) + + for i = 0; i < length; i++ { + h := types.Header{ + Number: big.NewInt(i + 1), + Difficulty: big.NewInt(diff), + ParentHash: parent, + Extra: extra, + } + headers = append(headers, h) + parent = h.Hash() } + + return headers } diff --git a/turbo/stages/headerdownload/header_algos.go b/turbo/stages/headerdownload/header_algos.go index e7b8a29e8fe..f9b64b074ea 100644 --- a/turbo/stages/headerdownload/header_algos.go +++ b/turbo/stages/headerdownload/header_algos.go @@ -8,13 +8,14 @@ import ( "encoding/base64" "errors" "fmt" - "github.com/ledgerwatch/erigon-lib/kv/dbutils" "io" "math/big" "sort" "strings" "time" + "github.com/ledgerwatch/erigon-lib/kv/dbutils" + libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/etl" "github.com/ledgerwatch/erigon-lib/kv" @@ -432,6 +433,8 @@ func (hd *HeaderDownload) requestMoreHeadersForPOS(currentTime time.Time) (timeo return } + hd.logger.Debug("[downloader] Request header", "numer", anchor.blockHeight-1, "length", 192) + // Request ancestors request = &HeaderRequest{ Anchor: anchor, @@ -482,7 +485,7 @@ func (hd *HeaderDownload) UpdateRetryTime(req *HeaderRequest, currentTime time.T func (hd *HeaderDownload) RequestSkeleton() *HeaderRequest { hd.lock.RLock() defer hd.lock.RUnlock() - hd.logger.Debug("[downloader] Request skeleton", "anchors", len(hd.anchors), "highestInDb", hd.highestInDb) + var stride uint64 if hd.initialCycle { stride = 192 @@ -495,6 +498,7 @@ func (hd *HeaderDownload) RequestSkeleton() *HeaderRequest { } else { from-- } + return &HeaderRequest{Number: from, Length: length, Skip: stride, Reverse: false} } @@ -892,24 +896,40 @@ func (hi *HeaderInserter) FeedHeaderPoW(db kv.StatelessRwTx, headerReader servic } // Calculate total difficulty of this header using parent's total difficulty td = new(big.Int).Add(parentTd, header.Difficulty) + // Now we can decide wether this header will create a change in the canonical head - if td.Cmp(hi.localTd) > 0 { - hi.newCanonical = true - forkingPoint, err := hi.ForkingPoint(db, header, parent) - if err != nil { - return nil, err + if td.Cmp(hi.localTd) >= 0 { + reorg := true + + // TODO: Add bor check here if required + // Borrowed from https://github.com/maticnetwork/bor/blob/master/core/forkchoice.go#L81 + if td.Cmp(hi.localTd) == 0 { + if blockHeight > hi.highest { + reorg = false + } else if blockHeight == hi.highest { + // Compare hashes of block in case of tie breaker. Lexicographically larger hash wins. + reorg = bytes.Compare(hi.highestHash.Bytes(), hash.Bytes()) < 0 + } } - hi.highest = blockHeight - hi.highestHash = hash - hi.highestTimestamp = header.Time - hi.canonicalCache.Add(blockHeight, hash) - // See if the forking point affects the unwindPoint (the block number to which other stages will need to unwind before the new canonical chain is applied) - if forkingPoint < hi.unwindPoint { - hi.unwindPoint = forkingPoint - hi.unwind = true + + if reorg { + hi.newCanonical = true + forkingPoint, err := hi.ForkingPoint(db, header, parent) + if err != nil { + return nil, err + } + hi.highest = blockHeight + hi.highestHash = hash + hi.highestTimestamp = header.Time + hi.canonicalCache.Add(blockHeight, hash) + // See if the forking point affects the unwindPoint (the block number to which other stages will need to unwind before the new canonical chain is applied) + if forkingPoint < hi.unwindPoint { + hi.unwindPoint = forkingPoint + hi.unwind = true + } + // This makes sure we end up choosing the chain with the max total difficulty + hi.localTd.Set(td) } - // This makes sure we end up choosing the chain with the max total difficulty - hi.localTd.Set(td) } if err = rawdb.WriteTd(db, hash, blockHeight, td); err != nil { return nil, fmt.Errorf("[%s] failed to WriteTd: %w", hi.logPrefix, err) @@ -946,6 +966,10 @@ func (hi *HeaderInserter) FeedHeaderPoS(db kv.RwTx, header *types.Header, hash l return nil } +func (hi *HeaderInserter) GetLocalTd() *big.Int { + return hi.localTd +} + func (hi *HeaderInserter) GetHighest() uint64 { return hi.highest } diff --git a/turbo/stages/mock/mock_sentry.go b/turbo/stages/mock/mock_sentry.go index de6ba517f34..fc92b99ec5c 100644 --- a/turbo/stages/mock/mock_sentry.go +++ b/turbo/stages/mock/mock_sentry.go @@ -11,6 +11,7 @@ import ( "time" "github.com/c2h5oh/datasize" + lru "github.com/hashicorp/golang-lru/arc/v2" "github.com/holiman/uint256" "github.com/ledgerwatch/log/v3" "google.golang.org/protobuf/types/known/emptypb" @@ -395,6 +396,16 @@ func MockWithEverything(tb testing.TB, gspec *types.Genesis, key *ecdsa.PrivateK var snapshotsDownloader proto_downloader.DownloaderClient + var ( + snapDb kv.RwDB + recents *lru.ARCCache[libcommon.Hash, *bor.Snapshot] + signatures *lru.ARCCache[libcommon.Hash, libcommon.Address] + ) + if bor, ok := engine.(*bor.Bor); ok { + snapDb = bor.DB + recents = bor.Recents + signatures = bor.Signatures + } // proof-of-stake mining assembleBlockPOS := func(param *core.BlockBuilderParameters, interrupt *int32) (*types.BlockWithReceipts, error) { miningStatePos := stagedsync.NewProposingState(&cfg.Miner) @@ -402,7 +413,7 @@ func MockWithEverything(tb testing.TB, gspec *types.Genesis, key *ecdsa.PrivateK proposingSync := stagedsync.New( stagedsync.MiningStages(mock.Ctx, stagedsync.StageMiningCreateBlockCfg(mock.DB, miningStatePos, *mock.ChainConfig, mock.Engine, mock.txPoolDB, param, tmpdir, mock.BlockReader), - stagedsync.StageBorHeimdallCfg(mock.DB, miningStatePos, *mock.ChainConfig, nil, mock.BlockReader, nil, nil), + stagedsync.StageBorHeimdallCfg(mock.DB, snapDb, miningStatePos, *mock.ChainConfig, nil, mock.BlockReader, nil, nil, recents, signatures), stagedsync.StageMiningExecCfg(mock.DB, miningStatePos, mock.Notifications.Events, *mock.ChainConfig, mock.Engine, &vm.Config{}, tmpdir, interrupt, param.PayloadId, mock.TxPool, mock.txPoolDB, mock.BlockReader), stagedsync.StageHashStateCfg(mock.DB, dirs, cfg.HistoryV3), stagedsync.StageTrieCfg(mock.DB, false, true, true, tmpdir, mock.BlockReader, nil, histV3, mock.agg), @@ -420,9 +431,9 @@ func MockWithEverything(tb testing.TB, gspec *types.Genesis, key *ecdsa.PrivateK blockRetire := freezeblocks.NewBlockRetire(1, dirs, mock.BlockReader, blockWriter, mock.DB, mock.Notifications.Events, logger) mock.Sync = stagedsync.New( stagedsync.DefaultStages(mock.Ctx, - stagedsync.StageSnapshotsCfg(mock.DB, *mock.ChainConfig, dirs, blockRetire, snapshotsDownloader, mock.BlockReader, mock.Notifications.Events, mock.HistoryV3, mock.agg, nil), + stagedsync.StageSnapshotsCfg(mock.DB, *mock.ChainConfig, dirs, blockRetire, snapshotsDownloader, mock.BlockReader, mock.Notifications.Events, mock.HistoryV3, mock.agg, false, nil), stagedsync.StageHeadersCfg(mock.DB, mock.sentriesClient.Hd, mock.sentriesClient.Bd, *mock.ChainConfig, sendHeaderRequest, propagateNewBlockHashes, penalize, cfg.BatchSize, false, mock.BlockReader, blockWriter, dirs.Tmp, mock.Notifications, engine_helpers.NewForkValidatorMock(1), nil), - stagedsync.StageBorHeimdallCfg(mock.DB, stagedsync.MiningState{}, *mock.ChainConfig, nil /* heimdallClient */, mock.BlockReader, nil, nil), + stagedsync.StageBorHeimdallCfg(mock.DB, snapDb, stagedsync.MiningState{}, *mock.ChainConfig, nil /* heimdallClient */, mock.BlockReader, nil, nil, recents, signatures), stagedsync.StageBlockHashesCfg(mock.DB, mock.Dirs.Tmp, mock.ChainConfig, blockWriter), stagedsync.StageBodiesCfg(mock.DB, mock.sentriesClient.Bd, sendBodyRequest, penalize, blockPropagator, cfg.Sync.BodyDownloadTimeoutSeconds, *mock.ChainConfig, mock.BlockReader, cfg.HistoryV3, blockWriter), stagedsync.StageSendersCfg(mock.DB, mock.ChainConfig, false, dirs.Tmp, prune, mock.BlockReader, mock.sentriesClient.Hd), @@ -485,7 +496,7 @@ func MockWithEverything(tb testing.TB, gspec *types.Genesis, key *ecdsa.PrivateK mock.MiningSync = stagedsync.New( stagedsync.MiningStages(mock.Ctx, stagedsync.StageMiningCreateBlockCfg(mock.DB, miner, *mock.ChainConfig, mock.Engine, nil, nil, dirs.Tmp, mock.BlockReader), - stagedsync.StageBorHeimdallCfg(mock.DB, miner, *mock.ChainConfig, nil /*heimdallClient*/, mock.BlockReader, nil, nil), + stagedsync.StageBorHeimdallCfg(mock.DB, snapDb, miner, *mock.ChainConfig, nil /*heimdallClient*/, mock.BlockReader, nil, nil, recents, signatures), stagedsync.StageMiningExecCfg(mock.DB, miner, nil, *mock.ChainConfig, mock.Engine, &vm.Config{}, dirs.Tmp, nil, 0, mock.TxPool, nil, mock.BlockReader), stagedsync.StageHashStateCfg(mock.DB, dirs, cfg.HistoryV3), stagedsync.StageTrieCfg(mock.DB, false, true, false, dirs.Tmp, mock.BlockReader, mock.sentriesClient.Hd, cfg.HistoryV3, mock.agg), diff --git a/turbo/stages/stageloop.go b/turbo/stages/stageloop.go index 56a1bbd6082..eff427be661 100644 --- a/turbo/stages/stageloop.go +++ b/turbo/stages/stageloop.go @@ -7,6 +7,7 @@ import ( "math/big" "time" + lru "github.com/hashicorp/golang-lru/arc/v2" "github.com/holiman/uint256" "github.com/ledgerwatch/log/v3" @@ -20,6 +21,7 @@ import ( "github.com/ledgerwatch/erigon-lib/state" "github.com/ledgerwatch/erigon/consensus" + "github.com/ledgerwatch/erigon/consensus/bor" "github.com/ledgerwatch/erigon/consensus/bor/finality/flags" "github.com/ledgerwatch/erigon/consensus/bor/heimdall" "github.com/ledgerwatch/erigon/consensus/misc" @@ -452,6 +454,7 @@ func silkwormForExecutionStage(silkworm *silkworm.Silkworm, cfg *ethconfig.Confi func NewDefaultStages(ctx context.Context, db kv.RwDB, + snapDb kv.RwDB, p2pCfg p2p.Config, cfg *ethconfig.Config, controlServer *sentry_multi_client.MultiClient, @@ -463,6 +466,8 @@ func NewDefaultStages(ctx context.Context, silkworm *silkworm.Silkworm, forkValidator *engine_helpers.ForkValidator, heimdallClient heimdall.IHeimdallClient, + recents *lru.ARCCache[libcommon.Hash, *bor.Snapshot], + signatures *lru.ARCCache[libcommon.Hash, libcommon.Address], logger log.Logger, ) []*stagedsync.Stage { dirs := cfg.Dirs @@ -479,9 +484,9 @@ func NewDefaultStages(ctx context.Context, } return stagedsync.DefaultStages(ctx, - stagedsync.StageSnapshotsCfg(db, *controlServer.ChainConfig, dirs, blockRetire, snapDownloader, blockReader, notifications.Events, cfg.HistoryV3, agg, silkworm), + stagedsync.StageSnapshotsCfg(db, *controlServer.ChainConfig, dirs, blockRetire, snapDownloader, blockReader, notifications.Events, cfg.HistoryV3, agg, cfg.InternalCL, silkworm), stagedsync.StageHeadersCfg(db, controlServer.Hd, controlServer.Bd, *controlServer.ChainConfig, controlServer.SendHeaderRequest, controlServer.PropagateNewBlockHashes, controlServer.Penalize, cfg.BatchSize, p2pCfg.NoDiscovery, blockReader, blockWriter, dirs.Tmp, notifications, forkValidator, loopBreakCheck), - stagedsync.StageBorHeimdallCfg(db, stagedsync.MiningState{}, *controlServer.ChainConfig, heimdallClient, blockReader, controlServer.Hd, controlServer.Penalize), + stagedsync.StageBorHeimdallCfg(db, snapDb, stagedsync.MiningState{}, *controlServer.ChainConfig, heimdallClient, blockReader, controlServer.Hd, controlServer.Penalize, recents, signatures), stagedsync.StageBlockHashesCfg(db, dirs.Tmp, controlServer.ChainConfig, blockWriter), stagedsync.StageBodiesCfg(db, controlServer.Bd, controlServer.SendBodyRequest, controlServer.Penalize, controlServer.BroadcastNewBlock, cfg.Sync.BodyDownloadTimeoutSeconds, *controlServer.ChainConfig, blockReader, cfg.HistoryV3, blockWriter), stagedsync.StageSendersCfg(db, controlServer.ChainConfig, false, dirs.Tmp, cfg.Prune, blockReader, controlServer.Hd), @@ -537,7 +542,7 @@ func NewPipelineStages(ctx context.Context, runInTestMode := cfg.ImportMode return stagedsync.PipelineStages(ctx, - stagedsync.StageSnapshotsCfg(db, *controlServer.ChainConfig, dirs, blockRetire, snapDownloader, blockReader, notifications.Events, cfg.HistoryV3, agg, silkworm), + stagedsync.StageSnapshotsCfg(db, *controlServer.ChainConfig, dirs, blockRetire, snapDownloader, blockReader, notifications.Events, cfg.HistoryV3, agg, cfg.InternalCL, silkworm), stagedsync.StageBlockHashesCfg(db, dirs.Tmp, controlServer.ChainConfig, blockWriter), stagedsync.StageSendersCfg(db, controlServer.ChainConfig, false, dirs.Tmp, cfg.Prune, blockReader, controlServer.Hd), stagedsync.StageExecuteBlocksCfg(