From 4b66f93274c7e0d889c84413144ad07ea4c5d4d5 Mon Sep 17 00:00:00 2001 From: Jordan Hrycaj Date: Tue, 20 Jun 2023 14:26:25 +0100 Subject: [PATCH] Aristo db with storage backends (#1603) * Generalised Aristo DB constructor for any type of backend details: * Records to be deleted are represented as key-void (rather than key-value) pairs by the put-function arguments * Allow direct driver access, iterators as example implementation and for testing. * Provide backend storage interface details: Stores the top layer onto backend tables * Implemented Rocks DB backend details: Transaction based `put()` functionality Iterators (based on direct RocksDB access) --- nimbus/db/aristo/aristo_debug.nim | 116 +++++--- nimbus/db/aristo/aristo_delete.nim | 6 +- nimbus/db/aristo/aristo_desc.nim | 28 +- nimbus/db/aristo/aristo_desc/aristo_error.nim | 76 ++--- .../aristo_desc/aristo_types_backend.nim | 48 ++- .../aristo_desc/aristo_types_identifiers.nim | 82 +++--- .../aristo_desc/aristo_types_private.nim | 15 - nimbus/db/aristo/aristo_get.nim | 91 +++--- nimbus/db/aristo/aristo_hashify.nim | 159 +++++++--- nimbus/db/aristo/aristo_hike.nim | 29 +- nimbus/db/aristo/aristo_init.nim | 81 ++++-- .../aristo/aristo_init/aristo_init_common.nim | 65 +++++ .../db/aristo/aristo_init/aristo_memory.nim | 182 ++++++++---- .../db/aristo/aristo_init/aristo_rocksdb.nim | 274 ++++++++++++++++++ .../aristo_init/aristo_rocksdb/rdb_desc.nim | 60 ++++ .../aristo_init/aristo_rocksdb/rdb_get.nim | 43 +++ .../aristo_init/aristo_rocksdb/rdb_init.nim | 107 +++++++ .../aristo_init/aristo_rocksdb/rdb_put.nim | 194 +++++++++++++ .../aristo_init/aristo_rocksdb/rdb_walk.nim | 178 ++++++++++++ nimbus/db/aristo/aristo_layer.nim | 129 +++++++++ nimbus/db/aristo/aristo_merge.nim | 172 ++++++++--- nimbus/db/aristo/aristo_path.nim | 3 +- nimbus/db/aristo/aristo_transcode.nim | 91 +++--- nimbus/db/aristo/aristo_vid.nim | 3 +- tests/test_aristo.nim | 25 +- tests/test_aristo/test_backend.nim | 256 ++++++++++++++++ tests/test_aristo/test_delete.nim | 5 +- tests/test_aristo/test_helpers.nim | 2 +- tests/test_aristo/test_merge.nim | 11 +- tests/test_aristo/test_transcode.nim | 24 +- 30 files changed, 2111 insertions(+), 444 deletions(-) delete mode 100644 nimbus/db/aristo/aristo_desc/aristo_types_private.nim create mode 100644 nimbus/db/aristo/aristo_init/aristo_init_common.nim create mode 100644 nimbus/db/aristo/aristo_init/aristo_rocksdb.nim create mode 100644 nimbus/db/aristo/aristo_init/aristo_rocksdb/rdb_desc.nim create mode 100644 nimbus/db/aristo/aristo_init/aristo_rocksdb/rdb_get.nim create mode 100644 nimbus/db/aristo/aristo_init/aristo_rocksdb/rdb_init.nim create mode 100644 nimbus/db/aristo/aristo_init/aristo_rocksdb/rdb_put.nim create mode 100644 nimbus/db/aristo/aristo_init/aristo_rocksdb/rdb_walk.nim create mode 100644 nimbus/db/aristo/aristo_layer.nim create mode 100644 tests/test_aristo/test_backend.nim diff --git a/nimbus/db/aristo/aristo_debug.nim b/nimbus/db/aristo/aristo_debug.nim index 24f65f1058..9bfa0395fd 100644 --- a/nimbus/db/aristo/aristo_debug.nim +++ b/nimbus/db/aristo/aristo_debug.nim @@ -14,8 +14,8 @@ import std/[algorithm, sequtils, sets, strutils, tables], eth/[common, trie/nibbles], stew/byteutils, - "."/[aristo_constants, aristo_desc, aristo_hike, aristo_vid], - ./aristo_desc/aristo_types_private + "."/[aristo_constants, aristo_desc, aristo_hike, aristo_init, aristo_vid], + ./aristo_init/[aristo_memory, aristo_rocksdb] # ------------------------------------------------------------------------------ # Ptivate functions @@ -34,7 +34,7 @@ proc sortedKeys(pPrf: HashSet[VertexID]): seq[VertexID] = pPrf.toSeq.mapIt(it.uint64).sorted.mapIt(it.VertexID) proc toPfx(indent: int; offset = 0): string = - if 0 < indent: "\n" & " ".repeat(indent+offset) else: "" + if 0 < indent+offset: "\n" & " ".repeat(indent+offset) else: "" proc labelVidUpdate(db: var AristoDb, lbl: HashLabel, vid: VertexID): string = if lbl.key.isValid and vid.isValid: @@ -70,8 +70,13 @@ proc squeeze(s: string; hex = false; ignLen = false): string = proc stripZeros(a: string): string = a.strip(leading=true, trailing=false, chars={'0'}).toLowerAscii -proc ppVid(vid: VertexID): string = - if vid.isValid: "$" & vid.uint64.toHex.stripZeros.toLowerAscii else: "$ø" +proc ppVid(vid: VertexID; pfx = true): string = + if pfx: + result = "$" + if vid.isValid: + result &= vid.uint64.toHex.stripZeros.toLowerAscii + else: + result &= "ø" proc vidCode(lbl: HashLabel, db: AristoDb): uint64 = if lbl.isValid: @@ -109,11 +114,11 @@ proc ppLabel(lbl: HashLabel; db: AristoDb): string = if not db.top.isNil: let vid = db.top.pAmk.getOrVoid lbl if vid.isValid: - return "£" & rid & vid.ppVid + return "£" & rid & vid.ppVid(pfx=false) block: let vid = db.xMap.getOrVoid lbl if vid.isValid: - return "£" & rid & vid.ppVid + return "£" & rid & vid.ppVid(pfx=false) "%" & rid & lbl.key.ByteArray32 .mapIt(it.toHex(2)).join.tolowerAscii @@ -192,17 +197,14 @@ proc ppXMap*( .filterIt(1 < it[1]).toTable proc ppNtry(n: uint64): string = + var s = "(" & VertexID(n).ppVid let lbl = kMap.getOrVoid VertexID(n) - var s = "(" & VertexID(n).ppVid & "," if lbl.isValid: - s &= lbl.ppLabel(db) - let vid = pAmk.getOrVoid lbl - if vid.isValid: - s &= ",ø" + if not vid.isValid: + s &= "," & lbl.ppLabel(db) & ",ø" elif vid != VertexID(n): - s &= "," & vid.ppVid - + s &= "," & lbl.ppLabel(db) & "," & vid.ppVid let count = dups.getOrDefault(VertexID(n), 0) if 0 < count: s &= ",*" & $count @@ -247,6 +249,20 @@ proc ppXMap*( else: result &= "}" +proc ppBe[T](be: T; db: AristoDb; indent: int): string = + ## Walk over backend tables + let pfx = indent.toPfx + result = "<" & $be.kind & ">" + result &= pfx & "vGen" & pfx & " [" & be.walkIdg.toSeq.mapIt( + it[2].pp + ).join(",") & "]" + result &= pfx & "sTab" & pfx & " {" & be.walkVtx.toSeq.mapIt( + $(1+it[0]) & "(" & it[1].ppVid & "," & it[2].ppVtx(db,it[1]) & ")" + ).join("," & pfx & " ") & "}" + result &= pfx & "kMap" & pfx & " {" & be.walkKey.toSeq.mapIt( + $(1+it[0]) & "(" & it[1].ppVid & "," & it[2].ppKey & ")" + ).join("," & pfx & " ") & "}" + # ------------------------------------------------------------------------------ # Public functions # ------------------------------------------------------------------------------ @@ -276,8 +292,8 @@ proc pp*(lty: LeafTie, db = AristoDb()): string = proc pp*(vid: VertexID): string = vid.ppVid -proc pp*(vid: openArray[VertexID]): string = - "[" & vid.mapIt(it.ppVid).join(",") & "]" +proc pp*(vGen: openArray[VertexID]): string = + "[" & vGen.mapIt(it.ppVid).join(",") & "]" proc pp*(p: PayloadRef, db = AristoDb()): string = p.ppPayload(db) @@ -337,9 +353,6 @@ proc pp*(lTab: Table[LeafTie,VertexID]; indent = 4): string = .mapIt("(" & it[0].ppLeafTie(db) & "," & it[1].ppVid & ")") .join("," & indent.toPfx(1)) & "}" -proc pp*(vGen: seq[VertexID]): string = - "[" & vGen.mapIt(it.ppVid).join(",") & "]" - proc pp*(pPrf: HashSet[VertexID]): string = "{" & pPrf.sortedKeys.mapIt(it.ppVid).join(",") & "}" @@ -347,8 +360,17 @@ proc pp*(leg: Leg; db = AristoDb()): string = result = "(" & leg.wp.vid.ppVid & "," if not db.top.isNil: let lbl = db.top.kMap.getOrVoid leg.wp.vid - result &= (if lbl.isValid: lbl.ppLabel(db) else: "ø") - result &= "," & $leg.nibble.ppNibble & "," & leg.wp.vtx.pp(db) & ")" + if not lbl.isValid: + result &= "ø" + elif leg.wp.vid != db.top.pAmk.getOrVoid lbl: + result &= lbl.ppLabel(db) + result &= "," + if leg.backend: + result &= "*" + result &= "," + if 0 <= leg.nibble: + result &= $leg.nibble.ppNibble + result &= "," & leg.wp.vtx.pp(db) & ")" proc pp*(hike: Hike; db = AristoDb(); indent = 4): string = let pfx = indent.toPfx(1) @@ -388,6 +410,13 @@ proc pp*(kMap: Table[VertexID,Hashlabel]; db: AristoDb; indent = 4): string = proc pp*(pAmk: Table[Hashlabel,VertexID]; db: AristoDb; indent = 4): string = db.ppXMap(db.top.kMap, pAmk, indent) +proc pp*( + be: MemBackendRef|RdbBackendRef; + db: AristoDb; + indent = 4; + ): string = + be.ppBe(db, indent) + # --------------------- proc pp*( @@ -395,45 +424,56 @@ proc pp*( sTabOk = true; lTabOk = true; kMapOk = true; - dKeyOk = true; pPrfOk = true; indent = 4; ): string = let - pfx1 = max(indent-1,0).toPfx - pfx2 = indent.toPfx - labelOk = 1 < sTabOk.ord + lTabOk.ord + kMapOk.ord + dKeyOk.ord + pPrfOk.ord + pfx1 = indent.toPfx + pfx2 = indent.toPfx(1) + tagOk = 1 < sTabOk.ord + lTabOk.ord + kMapOk.ord + pPrfOk.ord var - pfy1 = "" - pfy2 = "" + pfy = "" proc doPrefix(s: string): string = var rc: string - if labelOk: - rc = pfy1 & s & pfx2 - pfy1 = pfx1 + if tagOk: + rc = pfy & s & pfx2 + pfy = pfx1 else: - rc = pfy2 - pfy2 = pfx2 + rc = pfy + pfy = pfx2 rc if not db.top.isNil: if sTabOk: let info = "sTab(" & $db.top.sTab.len & ")" - result &= info.doPrefix & db.top.sTab.pp(db,indent) + result &= info.doPrefix & db.top.sTab.pp(db,indent+1) if lTabOk: let info = "lTab(" & $db.top.lTab.len & ")" - result &= info.doPrefix & db.top.lTab.pp(indent) + result &= info.doPrefix & db.top.lTab.pp(indent+1) if kMapOk: let info = "kMap(" & $db.top.kMap.len & "," & $db.top.pAmk.len & ")" - result &= info.doPrefix & db.ppXMap(db.top.kMap,db.top.pAmk,indent) - if dKeyOk: - let info = "dKey(" & $db.top.dkey.len & ")" - result &= info.doPrefix & db.top.dKey.pp + result &= info.doPrefix & db.ppXMap(db.top.kMap,db.top.pAmk,indent+1) if pPrfOk: let info = "pPrf(" & $db.top.pPrf.len & ")" result &= info.doPrefix & db.top.pPrf.pp +proc pp*( + be: AristoTypedBackendRef; + db: AristoDb; + indent = 4; + ): string = + + case (if be.isNil: BackendNone else: be.kind) + of BackendMemory: + be.MemBackendRef.ppBe(db, indent) + + of BackendRocksDB: + be.RdbBackendRef.ppBe(db, indent) + + of BackendNone: + "n/a" + # ------------------------------------------------------------------------------ # End # ------------------------------------------------------------------------------ diff --git a/nimbus/db/aristo/aristo_delete.nim b/nimbus/db/aristo/aristo_delete.nim index e00a40a6a0..e6b3170406 100644 --- a/nimbus/db/aristo/aristo_delete.nim +++ b/nimbus/db/aristo/aristo_delete.nim @@ -41,11 +41,11 @@ proc clearKey(db: AristoDb; vid: VertexID) = db.top.pAmk.del key elif db.getKeyBackend(vid).isOK: # Register for deleting on backend - db.top.dKey.incl vid + db.top.kMap[vid] = VOID_HASH_LABEL + db.top.pAmk.del key proc doneWith(db: AristoDb; vid: VertexID) = # Remove entry - db.top.dKey.excl vid # No need to register for deleting on backend db.vidDispose vid # Will be propagated to backend db.top.sTab.del vid let key = db.top.kMap.getOrVoid vid @@ -111,7 +111,7 @@ proc deleteImpl( # No need to keep it any longer db.top.lTab.del lty else: - # To be deleted in backend when it is updated + # To be recorded on change history db.top.lTab[lty] = VertexID(0) ok() diff --git a/nimbus/db/aristo/aristo_desc.nim b/nimbus/db/aristo/aristo_desc.nim index c64ad6f803..0d23dc2320 100644 --- a/nimbus/db/aristo/aristo_desc.nim +++ b/nimbus/db/aristo/aristo_desc.nim @@ -41,7 +41,6 @@ type sTab*: Table[VertexID,VertexRef] ## Structural vertex table lTab*: Table[LeafTie,VertexID] ## Direct access, path to leaf vertex kMap*: Table[VertexID,HashLabel] ## Merkle hash key mapping - dKey*: HashSet[VertexID] ## Locally deleted Merkle hash keys pAmk*: Table[HashLabel,VertexID] ## Reverse mapper for data import pPrf*: HashSet[VertexID] ## Locked vertices (proof nodes) vGen*: seq[VertexID] ## Unique vertex ID generator @@ -70,21 +69,38 @@ proc getOrVoid*[W](tab: Table[W,VertexID]; w: W): VertexID = # -------- -proc isValid*(vtx: VertexRef): bool = +func isValid*(vtx: VertexRef): bool = vtx != VertexRef(nil) -proc isValid*(nd: NodeRef): bool = +func isValid*(nd: NodeRef): bool = nd != NodeRef(nil) -proc isValid*(key: HashKey): bool = +func isValid*(key: HashKey): bool = key != VOID_HASH_KEY -proc isValid*(lbl: HashLabel): bool = +func isValid*(lbl: HashLabel): bool = lbl != VOID_HASH_LABEL -proc isValid*(vid: VertexID): bool = +func isValid*(vid: VertexID): bool = vid != VertexID(0) +# ------------------------------------------------------------------------------ +# Public functions, miscellaneous +# ------------------------------------------------------------------------------ + +# Note that the below `init()` function cannot go into +# `aristo_types_identifiers` as this would result in a circular import. + +func init*(key: var HashKey; data: openArray[byte]): bool = + ## Import argument `data` into `key` which must have length either `32`, or + ## `0`. The latter case is equivalent to an all zero byte array of size `32`. + if data.len == 32: + (addr key.ByteArray32[0]).copyMem(unsafeAddr data[0], data.len) + return true + if data.len == 0: + key = VOID_HASH_KEY + return true + # ------------------------------------------------------------------------------ # End # ------------------------------------------------------------------------------ diff --git a/nimbus/db/aristo/aristo_desc/aristo_error.nim b/nimbus/db/aristo/aristo_desc/aristo_error.nim index 4f637d0ace..90bf2fda65 100644 --- a/nimbus/db/aristo/aristo_desc/aristo_error.nim +++ b/nimbus/db/aristo/aristo_desc/aristo_error.nim @@ -23,35 +23,27 @@ type RlpRlpException RlpOtherException - # Db record decoder, `blobify()` - DbrNilArgument - DbrUnknown - DbrTooShort - DbrBranchTooShort - DbrBranchSizeGarbled - DbrBranchInxOutOfRange - DbrExtTooShort - DbrExtSizeGarbled - DbrExtGotLeafPrefix - DbrLeafSizeGarbled - DbrLeafGotExtPrefix - - # Db admin data decoder, `deblobify()` - ADbGarbledSize - ADbWrongType - - # Db record encoder, `blobify()` - VtxExPathOverflow - VtxLeafPathOverflow + # Data record transcoders, `deblobify()` and `blobify()` + BlobifyVtxExPathOverflow + BlobifyVtxLeafPathOverflow + + DeblobNilArgument + DeblobUnknown + DeblobTooShort + DeblobBranchTooShort + DeblobBranchSizeGarbled + DeblobBranchInxOutOfRange + DeblobExtTooShort + DeblobExtSizeGarbled + DeblobExtGotLeafPrefix + DeblobLeafSizeGarbled + DeblobLeafGotExtPrefix + DeblobSizeGarbled + DeblobWrongType # Converter `asNode()`, currenly for unit tests only CacheMissingNodekeys - # Get function `getVtxCascaded()` - GetVtxNotFound - GetTagNotFound - GetKeyNotFound - # Path function `hikeUp()` PathRootMissing PathLeafTooEarly @@ -60,10 +52,6 @@ type PathExtTailEmpty PathExtTailMismatch - # Memory backend - MemBeVtxNotFound - MemBeKeyNotFound - # Path/nibble/key conversions in `aisto_path.nim` PathExpected64Nibbles PathExpectedLeaf @@ -82,11 +70,13 @@ type MergeNonBranchProofModeLock MergeRootBranchLinkBusy - MergeHashKeyEmpty + MergeHashKeyInvalid + MergeRootVidInvalid + MergeRootKeyInvalid + MergeRevVidMustHaveBeenCached MergeHashKeyCachedAlready MergeHashKeyDiffersFromCached - MergeRootKeyEmpty - + MergeNodeVtxDiffersFromExisting MergeRootKeyDiffersForVid # Update `Merkle` hashes `hashify()` @@ -132,6 +122,26 @@ type DelExtLocked # Save permanently, `save()` - BackendMissing + SaveBackendMissing + + # Get functions form `aristo_get.nim` + GetLeafNotFound + + # All backend and get functions form `aristo_get.nim` + GetVtxNotFound + GetKeyNotFound + + # RocksDB backend + RdbBeCantCreateDataDir + RdbBeCantCreateBackupDir + RdbBeCantCreateTmpDir + RdbBeDriverInitError + RdbBeDriverGetError + RdbBeDriverDelError + RdbBeCreateSstWriter + RdbBeOpenSstWriter + RdbBeAddSstWriter + RdbBeFinishSstWriter + RdbBeIngestSstWriter # End diff --git a/nimbus/db/aristo/aristo_desc/aristo_types_backend.nim b/nimbus/db/aristo/aristo_desc/aristo_types_backend.nim index b931c5fcad..f8b532de3a 100644 --- a/nimbus/db/aristo/aristo_desc/aristo_types_backend.nim +++ b/nimbus/db/aristo/aristo_desc/aristo_types_backend.nim @@ -21,18 +21,18 @@ import type GetVtxFn* = proc(vid: VertexID): Result[VertexRef,AristoError] {.gcsafe, raises: [].} - ## Generic backend database retrieval function for a single structural - ## `Aristo DB` data record. + ## Generic backend database retrieval function for a single structural + ## `Aristo DB` data record. GetKeyFn* = proc(vid: VertexID): Result[HashKey,AristoError] {.gcsafe, raises: [].} - ## Generic backend database retrieval function for a single - ## `Aristo DB` hash lookup value. + ## Generic backend database retrieval function for a single + ## `Aristo DB` hash lookup value. GetIdgFn* = proc(): Result[seq[VertexID],AristoError] {.gcsafe, raises: [].} - ## Generic backend database retrieval function for a the ID generator - ## `Aristo DB` state record. + ## Generic backend database retrieval function for a the ID generator + ## `Aristo DB` state record. # ------------- @@ -49,16 +49,20 @@ type PutVtxFn* = proc(hdl: PutHdlRef; vrps: openArray[(VertexID,VertexRef)]) {.gcsafe, raises: [].} - ## Generic backend database bulk storage function. + ## Generic backend database bulk storage function, `VertexRef(nil)` + ## values indicate that records should be deleted. PutKeyFn* = proc(hdl: PutHdlRef; vkps: openArray[(VertexID,HashKey)]) {.gcsafe, raises: [].} - ## Generic backend database bulk storage function. + ## Generic backend database bulk storage function, `VOID_HASH_KEY` + ## values indicate that records should be deleted. PutIdgFn* = - proc(hdl: PutHdlRef; vs: openArray[VertexID]) {.gcsafe, raises: [].} - ## Generic backend database ID generator state storage function. + proc(hdl: PutHdlRef; vs: openArray[VertexID]) + {.gcsafe, raises: [].} + ## Generic backend database ID generator state storage function. This + ## function replaces the current generator state. PutEndFn* = proc(hdl: PutHdlRef): AristoError {.gcsafe, raises: [].} @@ -66,21 +70,14 @@ type # ------------- - DelVtxFn* = - proc(vids: openArray[VertexID]) - {.gcsafe, raises: [].} - ## Generic backend database delete function for the structural - ## `Aristo DB` data records - - DelKeyFn* = - proc(vids: openArray[VertexID]) - {.gcsafe, raises: [].} - ## Generic backend database delete function for the `Aristo DB` - ## Merkle hash key mappings. - - # ------------- + CloseFn* = + proc(flush: bool) {.gcsafe, raises: [].} + ## Generic destructor for the `Aristo DB` backend. The argument `flush` + ## indicates that a full database deletion is requested. If passed + ## `false` the outcome might differ depending on the type of backend + ## (e.g. in-memory backends would flush on close.) - AristoBackendRef* = ref object + AristoBackendRef* = ref object of RootRef ## Backend interface. getVtxFn*: GetVtxFn ## Read vertex record getKeyFn*: GetKeyFn ## Read Merkle hash/key @@ -90,8 +87,7 @@ type putKeyFn*: PutKeyFn ## Bulk store vertex hashes putIdgFn*: PutIdgFn ## Store ID generator state putEndFn*: PutEndFn ## Commit bulk store session - delVtxFn*: DelVtxFn ## Bulk delete vertex records - delKeyFn*: DelKeyFn ## Bulk delete vertex Merkle hashes + closeFn*: CloseFn ## Generic destructor # ------------------------------------------------------------------------------ # End diff --git a/nimbus/db/aristo/aristo_desc/aristo_types_identifiers.nim b/nimbus/db/aristo/aristo_desc/aristo_types_identifiers.nim index 75304e0704..9c5f178afe 100644 --- a/nimbus/db/aristo/aristo_desc/aristo_types_identifiers.nim +++ b/nimbus/db/aristo/aristo_desc/aristo_types_identifiers.nim @@ -15,12 +15,14 @@ {.push raises: [].} import - std/[strutils, hashes], + std/[sequtils, strutils, hashes], eth/[common, trie/nibbles], - stint, - ./aristo_types_private + stint type + ByteArray32* = array[32,byte] + ## Used for 32 byte hash components repurposed as Merkle hash labels. + VertexID* = distinct uint64 ## Unique identifier for a vertex of the `Aristo Trie`. The vertex is the ## prefix tree (aka `Patricia Trie`) component. When augmented by hash @@ -72,75 +74,75 @@ static: # Public helpers: `VertexID` scalar data model # ------------------------------------------------------------------------------ -proc `<`*(a, b: VertexID): bool {.borrow.} -proc `==`*(a, b: VertexID): bool {.borrow.} -proc cmp*(a, b: VertexID): int {.borrow.} -proc `$`*(a: VertexID): string = $a.uint64 +func `<`*(a, b: VertexID): bool {.borrow.} +func `==`*(a, b: VertexID): bool {.borrow.} +func cmp*(a, b: VertexID): int {.borrow.} +func `$`*(a: VertexID): string = $a.uint64 -proc `==`*(a: VertexID; b: static[uint]): bool = +func `==`*(a: VertexID; b: static[uint]): bool = a == VertexID(b) # ------------------------------------------------------------------------------ # Public helpers: `HashID` scalar data model # ------------------------------------------------------------------------------ -proc u256*(lp: HashID): UInt256 = lp.UInt256 -proc low*(T: type HashID): T = low(UInt256).T -proc high*(T: type HashID): T = high(UInt256).T +func u256*(lp: HashID): UInt256 = lp.UInt256 +func low*(T: type HashID): T = low(UInt256).T +func high*(T: type HashID): T = high(UInt256).T -proc `+`*(a: HashID; b: UInt256): HashID = (a.u256+b).HashID -proc `-`*(a: HashID; b: UInt256): HashID = (a.u256-b).HashID -proc `-`*(a, b: HashID): UInt256 = (a.u256 - b.u256) +func `+`*(a: HashID; b: UInt256): HashID = (a.u256+b).HashID +func `-`*(a: HashID; b: UInt256): HashID = (a.u256-b).HashID +func `-`*(a, b: HashID): UInt256 = (a.u256 - b.u256) -proc `==`*(a, b: HashID): bool = a.u256 == b.u256 -proc `<=`*(a, b: HashID): bool = a.u256 <= b.u256 -proc `<`*(a, b: HashID): bool = a.u256 < b.u256 +func `==`*(a, b: HashID): bool = a.u256 == b.u256 +func `<=`*(a, b: HashID): bool = a.u256 <= b.u256 +func `<`*(a, b: HashID): bool = a.u256 < b.u256 -proc cmp*(x, y: HashID): int = cmp(x.UInt256, y.UInt256) +func cmp*(x, y: HashID): int = cmp(x.UInt256, y.UInt256) # ------------------------------------------------------------------------------ # Public helpers: Conversions between `HashID`, `HashKey`, `Hash256` # ------------------------------------------------------------------------------ -proc to*(hid: HashID; T: type Hash256): T = +func to*(hid: HashID; T: type Hash256): T = result.data = hid.UInt256.toBytesBE -proc to*(hid: HashID; T: type HashKey): T = +func to*(hid: HashID; T: type HashKey): T = hid.UInt256.toBytesBE.T -proc to*(key: HashKey; T: type HashID): T = +func to*(key: HashKey; T: type HashID): T = UInt256.fromBytesBE(key.ByteArray32).T -proc to*(key: HashKey; T: type Hash256): T = +func to*(key: HashKey; T: type Hash256): T = T(data: ByteArray32(key)) -proc to*(hash: Hash256; T: type HashKey): T = +func to*(hash: Hash256; T: type HashKey): T = hash.data.T -proc to*(key: Hash256; T: type HashID): T = +func to*(key: Hash256; T: type HashID): T = key.data.HashKey.to(T) # ------------------------------------------------------------------------------ # Public helpers: Miscellaneous mappings # ------------------------------------------------------------------------------ -proc to*(key: HashKey; T: type Blob): T = +func to*(key: HashKey; T: type Blob): T = ## Representation of a `HashKey` as `Blob` (preserving full information) key.ByteArray32.toSeq -proc to*(key: HashKey; T: type NibblesSeq): T = +func to*(key: HashKey; T: type NibblesSeq): T = ## Representation of a `HashKey` as `NibbleSeq` (preserving full information) key.ByteArray32.initNibbleRange() -proc to*(hid: HashID; T: type NibblesSeq): T = +func to*(hid: HashID; T: type NibblesSeq): T = ## Representation of a `HashKey` as `NibbleSeq` (preserving full information) ByteArray32(hid.to(HashKey)).initNibbleRange() -proc to*(n: SomeUnsignedInt|UInt256; T: type HashID): T = +func to*(n: SomeUnsignedInt|UInt256; T: type HashID): T = ## Representation of a scalar as `HashID` (preserving full information) n.u256.T -proc digestTo*(data: Blob; T: type HashKey): T = +func digestTo*(data: Blob; T: type HashKey): T = ## Keccak hash of a `Blob`, represented as a `HashKey` keccakHash(data).data.T @@ -148,39 +150,39 @@ proc digestTo*(data: Blob; T: type HashKey): T = # Public helpers: `Tables` and `Rlp` support # ------------------------------------------------------------------------------ -proc hash*(a: HashID): Hash = +func hash*(a: HashID): Hash = ## Table/KeyedQueue mixin a.to(HashKey).ByteArray32.hash -proc hash*(a: HashKey): Hash = +func hash*(a: HashKey): Hash = ## Table/KeyedQueue mixin a.ByteArray32.hash -proc `==`*(a, b: HashKey): bool = +func `==`*(a, b: HashKey): bool = ## Table/KeyedQueue mixin a.ByteArray32 == b.ByteArray32 -proc read*[T: HashID|HashKey](rlp: var Rlp, W: type T): T +func read*[T: HashID|HashKey](rlp: var Rlp, W: type T): T {.gcsafe, raises: [RlpError].} = rlp.read(Hash256).to(T) -proc append*(writer: var RlpWriter, val: HashID|HashKey) = +func append*(writer: var RlpWriter, val: HashID|HashKey) = writer.append(val.to(Hash256)) # ------------------------------------------------------------------------------ # Public helpers: `LeafTie` scalar data model # ------------------------------------------------------------------------------ -proc `<`*(a, b: LeafTie): bool = +func `<`*(a, b: LeafTie): bool = a.root < b.root or (a.root == b.root and a.path < b.path) -proc `==`*(a, b: LeafTie): bool = +func `==`*(a, b: LeafTie): bool = a.root == b.root and a.path == b.path -proc cmp*(a, b: LeafTie): int = +func cmp*(a, b: LeafTie): int = if a < b: -1 elif a == b: 0 else: 1 -proc `$`*(a: LeafTie): string = +func `$`*(a: LeafTie): string = let w = $a.root.uint64.toHex & ":" & $a.path.Uint256.toHex w.strip(leading=true, trailing=false, chars={'0'}).toLowerAscii @@ -188,7 +190,7 @@ proc `$`*(a: LeafTie): string = # Miscellaneous helpers # ------------------------------------------------------------------------------ -proc `$`*(hid: HashID): string = +func `$`*(hid: HashID): string = if hid == high(HashID): "2^256-1" elif hid == 0.u256.HashID: @@ -204,7 +206,7 @@ proc `$`*(hid: HashID): string = else: hid.UInt256.toHex -proc `$`*(key: HashKey): string = +func `$`*(key: HashKey): string = $key.to(HashID) # ------------------------------------------------------------------------------ diff --git a/nimbus/db/aristo/aristo_desc/aristo_types_private.nim b/nimbus/db/aristo/aristo_desc/aristo_types_private.nim deleted file mode 100644 index 0d7ca8bf39..0000000000 --- a/nimbus/db/aristo/aristo_desc/aristo_types_private.nim +++ /dev/null @@ -1,15 +0,0 @@ -# nimbus-eth1 -# Copyright (c) 2021 Status Research & Development GmbH -# Licensed under either of -# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or -# http://www.apache.org/licenses/LICENSE-2.0) -# * MIT license ([LICENSE-MIT](LICENSE-MIT) or -# http://opensource.org/licenses/MIT) -# at your option. This file may not be copied, modified, or distributed -# except according to those terms. - -type - ByteArray32* = array[32,byte] - ## Used for 32 byte hash components repurposed as Merkle hash labels. - -# End diff --git a/nimbus/db/aristo/aristo_get.nim b/nimbus/db/aristo/aristo_get.nim index c8195cf8f9..72f8d3199e 100644 --- a/nimbus/db/aristo/aristo_get.nim +++ b/nimbus/db/aristo/aristo_get.nim @@ -14,7 +14,7 @@ {.push raises: [].} import - std/sets, + std/tables, stew/results, "."/aristo_desc @@ -35,7 +35,6 @@ proc getVtxBackend*( let be = db.backend if not be.isNil: return be.getVtxFn vid - err(GetVtxNotFound) proc getKeyBackend*( @@ -43,37 +42,12 @@ proc getKeyBackend*( vid: VertexID; ): Result[HashKey,AristoError] = ## Get the merkle hash/key from the backend - # key must not have been locally deleted (but not saved, yet) - if vid notin db.top.dKey: - let be = db.backend - if not be.isNil: - return be.getKeyFn vid - + let be = db.backend + if not be.isNil: + return be.getKeyFn vid err(GetKeyNotFound) - -proc getVtxCascaded*( - db: AristoDb; - vid: VertexID; - ): Result[VertexRef,AristoError] = - ## Get the vertex from the top layer or the `backened` layer if available. - let vtx = db.top.sTab.getOrVoid vid - if vtx.isValid: - return ok vtx - - db.getVtxBackend vid - -proc getKeyCascaded*( - db: AristoDb; - vid: VertexID; - ): Result[HashKey,AristoError] = - ## Get the Merkle hash/key from the top layer or the `backened` layer if - ## available. - let lbl = db.top.kMap.getOrVoid vid - if lbl.isValid: - return ok lbl.key - - db.getKeyBackend vid +# ------------------ proc getLeaf*( db: AristoDb; @@ -82,31 +56,50 @@ proc getLeaf*( ## Get the vertex from the top layer by the `Patricia Trie` path. This ## function does not search on the `backend` layer. let vid = db.top.lTab.getOrVoid lty - if vid.isValid: - let vtx = db.top.sTab.getOrVoid vid - if vtx.isValid: - return ok VidVtxPair(vid: vid, vtx: vtx) + if not vid.isValid: + return err(GetLeafNotFound) - err(GetTagNotFound) - -# --------- + let vtx = db.top.sTab.getOrVoid vid + if not vtx.isValid: + return err(GetVtxNotFound) -proc getVtx*(db: AristoDb; vid: VertexID): VertexRef = - ## Variant of `getVtxCascaded()` returning `nil` on error (while - ## ignoring the detailed error type information.) - db.getVtxCascaded(vid).get(otherwise = VertexRef(nil)) + ok VidVtxPair(vid: vid, vtx: vtx) -proc getVtx*(db: AristoDb; lty: LeafTie): VertexRef = - ## Variant of `getLeaf()` returning `nil` on error (while - ## ignoring the detailed error type information.) +proc getLeafVtx*(db: AristoDb; lty: LeafTie): VertexRef = + ## Variant of `getLeaf()` returning `nil` on error (while ignoring the + ## detailed error type information.) + ## let rc = db.getLeaf lty if rc.isOk: return rc.value.vtx - + +# ------------------ + +proc getVtx*(db: AristoDb; vid: VertexID): VertexRef = + ## Cascaded attempt to fetch a vertex from the top layer or the backend. + ## The function returns `nil` on error or failure. + ## + if db.top.sTab.hasKey vid: + # If the vertex is to be deleted on the backend, a `VertexRef(nil)` entry + # is kept in the local table in which case it is OK to return this value. + return db.top.sTab.getOrVoid vid + let rc = db.getVtxBackend vid + if rc.isOk: + return rc.value + VertexRef(nil) + proc getKey*(db: AristoDb; vid: VertexID): HashKey = - ## Variant of `getKeyCascaded()` returning `VOID_HASH_KEY` on error (while - ## ignoring the detailed error type information.) - db.getKeyCascaded(vid).get(otherwise = VOID_HASH_KEY) + ## Cascaded attempt to fetch a Merkle hash from the top layer or the backend. + ## The function returns `VOID_HASH_KEY` on error or failure. + ## + if db.top.kMap.hasKey vid: + # If the key is to be deleted on the backend, a `VOID_HASH_LABEL` entry + # is kept on the local table in which case it is OK to return this value. + return db.top.kMap.getOrVoid(vid).key + let rc = db.getKeyBackend vid + if rc.isOk: + return rc.value + VOID_HASH_KEY # ------------------------------------------------------------------------------ # End diff --git a/nimbus/db/aristo/aristo_hashify.nim b/nimbus/db/aristo/aristo_hashify.nim index 2a8c7a27b4..0abc51eb01 100644 --- a/nimbus/db/aristo/aristo_hashify.nim +++ b/nimbus/db/aristo/aristo_hashify.nim @@ -42,6 +42,7 @@ {.push raises: [].} import + std/[algorithm, sequtils, strutils], std/[sets, tables], chronicles, eth/common, @@ -49,9 +50,50 @@ import "."/[aristo_constants, aristo_debug, aristo_desc, aristo_get, aristo_hike, aristo_transcode, aristo_vid] +type + BackVidValRef = ref object + root: VertexID ## Root vertex + onBe: bool ## Table key vid refers to backend + toVid: VertexID ## Next/follow up vertex + + BackVidTab = + Table[VertexID,BackVidValRef] + logScope: topics = "aristo-hashify" +# ------------------------------------------------------------------------------ +# Private helpers +# ------------------------------------------------------------------------------ + +template logTxt(info: static[string]): static[string] = + "Hashify " & info + +func getOrVoid(tab: BackVidTab; vid: VertexID): BackVidValRef = + tab.getOrDefault(vid, BackVidValRef(nil)) + +func isValid(brv: BackVidValRef): bool = + brv != BackVidValRef(nil) + +# ------------------------------------------------------------------------------ +# Private helper, debugging +# ------------------------------------------------------------------------------ + +proc pp(w: BackVidValRef): string = + if w.isNil: + return "n/a" + result = "(" & w.root.pp & "," + if w.onBe: + result &= "*" + result &= "," & w.toVid.pp & ")" + +proc pp(t: BackVidTab): string = + proc pp(b: bool): string = + if b: "*" else: "" + "{" & t.keys.toSeq.mapIt(it.uint64).sorted.mapIt(it.VertexID) + .mapIt("(" & it.pp & "," & t.getOrVoid(it).pp & ")") + .join(",") & "}" + # ------------------------------------------------------------------------------ # Private functions # ------------------------------------------------------------------------------ @@ -80,6 +122,59 @@ proc toNode(vtx: VertexRef; db: AristoDb): Result[NodeRef,void] = node.key[0] = key return ok node + +proc updateHashKey( + db: AristoDb; # Database, top layer + root: VertexID; # Root ID + vid: VertexID; # Vertex ID to check for + expected: HashKey; # Hash key for vertex address by `vid` + backend: bool; # Set `true` id vertex is on backend + ): Result[void,AristoError] = + ## Update the argument hash key `expected` for the vertex addressed by `vid`. + ## + # If the Merkle hash has been cached locally, already it must match. + block: + let key = db.top.kMap.getOrVoid(vid).key + if key.isValid: + if key != expected: + let error = HashifyExistingHashMismatch + debug logTxt "hash update failed", vid, key, expected, error + return err(error) + return ok() + + # If the vertex had been cached locally, there would be no locally cached + # Merkle hash key. It will be created at the bottom end of the function. + # + # So there remains tha case when vertex is available on the backend only. + # The Merkle hash not cached locally. It might be overloaded (and eventually + # overwitten.) + if backend: + # Ok, vertex is on the backend. + let rc = db.getKeyBackend vid + if rc.isOk: + let key = rc.value + if key == expected: + return ok() + + # This step is a error in the sense that something the on the backend + # is fishy. There should not be contradicting Merkle hashes. Throwing + # an error heres would lead to a deadlock so we correct it. + debug "correcting backend hash key mismatch", vid, key, expected + # Proceed `vidAttach()`, below + + elif rc.error != GetKeyNotFound: + debug logTxt "backend key fetch failed", vid, expected, error=rc.error + return err(rc.error) + + else: + discard + # Proceed `vidAttach()`, below + + # Othwise there is no Merkle hash, so create one with the `expected` key + db.vidAttach(HashLabel(root: root, key: expected), vid) + ok() + + proc leafToRootHasher( db: AristoDb; # Database, top layer hike: Hike; # Hike for labelling leaf..root @@ -88,6 +183,7 @@ proc leafToRootHasher( for n in (hike.legs.len-1).countDown(0): let wp = hike.legs[n].wp + bg = hike.legs[n].backend rc = wp.vtx.toNode db if rc.isErr: return ok n @@ -99,13 +195,9 @@ proc leafToRootHasher( # Check against existing key, or store new key let key = rc.value.encode.digestTo(HashKey) - vfy = db.getKey wp.vid - if not vfy.isValid: - db.vidAttach(HashLabel(root: hike.root, key: key), wp.vid) - elif key != vfy: - let error = HashifyExistingHashMismatch - debug "hashify failed", vid=wp.vid, key, expected=vfy, error - return err((wp.vid,error)) + rx = db.updateHashKey(hike.root, wp.vid, key, bg) + if rx.isErr: + return err((wp.vid,rx.error)) ok -1 # all could be hashed @@ -121,7 +213,6 @@ proc hashifyClear*( if not locksOnly: db.top.pAmk.clear db.top.kMap.clear - db.top.dKey.clear db.top.pPrf.clear @@ -136,8 +227,8 @@ proc hashify*( completed: HashSet[VertexID] # Width-first leaf-to-root traversal structure - backLink: Table[VertexID,(VertexID,VertexID)] - downMost: Table[VertexID,(VertexID,VertexID)] + backLink: BackVidTab + downMost: BackVidTab for (lky,vid) in db.top.lTab.pairs: let hike = lky.hikeUp(db) @@ -148,7 +239,7 @@ proc hashify*( # Hash as much of the `hike` as possible let n = block: - let rc = db.leafToRootHasher(hike) + let rc = db.leafToRootHasher hike if rc.isErr: return err(rc.error) rc.value @@ -163,9 +254,15 @@ proc hashify*( # | | | # | backLink[] | downMost | # - downMost[hike.legs[n].wp.vid] = (hike.root, hike.legs[n-1].wp.vid) + downMost[hike.legs[n].wp.vid] = BackVidValRef( + root: hike.root, + onBe: hike.legs[n].backend, + toVid: hike.legs[n-1].wp.vid) for u in (n-1).countDown(1): - backLink[hike.legs[u].wp.vid] = (hike.root, hike.legs[u-1].wp.vid) + backLink[hike.legs[u].wp.vid] = BackVidValRef( + root: hike.root, + onBe: hike.legs[u].backend, + toVid: hike.legs[u-1].wp.vid) elif n < 0: completed.incl hike.root @@ -178,41 +275,33 @@ proc hashify*( # Update remaining hashes while 0 < downMost.len: var - redo: Table[VertexID,(VertexID,VertexID)] + redo: BackVidTab done: HashSet[VertexID] - for (fromVid,rootAndVid) in downMost.pairs: + for (vid,val) in downMost.pairs: # Try to convert vertex to a node. This is possible only if all link # references have Merkle hashes. # - # Also `db.getVtx(fromVid)` => not nil as it was fetched earlier, already - let rc = db.getVtx(fromVid).toNode(db) + # Also `db.getVtx(vid)` => not nil as it was fetched earlier, already + let rc = db.getVtx(vid).toNode(db) if rc.isErr: # Cannot complete with this vertex, so do it later - redo[fromVid] = rootAndVid + redo[vid] = val else: - # Register Hashes + # Update Merkle hash let - hashKey = rc.value.encode.digestTo(HashKey) - toVid = rootAndVid[1] - - # Update Merkle hash (aka `HashKey`) - let fromLbl = db.top.kMap.getOrVoid fromVid - if fromLbl.isValid: - db.vidAttach(HashLabel(root: rootAndVid[0], key: hashKey), fromVid) - elif hashKey != fromLbl.key: - let error = HashifyExistingHashMismatch - debug "hashify failed", vid=fromVid, key=hashKey, - expected=fromLbl.key.pp, error - return err((fromVid,error)) + key = rc.value.encode.digestTo(HashKey) + rx = db.updateHashKey(val.root, vid, key, val.onBe) + if rx.isErr: + return err((vid,rx.error)) - done.incl fromVid + done.incl vid # Proceed with back link - let nextItem = backLink.getOrDefault(toVid, (VertexID(0),VertexID(0))) - if nextItem[1].isValid: - redo[toVid] = nextItem + let nextItem = backLink.getOrVoid val.toVid + if nextItem.isValid: + redo[val.toVid] = nextItem # Make sure that the algorithm proceeds if done.len == 0: diff --git a/nimbus/db/aristo/aristo_hike.nim b/nimbus/db/aristo/aristo_hike.nim index 6e12bb25f7..e4b9e38a3a 100644 --- a/nimbus/db/aristo/aristo_hike.nim +++ b/nimbus/db/aristo/aristo_hike.nim @@ -12,6 +12,7 @@ import eth/[common, trie/nibbles], + stew/results, "."/[aristo_desc, aristo_get] type @@ -19,6 +20,7 @@ type ## For constructing a `VertexPath` wp*: VidVtxPair ## Vertex ID and data ref nibble*: int8 ## Next vertex selector for `Branch` (if any) + backend*: bool ## Sources from backend if `true` Hike* = object ## Trie traversal path @@ -74,15 +76,22 @@ proc hikeUp*( else: var vid = root while vid.isValid: - var vtx = db.getVtx vid - if not vtx.isValid: - break + var leg = Leg(wp: VidVtxPair(vid: vid), nibble: -1) - var leg = Leg(wp: VidVtxPair(vid: vid, vtx: vtx), nibble: -1) + # Fetch vertex to be checked on this lap + leg.wp.vtx = db.top.sTab.getOrVoid vid + if not leg.wp.vtx.isValid: - case vtx.vType: + # Register vertex fetched from backend (if any) + let rc = db.getVtxBackend vid + if rc.isErr: + break + leg.backend = true + leg.wp.vtx = rc.value + + case leg.wp.vtx.vType: of Leaf: - if result.tail.len == result.tail.sharedPrefixLen(vtx.lPfx): + if result.tail.len == result.tail.sharedPrefixLen(leg.wp.vtx.lPfx): # Bingo, got full path result.legs.add leg result.tail = EmptyNibbleSeq @@ -98,7 +107,7 @@ proc hikeUp*( let nibble = result.tail[0].int8 - nextVid = vtx.bVid[nibble] + nextVid = leg.wp.vtx.bVid[nibble] if not nextVid.isValid: result.error = PathBranchBlindEdge # Ooops @@ -116,13 +125,13 @@ proc hikeUp*( result.error = PathExtTailEmpty # Well, somehow odd break - if vtx.ePfx.len != result.tail.sharedPrefixLen(vtx.ePfx): + if leg.wp.vtx.ePfx.len != result.tail.sharedPrefixLen(leg.wp.vtx.ePfx): result.error = PathExtTailMismatch # Need to branch from here break result.legs.add leg - result.tail = result.tail.slice(vtx.ePfx.len) - vid = vtx.eVid + result.tail = result.tail.slice(leg.wp.vtx.ePfx.len) + vid = leg.wp.vtx.eVid proc hikeUp*(lty: LeafTie; db: AristoDb): Hike = ## Variant of `hike()` diff --git a/nimbus/db/aristo/aristo_init.nim b/nimbus/db/aristo/aristo_init.nim index d854dcb383..cbe3b043d1 100644 --- a/nimbus/db/aristo/aristo_init.nim +++ b/nimbus/db/aristo/aristo_init.nim @@ -11,33 +11,78 @@ ## Constructors for Aristo DB ## ========================== ## -## For a backend-less constructor use `AristoDb(top: AristoLayerRef())`. {.push raises: [].} import - ./aristo_init/[aristo_memory], + stew/results, + ./aristo_init/[aristo_init_common, aristo_memory, aristo_rocksdb], ./aristo_desc, - ./aristo_desc/aristo_types_private + ./aristo_desc/aristo_types_backend + +export + AristoBackendType, AristoStorageType, AristoTypedBackendRef # ------------------------------------------------------------------------------ -# Public functions +# Public database constuctors, destructor # ------------------------------------------------------------------------------ -proc init*(key: var HashKey; data: openArray[byte]): bool = - ## Import argument `data` into `key` which must have length either `32`, or - ## `0`. The latter case is equivalent to an all zero byte array of size `32`. - if data.len == 32: - (addr key.ByteArray32[0]).copyMem(unsafeAddr data[0], data.len) - return true - if data.len == 0: - key = VOID_HASH_KEY - return true - -proc init*(T: type AristoDb): T = - ## Constructor with memory backend. - T(top: AristoLayerRef(), - backend: memoryBackend()) +proc init*( + T: type AristoDb; + backend: static[AristoBackendType]; + basePath: string; + ): Result[T, AristoError] = + ## Generic constructor, `basePath` argument is ignored for `BackendNone` and + ## `BackendMemory` type backend database. Also, both of these backends + ## aways succeed initialising. + when backend == BackendNone: + ok T(top: AristoLayerRef()) + + elif backend == BackendMemory: + ok T(top: AristoLayerRef(), backend: memoryBackend()) + + elif backend == BackendRocksDB: + let rc = rocksDbBackend basePath + if rc.isErr: + return err(rc.error) + ok T(top: AristoLayerRef(), backend: rc.value) + + else: + {.error: "Unknown/unsupported Aristo DB backend".} + +proc init*( + T: type AristoDb; + backend: static[AristoBackendType]; + ): T = + ## Simplified prototype for `BackendNone` and `BackendMemory` type backend. + when backend == BackendNone: + T(top: AristoLayerRef()) + + elif backend == BackendMemory: + T(top: AristoLayerRef(), backend: memoryBackend()) + + elif backend == BackendRocksDB: + {.error: "Aristo DB backend \"BackendRocksDB\" needs basePath argument".} + + else: + {.error: "Unknown/unsupported Aristo DB backend".} + +# ----------------- + +proc finish*(db: var AristoDb; flush = false) = + ## backend destructor. The argument `flush` indicates that a full database + ## deletion is requested. If set ot left `false` the outcome might differ + ## depending on the type of backend (e.g. the `BackendMemory` backend will + ## always flush on close.) + if not db.backend.isNil: + db.backend.closeFn flush + db.top = AristoLayerRef(nil) + db.stack.setLen(0) + + +proc to*[W: MemBackendRef|RdbBackendRef](db: AristoDb; T: type W): T = + ## Handy helper for lew-level access to some backend functionality + db.backend.T # ------------------------------------------------------------------------------ # End diff --git a/nimbus/db/aristo/aristo_init/aristo_init_common.nim b/nimbus/db/aristo/aristo_init/aristo_init_common.nim new file mode 100644 index 0000000000..635e887fc4 --- /dev/null +++ b/nimbus/db/aristo/aristo_init/aristo_init_common.nim @@ -0,0 +1,65 @@ +# nimbus-eth1 +# Copyright (c) 2021 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. +{.push raises: [].} + +import + ../aristo_desc/aristo_types_backend + +const + verifyIxId = true # and false + ## Enforce session tracking + +type + AristoBackendType* = enum + BackendNone ## For providing backend-less constructor + BackendMemory + BackendRocksDB + + AristoTypedBackendRef* = ref object of AristoBackendRef + kind*: AristoBackendType ## Backend type identifier + when verifyIxId: + txGen: uint ## Transaction ID generator (for debugging) + txId: uint ## Active transaction ID (for debugging) + + TypedPutHdlRef* = ref object of PutHdlRef + when verifyIxId: + txId: uint ## Transaction ID (for debugging) + + AristoStorageType* = enum + ## Storage types, key prefix + IdgPfx = 0 ## ID generator + VtxPfx = 1 ## Vertex data + KeyPfx = 2 ## Key/hash data + +# ------------------------------------------------------------------------------ +# Public helpers +# ------------------------------------------------------------------------------ + +proc beginSession*(hdl: TypedPutHdlRef; db: AristoTypedBackendRef) = + when verifyIxId: + doAssert db.txId == 0 + if db.txGen == 0: + db.txGen = 1 + db.txId = db.txGen + hdl.txId = db.txGen + db.txGen.inc + +proc verifySession*(hdl: TypedPutHdlRef; db: AristoTypedBackendRef) = + when verifyIxId: + doAssert db.txId == hdl.txId + +proc finishSession*(hdl: TypedPutHdlRef; db: AristoTypedBackendRef) = + when verifyIxId: + doAssert db.txId == hdl.txId + db.txId = 0 + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/nimbus/db/aristo/aristo_init/aristo_memory.nim b/nimbus/db/aristo/aristo_init/aristo_memory.nim index c30b19050d..314fb46fb6 100644 --- a/nimbus/db/aristo/aristo_init/aristo_memory.nim +++ b/nimbus/db/aristo/aristo_init/aristo_memory.nim @@ -10,31 +10,63 @@ ## In-memory backend for Aristo DB ## =============================== - +## +## The iterators provided here are currently available only by direct +## backend access +## :: +## import +## aristo/aristo_init, +## aristo/aristo_init/aristo_memory +## +## let rc = AristoDb.init(BackendMemory) +## if rc.isOk: +## let be = rc.value.to(MemBackendRef) +## for (n, key, vtx) in be.walkVtx: +## ... +## {.push raises: [].} import - std/[sequtils, tables], + std/[algorithm, sequtils, tables], + eth/common, stew/results, + ../aristo_constants, ../aristo_desc, - ../aristo_desc/aristo_types_backend + ../aristo_desc/aristo_types_backend, + ../aristo_transcode, + ./aristo_init_common type - MemBackendRef = ref object + MemBackendRef* = ref object of AristoTypedBackendRef + ## Inheriting table so access can be extended for debugging purposes sTab: Table[VertexID,VertexRef] ## Structural vertex table making up a trie kMap: Table[VertexID,HashKey] ## Merkle hash key mapping vGen: seq[VertexID] - txGen: uint ## Transaction ID generator (for debugging) - txId: uint ## Active transaction ID (for debugging) - MemPutHdlRef = ref object of PutHdlRef - txId: uint ## Transaction ID (for debugging) + MemPutHdlRef = ref object of TypedPutHdlRef + sTab: Table[VertexID,VertexRef] + kMap: Table[VertexID,HashKey] + vGen: seq[VertexID] + vGenOk: bool + +# ------------------------------------------------------------------------------ +# Private helpers +# ------------------------------------------------------------------------------ -const - VerifyIxId = true # for debugging +proc newSession(db: MemBackendRef): MemPutHdlRef = + new result + result.TypedPutHdlRef.beginSession db + +proc getSession(hdl: PutHdlRef; db: MemBackendRef): MemPutHdlRef = + hdl.TypedPutHdlRef.verifySession db + hdl.MemPutHdlRef + +proc endSession(hdl: PutHdlRef; db: MemBackendRef): MemPutHdlRef = + hdl.TypedPutHdlRef.finishSession db + hdl.MemPutHdlRef # ------------------------------------------------------------------------------ -# Private functions +# Private functions: interface # ------------------------------------------------------------------------------ proc getVtxFn(db: MemBackendRef): GetVtxFn = @@ -43,7 +75,7 @@ proc getVtxFn(db: MemBackendRef): GetVtxFn = let vtx = db.sTab.getOrDefault(vid, VertexRef(nil)) if vtx != VertexRef(nil): return ok vtx - err(MemBeVtxNotFound) + err(GetVtxNotFound) proc getKeyFn(db: MemBackendRef): GetKeyFn = result = @@ -51,7 +83,7 @@ proc getKeyFn(db: MemBackendRef): GetKeyFn = let key = db.kMap.getOrDefault(vid, VOID_HASH_KEY) if key != VOID_HASH_KEY: return ok key - err(MemBeKeyNotFound) + err(GetKeyNotFound) proc getIdgFn(db: MemBackendRef): GetIdgFn = result = @@ -63,78 +95,126 @@ proc getIdgFn(db: MemBackendRef): GetIdgFn = proc putBegFn(db: MemBackendRef): PutBegFn = result = proc(): PutHdlRef = - when VerifyIxId: - doAssert db.txId == 0 - db.txGen.inc - MemPutHdlRef(txId: db.txGen) + db.newSession() proc putVtxFn(db: MemBackendRef): PutVtxFn = result = proc(hdl: PutHdlRef; vrps: openArray[(VertexID,VertexRef)]) = - when VerifyIxId: - doAssert db.txId == hdl.MemPutHdlRef.txId + let hdl = hdl.getSession db for (vid,vtx) in vrps: - db.sTab[vid] = vtx + hdl.sTab[vid] = vtx proc putKeyFn(db: MemBackendRef): PutKeyFn = result = proc(hdl: PutHdlRef; vkps: openArray[(VertexID,HashKey)]) = - when VerifyIxId: - doAssert db.txId == hdl.MemPutHdlRef.txId + let hdl = hdl.getSession db for (vid,key) in vkps: - db.kMap[vid] = key + hdl.kMap[vid] = key proc putIdgFn(db: MemBackendRef): PutIdgFn = result = proc(hdl: PutHdlRef; vs: openArray[VertexID]) = - when VerifyIxId: - doAssert db.txId == hdl.MemPutHdlRef.txId - db.vGen = vs.toSeq + let hdl = hdl.getSession db + hdl.vGen = vs.toSeq + hdl.vGenOk = true proc putEndFn(db: MemBackendRef): PutEndFn = result = proc(hdl: PutHdlRef): AristoError = - when VerifyIxId: - doAssert db.txId == hdl.MemPutHdlRef.txId - db.txId = 0 + let hdl = hdl.endSession db + + for (vid,vtx) in hdl.sTab.pairs: + if vtx.isValid: + db.sTab[vid] = vtx + else: + db.sTab.del vid + + for (vid,key) in hdl.kMap.pairs: + if key.isValid: + db.kMap[vid] = key + else: + db.kMap.del vid + + if hdl.vGenOk: + db.vGen = hdl.vGen AristoError(0) # ------------- -proc delVtxFn(db: MemBackendRef): DelVtxFn = +proc closeFn(db: MemBackendRef): CloseFn = result = - proc(vids: openArray[VertexID]) = - for vid in vids: - db.sTab.del vid - -proc delKeyFn(db: MemBackendRef): DelKeyFn = - result = - proc(vids: openArray[VertexID]) = - for vid in vids: - db.kMap.del vid + proc(ignore: bool) = + discard # ------------------------------------------------------------------------------ # Public functions # ------------------------------------------------------------------------------ proc memoryBackend*(): AristoBackendRef = - let db = MemBackendRef() + let db = MemBackendRef(kind: BackendMemory) + + db.getVtxFn = getVtxFn db + db.getKeyFn = getKeyFn db + db.getIdgFn = getIdgFn db + + db.putBegFn = putBegFn db + db.putVtxFn = putVtxFn db + db.putKeyFn = putKeyFn db + db.putIdgFn = putIdgFn db + db.putEndFn = putEndFn db - AristoBackendRef( - getVtxFn: getVtxFn db, - getKeyFn: getKeyFn db, - getIdgFn: getIdgFn db, + db.closeFn = closeFn db - putBegFn: putBegFn db, - putVtxFn: putVtxFn db, - putKeyFn: putKeyFn db, - putIdgFn: putIdgFn db, - putEndFn: putEndFn db, + db + +# ------------------------------------------------------------------------------ +# Public iterators (needs direct backend access) +# ------------------------------------------------------------------------------ - delVtxFn: delVtxFn db, - delKeyFn: delKeyFn db) +iterator walkIdg*( + be: MemBackendRef; + ): tuple[n: int, vid: VertexID, vGen: seq[VertexID]] = + ## Iteration over the ID generator sub-table (there is at most one instance). + if 0 < be.vGen.len: + yield(0, VertexID(0), be.vGen) + +iterator walkVtx*( + be: MemBackendRef; + ): tuple[n: int, vid: VertexID, vtx: VertexRef] = + ## Iteration over the vertex sub-table. + for n,vid in be.sTab.keys.toSeq.mapIt(it.uint64).sorted.mapIt(it.VertexID): + let vtx = be.sTab.getOrVoid vid + if vtx.isValid: + yield (n, vid, vtx) + +iterator walkKey*( + be: MemBackendRef; + ): tuple[n: int, vid: VertexID, key: HashKey] = + ## Iteration over the Markle hash sub-table. + for n,vid in be.kMap.keys.toSeq.mapIt(it.uint64).sorted.mapIt(it.VertexID): + let key = be.kMap.getOrDefault(vid, VOID_HASH_KEY) + if key.isValid: + yield (n, vid, key) + +iterator walk*( + be: MemBackendRef; + ): tuple[n: int, pfx: AristoStorageType, vid: VertexID, data: Blob] = + ## Walk over all key-value pairs of the database. + ## + ## Non-decodable entries are stepped over while the counter `n` of the + ## yield record is still incremented. + for (n,vid,vGen) in be.walkIdg: + yield (n, IdgPfx, vid, vGen.blobify) + + for (n,vid,vtx) in be.walkVtx: + let rc = vtx.blobify + if rc.isOk: + yield (n, VtxPfx, vid, rc.value) + + for (n,vid,key) in be.walkKey: + yield (n, KeyPfx, vid, key.to(Blob)) # ------------------------------------------------------------------------------ # End diff --git a/nimbus/db/aristo/aristo_init/aristo_rocksdb.nim b/nimbus/db/aristo/aristo_init/aristo_rocksdb.nim new file mode 100644 index 0000000000..f95ec64630 --- /dev/null +++ b/nimbus/db/aristo/aristo_init/aristo_rocksdb.nim @@ -0,0 +1,274 @@ +# nimbus-eth1 +# Copyright (c) 2021 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +## Rocksdb backend for Aristo DB +## ============================= +## +## The iterators provided here are currently available only by direct +## backend access +## :: +## import +## aristo/aristo_init, +## aristo/aristo_init/aristo_rocksdb +## +## let rc = AristoDb.init(BackendRocksDB, "/var/tmp") +## if rc.isOk: +## let be = rc.value.to(RdbBackendRef) +## for (n, key, vtx) in be.walkVtx: +## ... +## +{.push raises: [].} + +import + chronicles, + eth/common, + rocksdb, + stew/results, + ".."/[aristo_desc, aristo_transcode], + ../aristo_desc/aristo_types_backend, + ./aristo_rocksdb/[rdb_desc, rdb_get, rdb_init, rdb_put, rdb_walk], + ./aristo_init_common + +logScope: + topics = "aristo-backend" + +type + RdbBackendRef* = ref object of AristoTypedBackendRef + rdb: RdbInst ## Allows low level access to database + + RdbPutHdlErr = tuple + pfx: AristoStorageType ## Error sub-table + vid: VertexID ## Vertex ID where the error occured + code: AristoError ## Error code (if any) + + RdbPutHdlRef = ref object of TypedPutHdlRef + cache: RdbTabs ## Tranaction cache + error: RdbPutHdlErr ## Track error while collecting transaction + +const + extraTraceMessages = false or true + ## Enabled additional logging noise + + # ---------- + + maxOpenFiles = 512 ## Rocks DB setup, open files limit + +# ------------------------------------------------------------------------------ +# Private helpers +# ------------------------------------------------------------------------------ + +template logTxt(info: static[string]): static[string] = + "RocksDB " & info + + +proc newSession(db: RdbBackendRef): RdbPutHdlRef = + new result + result.TypedPutHdlRef.beginSession db + +proc getSession(hdl: PutHdlRef; db: RdbBackendRef): RdbPutHdlRef = + hdl.TypedPutHdlRef.verifySession db + hdl.RdbPutHdlRef + +proc endSession(hdl: PutHdlRef; db: RdbBackendRef): RdbPutHdlRef = + hdl.TypedPutHdlRef.finishSession db + hdl.RdbPutHdlRef + +# ------------------------------------------------------------------------------ +# Private functions: interface +# ------------------------------------------------------------------------------ + +proc getVtxFn(db: RdbBackendRef): GetVtxFn = + result = + proc(vid: VertexID): Result[VertexRef,AristoError] = + + # Fetch serialised data record + let rc = db.rdb.get vid.toOpenArray(VtxPfx) + if rc.isErr: + debug logTxt "getVtxFn() failed", vid, + error=rc.error[0], info=rc.error[1] + return err(rc.error[0]) + + # Decode data record + if 0 < rc.value.len: + return rc.value.deblobify VertexRef + + err(GetVtxNotFound) + +proc getKeyFn(db: RdbBackendRef): GetKeyFn = + result = + proc(vid: VertexID): Result[HashKey,AristoError] = + + # Fetch serialised data record + let rc = db.rdb.get vid.toOpenArray(KeyPfx) + if rc.isErr: + debug logTxt "getKeyFn: failed", vid, + error=rc.error[0], info=rc.error[1] + return err(rc.error[0]) + + # Decode data record + var key: HashKey + if key.init rc.value: + return ok key + + err(GetKeyNotFound) + +proc getIdgFn(db: RdbBackendRef): GetIdgFn = + result = + proc(): Result[seq[VertexID],AristoError]= + + # Fetch serialised data record + let rc = db.rdb.get VertexID(0).toOpenArray(IdgPfx) + if rc.isErr: + debug logTxt "getIdgFn: failed", error=rc.error[1] + return err(rc.error[0]) + + # Decode data record + return rc.value.deblobify seq[VertexID] + +# ------------- + +proc putBegFn(db: RdbBackendRef): PutBegFn = + result = + proc(): PutHdlRef = + db.newSession() + + +proc putVtxFn(db: RdbBackendRef): PutVtxFn = + result = + proc(hdl: PutHdlRef; vrps: openArray[(VertexID,VertexRef)]) = + let hdl = hdl.getSession db + if hdl.error.code == AristoError(0): + for (vid,vtx) in vrps: + if vtx.isValid: + let rc = vtx.blobify + if rc.isErr: + hdl.error = (VtxPfx, vid, rc.error) + return + hdl.cache[VtxPfx][vid] = rc.value + else: + hdl.cache[VtxPfx][vid] = EmptyBlob + +proc putKeyFn(db: RdbBackendRef): PutKeyFn = + result = + proc(hdl: PutHdlRef; vkps: openArray[(VertexID,HashKey)]) = + let hdl = hdl.getSession db + if hdl.error.code == AristoError(0): + for (vid,key) in vkps: + if key.isValid: + hdl.cache[KeyPfx][vid] = key.to(Blob) + else: + hdl.cache[KeyPfx][vid] = EmptyBlob + + +proc putIdgFn(db: RdbBackendRef): PutIdgFn = + result = + proc(hdl: PutHdlRef; vs: openArray[VertexID]) = + let hdl = hdl.getSession db + if hdl.error.code == AristoError(0): + if 0 < vs.len: + hdl.cache[IdgPfx][VertexID(0)] = vs.blobify + else: + hdl.cache[IdgPfx][VertexID(0)] = EmptyBlob + + +proc putEndFn(db: RdbBackendRef): PutEndFn = + result = + proc(hdl: PutHdlRef): AristoError = + let hdl = hdl.endSession db + if hdl.error.code != AristoError(0): + debug logTxt "putEndFn: failed", + pfx=hdl.error.pfx, vid=hdl.error.vid, error=hdl.error.code + return hdl.error.code + let rc = db.rdb.put hdl.cache + if rc.isErr: + when extraTraceMessages: + debug logTxt "putEndFn: failed", + error=rc.error[0], info=rc.error[1] + return rc.error[0] + AristoError(0) + + +proc closeFn(db: RdbBackendRef): CloseFn = + result = + proc(flush: bool) = + db.rdb.destroy(flush) + +# ------------------------------------------------------------------------------ +# Public functions +# ------------------------------------------------------------------------------ + +proc rocksDbBackend*(path: string): Result[AristoBackendRef,AristoError] = + let + db = RdbBackendRef(kind: BackendRocksDB) + rc = db.rdb.init(path, maxOpenFiles) + if rc.isErr: + when extraTraceMessages: + trace logTxt "constructor failed", + error=rc.error[0], info=rc.error[1] + return err(rc.error[0]) + + db.getVtxFn = getVtxFn db + db.getKeyFn = getKeyFn db + db.getIdgFn = getIdgFn db + + db.putBegFn = putBegFn db + db.putVtxFn = putVtxFn db + db.putKeyFn = putKeyFn db + db.putIdgFn = putIdgFn db + db.putEndFn = putEndFn db + + db.closeFn = closeFn db + + ok db + +# ------------------------------------------------------------------------------ +# Public iterators (needs direct backend access) +# ------------------------------------------------------------------------------ + +iterator walk*( + be: RdbBackendRef; + ): tuple[n: int, pfx: AristoStorageType, vid: VertexID, data: Blob] = + ## Walk over all key-value pairs of the database. + ## + ## Non-decodable entries are stepped over while the counter `n` of the + ## yield record is still incremented. + for w in be.rdb.walk: + yield w + +iterator walkIdg*( + be: RdbBackendRef; + ): tuple[n: int, vid: VertexID, vGen: seq[VertexID]] = + ## Variant of `walk()` iteration over the ID generator sub-table. + for (n, vid, data) in be.rdb.walk IdgPfx: + let rc = data.deblobify seq[VertexID] + if rc.isOk: + yield (n, vid, rc.value) + +iterator walkVtx*( + be: RdbBackendRef; + ): tuple[n: int, vid: VertexID, vtx: VertexRef] = + ## Variant of `walk()` iteration over the vertex sub-table. + for (n, vid, data) in be.rdb.walk VtxPfx: + let rc = data.deblobify VertexRef + if rc.isOk: + yield (n, vid, rc.value) + +iterator walkkey*( + be: RdbBackendRef; + ): tuple[n: int, vid: VertexID, key: HashKey] = + ## Variant of `walk()` iteration over the Markle hash sub-table. + for (n, vid, data) in be.rdb.walk KeyPfx: + var hashKey: HashKey + if hashKey.init data: + yield (n, vid, hashKey) + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/nimbus/db/aristo/aristo_init/aristo_rocksdb/rdb_desc.nim b/nimbus/db/aristo/aristo_init/aristo_rocksdb/rdb_desc.nim new file mode 100644 index 0000000000..d9b3d40da7 --- /dev/null +++ b/nimbus/db/aristo/aristo_init/aristo_rocksdb/rdb_desc.nim @@ -0,0 +1,60 @@ +# nimbus-eth1 +# Copyright (c) 2021 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +## Rocks DB internal driver descriptor +## =================================== + +{.push raises: [].} + +import + std/tables, + eth/common, + rocksdb, + stint, + ../../aristo_desc, + ../aristo_init_common.nim + +type + RdbInst* = object + store*: RocksDBInstance ## Rocks DB database handler + basePath*: string ## Database directory + + # Low level Rocks DB access for bulk store + envOpt*: rocksdb_envoptions_t + impOpt*: rocksdb_ingestexternalfileoptions_t + + RdbKey* = array[1 + sizeof VertexID, byte] + ## Sub-table key, + VertexID + + RdbTabs* = array[AristoStorageType,Table[VertexID,Blob]] + ## Combined table for caching data to be stored/updated + +const + BaseFolder* = "nimbus" # Same as for Legacy DB has "backups" + DataFolder* = "aristo" # Legacy DB has "data" + BackupFolder* = "history" # Legacy DB has "backups" + TempFolder* = "tmp" # Not used with legacy DB + SstCache* = "bulkput" # Rocks DB bulk load file name in temp folder + +# ------------------------------------------------------------------------------ +# Public functions +# ------------------------------------------------------------------------------ + +proc toRdbKey*(vid: VertexID; pfx: AristoStorageType): Rdbkey = + let vidKey = vid.uint64.toBytesBE + result[0] = pfx.ord.byte + copyMem(addr result[1], unsafeAddr vidKey, sizeof vidKey) + +template toOpenArray*(vid: VertexID; pfx: AristoStorageType): openArray[byte] = + vid.toRdbKey(pfx).toOpenArray(0, sizeof VertexID) + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/nimbus/db/aristo/aristo_init/aristo_rocksdb/rdb_get.nim b/nimbus/db/aristo/aristo_init/aristo_rocksdb/rdb_get.nim new file mode 100644 index 0000000000..167ce4220f --- /dev/null +++ b/nimbus/db/aristo/aristo_init/aristo_rocksdb/rdb_get.nim @@ -0,0 +1,43 @@ +# nimbus-eth1 +# Copyright (c) 2021 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +## Rocks DB fetch data record +## ========================== + +{.push raises: [].} + +import + eth/common, + rocksdb, + stew/results, + "../.."/[aristo_constants, aristo_desc], + ./rdb_desc + +# ------------------------------------------------------------------------------ +# Public functions +# ------------------------------------------------------------------------------ + +proc get*( + rdb: RdbInst; + key: openArray[byte], + ): Result[Blob,(AristoError,string)] = + var res: Blob + let onData: DataProc = proc(data: openArray[byte]) = + res = @data + let rc = rdb.store.get(key, onData) + if rc.isErr: + return err((RdbBeDriverGetError,rc.error)) + if not rc.value: + res = EmptyBlob + ok res + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/nimbus/db/aristo/aristo_init/aristo_rocksdb/rdb_init.nim b/nimbus/db/aristo/aristo_init/aristo_rocksdb/rdb_init.nim new file mode 100644 index 0000000000..8f70ed7174 --- /dev/null +++ b/nimbus/db/aristo/aristo_init/aristo_rocksdb/rdb_init.nim @@ -0,0 +1,107 @@ +# nimbus-eth1 +# Copyright (c) 2021 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +## Rocksdb constructor/destructor for Aristo DB +## ============================================ + +{.push raises: [].} + +import + std/os, + chronicles, + rocksdb, + stew/results, + ../../aristo_desc, + ./rdb_desc + +logScope: + topics = "aristo-backend" + +# ------------------------------------------------------------------------------ +# Private helpers +# ------------------------------------------------------------------------------ + +template logTxt(info: static[string]): static[string] = + "RocksDB/init " & info + +# ------------------------------------------------------------------------------ +# Public constructor +# ------------------------------------------------------------------------------ + +proc init*( + rdb: var RdbInst; + basePath: string; + openMax: int; + ): Result[void,(AristoError,string)] = + ## Constructor c ode inspired by `RocksStoreRef.init()` from + ## kvstore_rocksdb.nim + let + dataDir = basePath / BaseFolder / DataFolder + backupsDir = basePath / BaseFolder / BackupFolder + tmpDir = basePath / BaseFolder / TempFolder + + try: + dataDir.createDir + except OSError, IOError: + return err((RdbBeCantCreateDataDir, "")) + try: + backupsDir.createDir + except OSError, IOError: + return err((RdbBeCantCreateBackupDir, "")) + try: + tmpDir.createDir + except OSError, IOError: + return err((RdbBeCantCreateTmpDir, "")) + + let rc = rdb.store.init( + dbPath=dataDir, dbBackuppath=backupsDir, readOnly=false, + maxOpenFiles=openMax) + if rc.isErr: + let error = RdbBeDriverInitError + debug logTxt "driver failed", dataDir, backupsDir, openMax, + error, info=rc.error + return err((RdbBeDriverInitError, rc.error)) + + # The following is a default setup (subject to change) + rdb.impOpt = rocksdb_ingestexternalfileoptions_create() + rdb.envOpt = rocksdb_envoptions_create() + + rdb.basePath = basePath + ok() + + +proc destroy*(rdb: var RdbInst; flush: bool) = + ## Destructor + rdb.envOpt.rocksdb_envoptions_destroy() + rdb.impOpt.rocksdb_ingestexternalfileoptions_destroy() + rdb.store.close() + + let + base = rdb.basePath / BaseFolder + try: + (base / TempFolder).removeDir + + if flush: + (base / DataFolder).removeDir + + # Remove the base folder if it is empty + block done: + for w in base.walkDirRec: + # Ignore backup files + if 0 < w.len and w[^1] != '~': + break done + base.removeDir + + except CatchableError: + discard + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/nimbus/db/aristo/aristo_init/aristo_rocksdb/rdb_put.nim b/nimbus/db/aristo/aristo_init/aristo_rocksdb/rdb_put.nim new file mode 100644 index 0000000000..66aae42d81 --- /dev/null +++ b/nimbus/db/aristo/aristo_init/aristo_rocksdb/rdb_put.nim @@ -0,0 +1,194 @@ +# nimbus-eth1 +# Copyright (c) 2021 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +## Rocks DB store data record +## ========================== + +{.push raises: [].} + +import + std/[algorithm, os, sequtils, sets, tables], + chronicles, + eth/common, + rocksdb, + stew/results, + "../.."/[aristo_constants, aristo_desc], + ../aristo_init_common, + ./rdb_desc + +logScope: + topics = "aristo-backend" + +type + RdbPutSession = object + writer: rocksdb_sstfilewriter_t + sstPath: string + nRecords: int + +const + extraTraceMessages = false or true + ## Enable additional logging noise + +# ------------------------------------------------------------------------------ +# Private helpers +# ------------------------------------------------------------------------------ + +template logTxt(info: static[string]): static[string] = + "RocksDB/put " & info + +proc getFileSize(fileName: string): int64 = + var f: File + if f.open fileName: + defer: f.close + try: + result = f.getFileSize + except: + discard + +proc rmFileIgnExpt(fileName: string) = + try: + fileName.removeFile + except: + discard + +# ------------------------------------------------------------------------------ +# Private functions +# ------------------------------------------------------------------------------ + +proc destroy(rps: RdbPutSession) = + rps.writer.rocksdb_sstfilewriter_destroy() + rps.sstPath.rmFileIgnExpt + +proc begin( + rdb: var RdbInst; + ): Result[RdbPutSession,(AristoError,string)] = + ## Begin a new bulk load session storing data into a temporary cache file + ## `fileName`. When finished, this file will bi direcly imported into the + ## database. + var csError: cstring + + var session = RdbPutSession( + writer: rocksdb_sstfilewriter_create(rdb.envOpt, rdb.store.options), + sstPath: rdb.basePath / BaseFolder / TempFolder / SstCache) + + if session.writer.isNil: + return err((RdbBeCreateSstWriter, "Cannot create sst writer session")) + + session.sstPath.rmFileIgnExpt + + session.writer.rocksdb_sstfilewriter_open( + session.sstPath.cstring, addr csError) + if not csError.isNil: + session.destroy() + return err((RdbBeOpenSstWriter, $csError)) + + ok session + + +proc add( + session: var RdbPutSession; + key: openArray[byte]; + val: openArray[byte]; + ): Result[void,(AristoError,string)] = + ## Append a record to the SST file. Note that consecutive records must be + ## strictly increasing. + ## + ## This function is a wrapper around `rocksdb_sstfilewriter_add()` or + ## `rocksdb_sstfilewriter_put()` (stragely enough, there are two functions + ## with exactly the same impementation code.) + var csError: cstring + + session.writer.rocksdb_sstfilewriter_add( + cast[cstring](unsafeAddr key[0]), csize_t(key.len), + cast[cstring](unsafeAddr val[0]), csize_t(val.len), addr csError) + if not csError.isNil: + return err((RdbBeAddSstWriter, $csError)) + + session.nRecords.inc + ok() + + +proc commit( + rdb: var RdbInst; + session: RdbPutSession; + ): Result[void,(AristoError,string)] = + ## Commit collected and cached data to the database. This function implies + ## `destroy()` if successful. Otherwise `destroy()` must be called + ## explicitely, e.g. after error analysis. + var csError: cstring + + if 0 < session.nRecords: + session.writer.rocksdb_sstfilewriter_finish(addr csError) + if not csError.isNil: + return err((RdbBeFinishSstWriter, $csError)) + + rdb.store.db.rocksdb_ingest_external_file( + [session.sstPath].allocCStringArray, 1, rdb.impOpt, addr csError) + if not csError.isNil: + return err((RdbBeIngestSstWriter, $csError)) + + when extraTraceMessages: + let fileSize = session.sstPath.getFileSize + trace logTxt "finished sst", fileSize + + session.destroy() + ok() + +# ------------------------------------------------------------------------------ +# Public functions +# ------------------------------------------------------------------------------ + +proc put*( + rdb: var RdbInst; + tabs: RdbTabs, + ): Result[void,(AristoError,string)] = + + var session = block: + let rc = rdb.begin() + if rc.isErr: + return err(rc.error) + rc.value + + # Vertices with empty table values will be deleted + var delKey: HashSet[RdbKey] + + for pfx in low(AristoStorageType) .. high(AristoStorageType): + when extraTraceMessages: + trace logTxt "sub-table", pfx, nItems=tabs[pfx].len + + for vid in tabs[pfx].keys.toSeq.mapIt(it.uint64).sorted.mapIt(it.VertexID): + let + key = vid.toRdbKey pfx + val = tabs[pfx].getOrDefault(vid, EmptyBlob) + if val.len == 0: + delKey.incl key + else: + let rc = session.add(key, val) + if rc.isErr: + session.destroy() + return err(rc.error) + + block: + let rc = rdb.commit session + if rc.isErr: + trace logTxt "commit error", error=rc.error[0], info=rc.error[1] + return err(rc.error) + + # Delete vertices after successful updating veritces with non-zero values. + for key in delKey: + let rc = rdb.store.del key + if rc.isErr: + return err((RdbBeDriverDelError,rc.error)) + + ok() + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/nimbus/db/aristo/aristo_init/aristo_rocksdb/rdb_walk.nim b/nimbus/db/aristo/aristo_init/aristo_rocksdb/rdb_walk.nim new file mode 100644 index 0000000000..73480fcf54 --- /dev/null +++ b/nimbus/db/aristo/aristo_init/aristo_rocksdb/rdb_walk.nim @@ -0,0 +1,178 @@ +# nimbus-eth1 +# Copyright (c) 2021 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +## Rocks DB store data iterator +## ============================ +## +{.push raises: [].} + +import + std/sequtils, + eth/common, + rocksdb, + ../../aristo_desc, + ../aristo_init_common, + ./rdb_desc + +# ------------------------------------------------------------------------------ +# Private functions +# ------------------------------------------------------------------------------ + +func keyPfx(kData: cstring, kLen: csize_t): int = + if not kData.isNil and kLen == 1 + sizeof(VertexID): + kData.toOpenArrayByte(0,0)[0].int + else: + -1 + +func keyVid(kData: cstring, kLen: csize_t): VertexID = + if not kData.isNil and kLen == 1 + sizeof(VertexID): + var data = kData.toOpenArrayByte(1,int(kLen)-1).toSeq + return uint64.fromBytesBE(data).VertexID + +func to(vid: VertexID; T: type Blob): T = + vid.uint64.toBytesBE.toSeq + +func valBlob(vData: cstring, vLen: csize_t): Blob = + if not vData.isNil and 0 < vLen: + return vData.toOpenArrayByte(0,int(vLen)-1).toSeq + +# ------------------------------------------------------------------------------ +# Public iterators +# ------------------------------------------------------------------------------ + +iterator walk*( + rdb: RdbInst; + ): tuple[n: int, pfx: AristoStorageType, vid: VertexID, data: Blob] = + ## Walk over all key-value pairs of the database. + ## + ## Non-decodable entries are stepped over while the counter `n` of the + ## yield record is still incremented. + let rit = rdb.store.db.rocksdb_create_iterator(rdb.store.readOptions) + defer: rit.rocksdb_iter_destroy() + + rit.rocksdb_iter_seek_to_first() + var count = 0 + + while rit.rocksdb_iter_valid() != 0: + var kLen: csize_t + let kData = rit.rocksdb_iter_key(addr kLen) + + let pfx = kData.keyPfx(kLen) + if 0 <= pfx: + if high(AristoStorageType).ord < pfx: + break + + let vid = kData.keyVid(kLen) + if vid.isValid: + var vLen: csize_t + let vData = rit.rocksdb_iter_value(addr vLen) + + let val = vData.valBlob(vLen) + if 0 < val.len: + yield (count, pfx.AristoStorageType, vid, val) + + # Update Iterator (might overwrite kData/vdata) + rit.rocksdb_iter_next() + count.inc + # End while + +iterator walk*( + rdb: RdbInst; + pfx: AristoStorageType; + ): tuple[n: int, vid: VertexID, data: Blob] = + ## Walk over key-value pairs of the table referted to by the argument `pfx`. + ## + ## Non-decodable entries are stepped over while the counter `n` of the + ## yield record is still incremented. + let rit = rdb.store.db.rocksdb_create_iterator(rdb.store.readOptions) + defer: rit.rocksdb_iter_destroy() + + rit.rocksdb_iter_seek_to_first() + var + count = 0 + kLen: csize_t + kData: cstring + + block walkBody: + # Skip over admin records (if any) and advance to the key sub-table + if rit.rocksdb_iter_valid() == 0: + break walkBody + kData = rit.rocksdb_iter_key(addr kLen) + + case pfx: + of IdgPfx: + discard + of VtxPfx, KeyPfx: + # Skip over admin records until vertex sub-table reached + while kData.keyPfx(kLen) < VtxPfx.ord: + + # Update Iterator and fetch next item + rit.rocksdb_iter_next() + if rit.rocksdb_iter_valid() == 0: + break walkBody + kData = rit.rocksdb_iter_key(addr kLen) + # End while + + case pfx: + of IdgPfx, VtxPfx: + discard + of KeyPfx: + # Reposition search head to key sub-table + while kData.keyPfx(kLen) < KeyPfx.ord: + + # Move search head to the first Merkle hash entry by seeking the same + # vertex ID on the key table. This might skip over stale keys smaller + # than the current one. + let key = @[KeyPfx.ord.byte] & kData.keyVid(kLen).to(Blob) + rit.rocksdb_iter_seek(cast[cstring](unsafeAddr key[0]), csize_t(kLen)) + + # It is not clear what happens when the `key` does not exist. The guess + # is that nothing would happen and the interation will proceed at the + # next vertex position. + kData = rit.rocksdb_iter_key(addr kLen) + if KeyPfx.ord <= kData.keyPfx(kLen): + # OK, reached Merkle hash table + break + + # Update Iterator + rit.rocksdb_iter_next() + if rit.rocksdb_iter_valid() == 0: + break walkBody + kData = rit.rocksdb_iter_key(addr kLen) + # End while + + # Fetch sub-table data + while true: + let kPfx = kData.keyPfx(kLen) + if pfx.ord < kPfx: + break walkBody # done + + let vid = kData.keyVid(kLen) + if vid.isValid: + + # Fetch value data + var vLen: csize_t + let vData = rit.rocksdb_iter_value(addr vLen) + + let val = vData.valBlob(vLen) + if 0 < val.len: + yield (count, vid, val) + + # Update Iterator + rit.rocksdb_iter_next() + if rit.rocksdb_iter_valid() == 0: + break walkBody + kData = rit.rocksdb_iter_key(addr kLen) + count.inc + # End while + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/nimbus/db/aristo/aristo_layer.nim b/nimbus/db/aristo/aristo_layer.nim new file mode 100644 index 0000000000..7a16d149bf --- /dev/null +++ b/nimbus/db/aristo/aristo_layer.nim @@ -0,0 +1,129 @@ +# nimbus-eth1 +# Copyright (c) 2021 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +## Aristo DB -- Patricia Trie layer management +## =========================================== +## + +import + std/[sequtils, tables], + stew/results, + "."/[aristo_desc, aristo_get] + +type + DeltaHistoryRef* = ref object + ## Change history for backend saving + leafs*: Table[LeafTie,PayloadRef] ## Changed leafs after merge into backend + +# ------------------------------------------------------------------------------ +# Private functions +# ------------------------------------------------------------------------------ + +proc cpy(layer: AristoLayerRef): AristoLayerRef = + new result + result[] = layer[] + +# ------------------------------------------------------------------------------ +# Public functions +# ------------------------------------------------------------------------------ + +proc push*(db: var AristoDb) = + ## Save a copy of the current delta state on the stack of non-persistent + ## state layers. + db.stack.add db.top.cpy + + +proc pop*(db: var AristoDb; merge = true): AristoError = + ## Remove the current state later and merge it into the next layer if the + ## argument `merge` is `true`, and discard it otherwise. The next layer then + ## becomes the current state. + ## + ## Note that merging is sort of a virtual action because the top layer has + ## always the full latest non-persistent delta state. This implies that in + ## the case of *merging* just the first parent layer will be discarded. + ## + if 0 < db.stack.len: + if not merge: + # Roll back to parent layer state + db.top = db.stack[^1] + db.stack.setLen(db.stack.len-1) + + elif merge: + # Accept as-is (there is no parent layer) + discard + + elif db.backend.isNil: + db.top = AristoLayerRef() + + else: + # Initialise new top layer from the backend + let rc = db.backend.getIdgFn() + if rc.isErr: + return rc.error + db.top = AristoLayerRef(vGen: rc.value) + + AristoError(0) + + +proc save*( + db: var AristoDb; # Database to be updated + clear = true; # Clear current top level cache + ): Result[DeltaHistoryRef,AristoError] = + ## Save top layer into persistent database. There is no check whether the + ## current layer is fully consistent as a Merkle Patricia Tree. It is + ## advised to run `hashify()` on the top layer before calling `save()`. + ## + ## After successful storage, all parent layers are cleared. The top layer + ## is also cleared if the `clear` flag is set `true`. + ## + ## Upon successful return, the previous state of the backend data is returned + ## relative to the changes made. + ## + let be = db.backend + if be.isNil: + return err(SaveBackendMissing) + + let hst = DeltaHistoryRef() # Change history + + # Record changed `Leaf` nodes into the history table + for (lky,vid) in db.top.lTab.pairs: + if vid.isValid: + # Get previous payload for this vertex + let rc = db.getVtxBackend vid + if rc.isErr: + if rc.error != GetVtxNotFound: + return err(rc.error) # Stop + hst.leafs[lky] = PayloadRef(nil) # So this is a new leaf vertex + elif rc.value.vType == Leaf: + hst.leafs[lky] = rc.value.lData # Record previous payload + else: + hst.leafs[lky] = PayloadRef(nil) # Was re-puropsed as leaf vertex + else: + hst.leafs[lky] = PayloadRef(nil) # New leaf vertex + + # Save structural and other table entries + let txFrame = be.putBegFn() + be.putVtxFn(txFrame, db.top.sTab.pairs.toSeq) + be.putKeyFn(txFrame, db.top.kMap.pairs.toSeq.mapIt((it[0],it[1].key))) + be.putIdgFn(txFrame, db.top.vGen) + let w = be.putEndFn txFrame + if w != AristoError(0): + return err(w) + + # Delete stack and clear top + db.stack.setLen(0) + if clear: + db.top = AristoLayerRef(vGen: db.top.vgen) + + ok hst + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/nimbus/db/aristo/aristo_merge.nim b/nimbus/db/aristo/aristo_merge.nim index a4c081eb65..564b0621d8 100644 --- a/nimbus/db/aristo/aristo_merge.nim +++ b/nimbus/db/aristo/aristo_merge.nim @@ -25,7 +25,7 @@ {.push raises: [].} import - std/[sequtils, sets, tables], + std/[algorithm, sequtils, strutils, sets, tables], chronicles, eth/[common, trie/nibbles], stew/results, @@ -81,7 +81,8 @@ proc clearMerkleKeys( db.top.pAmk.del key elif db.getKeyBackend(vid).isOK: # Register for deleting on backend - db.top.dKey.incl vid + db.top.kMap[vid] = VOID_HASH_LABEL + db.top.pAmk.del key # ----------- @@ -200,6 +201,7 @@ proc insertBranch( wp: VidVtxPair( vid: extVtx.eVid, vtx: forkVtx)) + else: db.top.sTab[linkID] = forkVtx result.legs.add Leg( @@ -381,7 +383,6 @@ proc topIsEmptyAddLeaf( ## Append a `Leaf` vertex derived from the argument `payload` after the ## argument vertex `rootVtx` and append both the empty arguent `hike`. if rootVtx.vType == Branch: - let nibble = hike.tail[0].int8 if rootVtx.bVid[nibble].isValid: return Hike(error: MergeRootBranchLinkBusy) @@ -429,53 +430,69 @@ proc mergeNodeImpl( if node.error != AristoError(0): return err(node.error) if not rootVid.isValid: - return err(MergeRootKeyEmpty) + return err(MergeRootKeyInvalid) # Verify `hashKey` if not hashKey.isValid: - return err(MergeHashKeyEmpty) + return err(MergeHashKeyInvalid) - # Check whether the node exists, already. If not then create a new vertex ID + # Make sure that the `vid<->hashLbl` reverse mapping has been cached, + # already. This is provided for if the `nodes` are processed in the right + # order `root->.. ->leaf`. var hashLbl = HashLabel(root: rootVid, key: hashKey) vid = db.top.pAmk.getOrVoid hashLbl if not vid.isValid: - vid = db.vidAttach hashLbl - else: - let lbl = db.top.kMap.getOrVoid vid - if lbl == hashLbl: - if db.top.sTab.hasKey vid: - # This is tyically considered OK - return err(MergeHashKeyCachedAlready) - # Otherwise proceed - elif lbl.isValid: - # Different key assigned => error - return err(MergeHashKeyDiffersFromCached) - - let vtx = node.to(VertexRef) # the vertex IDs need to be set up now (if any) + return err(MergeRevVidMustHaveBeenCached) + + let lbl = db.top.kMap.getOrVoid vid + if lbl == hashLbl: + if db.top.sTab.hasKey vid: + # This is tyically considered OK + return err(MergeHashKeyCachedAlready) + # Otherwise proceed + elif lbl.isValid: + # Different key assigned => error + return err(MergeHashKeyDiffersFromCached) + + let (vtx, hasVtx) = block: + let vty = db.getVtx vid + if vty.isValid: + (vty, true) + else: + (node.to(VertexRef), false) + # The `vertexID <-> hashLabel` mappings need to be set up now (if any) case node.vType: of Leaf: discard of Extension: if node.key[0].isValid: - let - eLbl = HashLabel(root: rootVid, key: node.key[0]) - eVid = db.top.pAmk.getOrVoid eLbl - if eVid.isValid: - vtx.eVid = eVid + let eLbl = HashLabel(root: rootVid, key: node.key[0]) + if hasVtx: + if not vtx.eVid.isValid: + return err(MergeNodeVtxDiffersFromExisting) + db.top.pAmk[eLbl] = vtx.eVid else: - vtx.eVid = db.vidAttach eLbl + let eVid = db.top.pAmk.getOrVoid eLbl + if eVid.isValid: + vtx.eVid = eVid + else: + vtx.eVid = db.vidAttach eLbl of Branch: for n in 0..15: if node.key[n].isValid: - let - bLbl = HashLabel(root: rootVid, key: node.key[n]) - bVid = db.top.pAmk.getOrVoid bLbl - if bVid.isValid: - vtx.bVid[n] = bVid + let bLbl = HashLabel(root: rootVid, key: node.key[n]) + if hasVtx: + if not vtx.bVid[n].isValid: + return err(MergeNodeVtxDiffersFromExisting) + db.top.pAmk[bLbl] = vtx.bVid[n] else: - vtx.bVid[n] = db.vidAttach bLbl + let bVid = db.top.pAmk.getOrVoid bLbl + if bVid.isValid: + vtx.bVid[n] = bVid + else: + vtx.bVid[n] = db.vidAttach bLbl db.top.pPrf.incl vid db.top.sTab[vid] = vtx @@ -514,9 +531,9 @@ proc merge*( else: # Empty hike let rootVtx = db.getVtx hike.root - if rootVtx.isValid: result = db.topIsEmptyAddLeaf(hike,rootVtx,leaf.payload) + else: # Bootstrap for existing root ID let wp = VidVtxPair( @@ -541,7 +558,7 @@ proc merge*( ## Variant of `merge()` for leaf lists. var (merged, dups) = (0, 0) for n,w in leafs: - let hike = db.merge(w) + let hike = db.merge w if hike.error == AristoError(0): merged.inc elif hike.error == MergeLeafPathCachedAlready: @@ -562,18 +579,87 @@ proc merge*( ## The function merges the argument `proof` list of RLP encoded node records ## into the `Aristo Trie` database. This function is intended to be used with ## the proof nodes as returened by `snap/1` messages. - var (merged, dups) = (0, 0) - for n,w in proof: + ## + if not rootVid.isValid: + return (0,0,MergeRootVidInvalid) + let rootKey = db.getKey rootVid + if not rootKey.isValid: + return (0,0,MergeRootKeyInvalid) + + # Expand and collect hash keys and nodes + var nodeTab: Table[HashKey,NodeRef] + for w in proof: let key = w.Blob.digestTo(HashKey) node = w.Blob.decode(NodeRef) - rc = db.mergeNodeImpl(key, node, rootVid) - if rc.isOK: - merged.inc - elif rc.error == MergeHashKeyCachedAlready: - dups.inc - else: - return (n, dups, rc.error) + nodeTab[key] = node + + # Create a table with back links + var + backLink: Table[HashKey,HashKey] + blindNodes: HashSet[HashKey] + for (key,node) in nodeTab.pairs: + case node.vType: + of Leaf: + blindNodes.incl key + of Extension: + if nodeTab.hasKey node.key[0]: + backLink[node.key[0]] = key + else: + blindNodes.incl key + of Branch: + var isBlind = true + for n in 0 .. 15: + if nodeTab.hasKey node.key[n]: + isBlind = false + backLink[node.key[n]] = key + if isBlind: + blindNodes.incl key + + # Run over blind nodes and build chains from a blind/bottom level node up + # to the root node. Select only chains that end up at the pre-defined root + # node. + var chains: seq[seq[HashKey]] + for w in blindNodes: + # Build a chain of nodes up to the root node + var + chain: seq[HashKey] + nodeKey = w + while nodeKey.isValid and nodeTab.hasKey nodeKey: + chain.add nodeKey + nodeKey = backLink.getOrDefault(nodeKey, VOID_HASH_KEY) + if 0 < chain.len and chain[^1] == rootKey: + chains.add chain + + # Make sure that the reverse lookup for the root vertex label is available. + block: + let + lbl = HashLabel(root: rootVid, key: rootKey) + vid = db.top.pAmk.getOrVoid lbl + if not vid.isvalid: + db.top.pAmk[lbl] = rootVid + + # Process over chains in reverse mode starting with the root node. This + # allows the algorithm to find existing nodes on the backend. + var + seen: HashSet[HashKey] + (merged, dups) = (0, 0) + # Process the root ID which is common to all chains + for chain in chains: + for key in chain.reversed: + if key in seen: + discard + else: + seen.incl key + let + node = nodeTab.getOrDefault(key, NodeRef(nil)) + rc = db.mergeNodeImpl(key, node, rootVid) + if rc.isOK: + merged.inc + elif rc.error == MergeHashKeyCachedAlready: + dups.inc + else: + return (merged, dups, rc.error) (merged, dups, AristoError(0)) @@ -597,7 +683,7 @@ proc merge*( ## Upon successful return, the vertex ID assigned to the root key is returned. ## if not rootKey.isValid: - return err(MergeRootKeyEmpty) + return err(MergeRootKeyInvalid) if rootVid.isValid and rootVid != VertexID(1): let key = db.getKey rootVid diff --git a/nimbus/db/aristo/aristo_path.nim b/nimbus/db/aristo/aristo_path.nim index 77cc6d861b..264fc37344 100644 --- a/nimbus/db/aristo/aristo_path.nim +++ b/nimbus/db/aristo/aristo_path.nim @@ -14,8 +14,7 @@ import std/sequtils, eth/[common, trie/nibbles], stew/results, - ./aristo_desc, - ./aristo_desc/aristo_types_private + ./aristo_desc # Info snippet (just a reminder to keep somewhere) # diff --git a/nimbus/db/aristo/aristo_transcode.nim b/nimbus/db/aristo/aristo_transcode.nim index cb7a08c04d..adef9267c7 100644 --- a/nimbus/db/aristo/aristo_transcode.nim +++ b/nimbus/db/aristo/aristo_transcode.nim @@ -14,7 +14,7 @@ import std/[bitops, sequtils], eth/[common, trie/nibbles], stew/results, - "."/[aristo_constants, aristo_desc, aristo_init] + "."/[aristo_constants, aristo_desc] # ------------------------------------------------------------------------------ # Private functions @@ -169,14 +169,14 @@ proc blobify*(node: VertexRef; data: var Blob): AristoError = pSegm = node.ePfx.hexPrefixEncode(isleaf = false) psLen = pSegm.len.byte if psLen == 0 or 33 < pslen: - return VtxExPathOverflow + return BlobifyVtxExPathOverflow data = node.eVid.uint64.toBytesBE.toSeq & pSegm & @[0x80u8 or psLen] of Leaf: let pSegm = node.lPfx.hexPrefixEncode(isleaf = true) psLen = pSegm.len.byte if psLen == 0 or 33 < psLen: - return VtxLeafPathOverflow + return BlobifyVtxLeafPathOverflow data = node.lData.convertTo(Blob) & pSegm & @[0xC0u8 or psLen] proc blobify*(node: VertexRef): Result[Blob, AristoError] = @@ -189,41 +189,39 @@ proc blobify*(node: VertexRef): Result[Blob, AristoError] = ok(data) -proc blobify*(db: AristoDb; data: var Blob) = - ## This function serialises some maintenance data for the `AristoDb` - ## descriptor. At the moment, this contains the recycliing table for the - ## `VertexID` values, only. +proc blobify*(vGen: openArray[VertexID]; data: var Blob) = + ## This function serialises the key generator used in the `AristoDb` + ## descriptor. ## - ## This data recoed is supposed to be stored as the table value with the - ## zero key for persistent tables. + ## This data record is supposed to be as in a dedicated slot in the + ## persistent tables. ## :: ## Admin: ## uint64, ... -- list of IDs ## 0x40 ## data.setLen(0) - if not db.top.isNil: - for w in db.top.vGen: - data &= w.uint64.toBytesBE.toSeq + for w in vGen: + data &= w.uint64.toBytesBE.toSeq data.add 0x40u8 -proc blobify*(db: AristoDb): Blob = - ## Variant of `toDescRecord()` - db.blobify result +proc blobify*(vGen: openArray[VertexID]): Blob = + ## Variant of `blobify()` + vGen.blobify result proc deblobify*(record: Blob; vtx: var VertexRef): AristoError = ## De-serialise a data record encoded with `blobify()`. The second ## argument `vtx` can be `nil`. if record.len < 3: # minimum `Leaf` record - return DbrTooShort + return DeblobTooShort case record[^1] shr 6: of 0: # `Branch` vertex if record.len < 19: # at least two edges - return DbrBranchTooShort + return DeblobBranchTooShort if (record.len mod 8) != 3: - return DbrBranchSizeGarbled + return DeblobBranchSizeGarbled let maxOffset = record.len - 11 aInx = record.len - 3 @@ -234,7 +232,7 @@ proc deblobify*(record: Blob; vtx: var VertexRef): AristoError = vtxList: array[16,VertexID] while access != 0: if maxOffset < offs: - return DbrBranchInxOutOfRange + return DeblobBranchInxOutOfRange let n = access.firstSetBit - 1 access.clearBit n vtxList[n] = (uint64.fromBytesBE record[offs ..< offs+8]).VertexID @@ -249,12 +247,12 @@ proc deblobify*(record: Blob; vtx: var VertexRef): AristoError = sLen = record[^1].int and 0x3f # length of path segment rlen = record.len - 1 # `vertexID` + path segm if record.len < 10: - return DbrExtTooShort + return DeblobExtTooShort if 8 + sLen != rlen: # => slen is at least 1 - return DbrExtSizeGarbled + return DeblobExtSizeGarbled let (isLeaf, pathSegment) = hexPrefixDecode record[8 ..< rLen] if isLeaf: - return DbrExtGotLeafPrefix + return DeblobExtGotLeafPrefix vtx = VertexRef( vType: Extension, eVid: (uint64.fromBytesBE record[0 ..< 8]).VertexID, @@ -266,10 +264,10 @@ proc deblobify*(record: Blob; vtx: var VertexRef): AristoError = rlen = record.len - 1 # payload + path segment pLen = rLen - sLen # payload length if rlen < sLen: - return DbrLeafSizeGarbled + return DeblobLeafSizeGarbled let (isLeaf, pathSegment) = hexPrefixDecode record[pLen ..< rLen] if not isLeaf: - return DbrLeafGotExtPrefix + return DeblobLeafGotExtPrefix vtx = VertexRef( vType: Leaf, lPfx: pathSegment, @@ -277,41 +275,38 @@ proc deblobify*(record: Blob; vtx: var VertexRef): AristoError = pType: BlobData, blob: record[0 ..< plen])) else: - return DbrUnknown + return DeblobUnknown +proc deblobify*(data: Blob; T: type VertexRef): Result[T,AristoError] = + ## Variant of `deblobify()` for vertex deserialisation. + var vtx = T(nil) # will be auto-initialised + let info = data.deblobify vtx + if info != AristoError(0): + return err(info) + ok vtx -proc deblobify*(data: Blob; db: var AristoDb): AristoError = - ## De-serialise the data record encoded with `blobify()` into a new current - ## top layer. If present, the previous top layer of the `db` descriptor is - ## pushed onto the parent layers stack. - if not db.top.isNil: - db.stack.add db.top - db.top = AristoLayerRef() + +proc deblobify*(data: Blob; vGen: var seq[VertexID]): AristoError = + ## De-serialise the data record encoded with `blobify()` into the vertex ID + ## generator argument `vGen`. if data.len == 0: - db.top.vGen = @[1.VertexID] + vGen = @[1.VertexID] else: if (data.len mod 8) != 1: - return ADbGarbledSize + return DeblobSizeGarbled if data[^1] shr 6 != 1: - return ADbWrongType + return DeblobWrongType for n in 0 ..< (data.len div 8): let w = n * 8 - db.top.vGen.add (uint64.fromBytesBE data[w ..< w + 8]).VertexID + vGen.add (uint64.fromBytesBE data[w ..< w + 8]).VertexID -proc deblobify*[W: VertexRef|AristoDb]( - record: Blob; - T: type W; - ): Result[T,AristoError] = - ## Variant of `deblobify()` for either `VertexRef` or `AristoDb` - var obj: T # isNil, will be auto-initialised - let info = record.deblobify obj +proc deblobify*(data: Blob; T: type seq[VertexID]): Result[T,AristoError] = + ## Variant of `deblobify()` for deserialising the vertex ID generator state + var vGen: seq[VertexID] + let info = data.deblobify vGen if info != AristoError(0): return err(info) - ok(obj) - -proc deblobify*(record: Blob): Result[VertexRef,AristoError] = - ## Default variant of `deblobify()` for `VertexRef`. - record.deblobify VertexRef + ok vGen # ------------------------------------------------------------------------------ # End diff --git a/nimbus/db/aristo/aristo_vid.nim b/nimbus/db/aristo/aristo_vid.nim index 62a144b18f..0b57487e4a 100644 --- a/nimbus/db/aristo/aristo_vid.nim +++ b/nimbus/db/aristo/aristo_vid.nim @@ -14,7 +14,7 @@ {.push raises: [].} import - std/[algorithm, sequtils, sets, tables], + std/[algorithm, sequtils, tables], ./aristo_desc # ------------------------------------------------------------------------------ @@ -92,7 +92,6 @@ proc vidReorg*(db: AristoDb) = proc vidAttach*(db: AristoDb; lbl: HashLabel; vid: VertexID) = ## Attach (i.r. register) a Merkle hash key to a vertex ID. - db.top.dKey.excl vid db.top.pAmk[lbl] = vid db.top.kMap[vid] = lbl diff --git a/tests/test_aristo.nim b/tests/test_aristo.nim index c5304b176a..e91283b053 100644 --- a/tests/test_aristo.nim +++ b/tests/test_aristo.nim @@ -24,7 +24,8 @@ import ./replay/[pp, undump_accounts, undump_storages], ./test_sync_snap/[snap_test_xx, test_accounts, test_types], ./test_aristo/[ - test_delete, test_helpers, test_merge, test_nearby, test_transcode] + test_backend, test_delete, test_helpers, test_merge, test_nearby, + test_transcode] const baseDir = [".", "..", ".."/"..", $DirSep] @@ -158,7 +159,7 @@ proc transcodeRunner(noisy =true; sample=accSample; stopAfter=high(int)) = accLst = sample.to(seq[UndumpAccounts]) root = accLst[0].root tmpDir = getTmpDir() - db = tmpDir.testDbs(sample.name & "-accounts", instances=2, persistent=true) + db = tmpDir.testDbs(sample.name&"-transcode", instances=2, persistent=true) info = if db.persistent: &"persistent db on \"{db.baseDir}\"" else: "in-memory db" fileInfo = sample.file.splitPath.tail.replace(".txt.gz","") @@ -192,6 +193,11 @@ proc accountsRunner(noisy=true; sample=accSample, resetDb=false) = accLst = sample.to(seq[UndumpAccounts]).to(seq[ProofTrieData]) fileInfo = sample.file.splitPath.tail.replace(".txt.gz","") listMode = if resetDb: "" else: ", merged data lists" + baseDir = getTmpDir() / sample.name & "-accounts" + dbDir = baseDir / "tmp" + + defer: + try: baseDir.removeDir except CatchableError: discard suite &"Aristo: accounts data dump from {fileInfo}{listMode}": @@ -201,6 +207,9 @@ proc accountsRunner(noisy=true; sample=accSample, resetDb=false) = test &"Merge {accLst.len} proof & account lists to database": check noisy.test_mergeProofAndKvpList(accLst, resetDb) + test &"Store {accLst.len} account lists on database backends": + check noisy.test_backendConsistency(accLst, dbDir, resetDb) + test &"Traverse accounts database w/{accLst.len} account lists": check noisy.test_nearbyKvpList(accLst, resetDb) @@ -218,6 +227,11 @@ proc storagesRunner( stoLst = sample.to(seq[UndumpStorages]).to(seq[ProofTrieData]) fileInfo = sample.file.splitPath.tail.replace(".txt.gz","") listMode = if resetDb: "" else: ", merged data lists" + baseDir = getTmpDir() / sample.name & "-storage" + dbDir = baseDir / "tmp" + + defer: + try: baseDir.removeDir except CatchableError: discard suite &"Aristo: storages data dump from {fileInfo}{listMode}": @@ -227,6 +241,9 @@ proc storagesRunner( test &"Merge {stoLst.len} proof & slots lists to database": check noisy.test_mergeProofAndKvpList(stoLst, resetDb, fileInfo, oops) + test &"Store {stoLst.len} slot lists on database backends": + check noisy.test_backendConsistency(stoLst, dbDir, resetDb) + test &"Traverse storage slots database w/{stoLst.len} account lists": check noisy.test_nearbyKvpList(stoLst, resetDb) @@ -249,11 +266,11 @@ when isMainModule: setErrorLevel() - when true: # and false: + when true and false: noisy.miscRunner() # Borrowed from `test_sync_snap.nim` - when true: # and false: + when true and false: for n,sam in snapTestList: noisy.transcodeRunner(sam) for n,sam in snapTestStorageList: diff --git a/tests/test_aristo/test_backend.nim b/tests/test_aristo/test_backend.nim new file mode 100644 index 0000000000..e28e682d42 --- /dev/null +++ b/tests/test_aristo/test_backend.nim @@ -0,0 +1,256 @@ +# Nimbus +# Copyright (c) 2018-2021 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or +# distributed except according to those terms. + +## Aristo (aka Patricia) DB records merge test + +import + std/[algorithm, sequtils, sets, tables], + eth/common, + stew/results, + unittest2, + ../../nimbus/sync/protocol, + ../../nimbus/db/aristo/aristo_init/[ + aristo_memory, aristo_rocksdb], + ../../nimbus/db/aristo/[ + aristo_desc, aristo_debug, aristo_hashify, aristo_init, aristo_layer, + aristo_merge], + ./test_helpers + +# ------------------------------------------------------------------------------ +# Private helpers +# ------------------------------------------------------------------------------ + +proc mergeData( + db: AristoDb; + rootKey: HashKey; + rootVid: VertexID; + proof: openArray[SnapProof]; + leafs: openArray[LeafTiePayload]; + noisy: bool; + ): bool = + ## Simplified loop body of `test_mergeProofAndKvpList()` + if 0 < proof.len: + let rc = db.merge(rootKey, rootVid) + if rc.isErr: + check rc.error == AristoError(0) + return + + let proved = db.merge(proof, rc.value) + if proved.error notin {AristoError(0),MergeHashKeyCachedAlready}: + check proved.error in {AristoError(0),MergeHashKeyCachedAlready} + return + + let merged = db.merge leafs + if merged.error notin {AristoError(0), MergeLeafPathCachedAlready}: + check merged.error in {AristoError(0), MergeLeafPathCachedAlready} + return + + block: + let rc = db.hashify # (noisy, true) + if rc.isErr: + when true: # and false: + noisy.say "***", "dataMerge(9)", + " nLeafs=", leafs.len, + "\n cache dump\n ", db.pp, + "\n backend dump\n ", db.backend.AristoTypedBackendRef.pp(db) + check rc.error == (VertexID(0),AristoError(0)) + return + + true + +proc verify( + ly: AristoLayerRef; # Database layer + be: MemBackendRef|RdbBackendRef; # Backend + noisy: bool; + ): bool = + ## .. + + let + beSTab = be.walkVtx.toSeq.mapIt((it[1],it[2])).toTable + beKMap = be.walkKey.toSeq.mapIt((it[1],it[2])).toTable + beIdg = be.walkIdg.toSeq + + for vid in beSTab.keys.toSeq.mapIt(it.uint64).sorted.mapIt(it.VertexID): + let + nVtx = ly.sTab.getOrVoid vid + mVtx = beSTab.getOrVoid vid + if not nVtx.isValid and not mVtx.isValid: + check nVtx != VertexRef(nil) + check mVtx != VertexRef(nil) + return + if nVtx != mVtx: + noisy.say "***", "verify", + " beType=", be.typeof, + " vid=", vid.pp, + " nVtx=", nVtx.pp, + " mVtx=", mVtx.pp + check nVtx == mVtx + return + + if beSTab.len != ly.sTab.len or + beKMap.len != ly.kMap.len: + check beSTab.len == ly.sTab.len + check beKMap.len == ly.kMap.len + return + + true + +# ------------------------------------------------------------------------------ +# Public test function +# ------------------------------------------------------------------------------ + +proc test_backendConsistency*( + noisy: bool; + list: openArray[ProofTrieData]; # Test data + rdbPath: string; # Rocks DB storage directory + resetDb = false; + doRdbOk = true; + ): bool = + ## Import accounts + var + ndb: AristoDb # Reference cache + mdb: AristoDb # Memory backend database + rdb: AristoDb # Rocks DB backend database + rootKey = HashKey.default + count = 0 + + defer: + rdb.finish(flush=true) + + for n,w in list: + if w.root != rootKey or resetDB: + rootKey = w.root + count = 0 + ndb = AristoDb.init(BackendNone) + mdb = AristoDb.init(BackendMemory) + if doRdbOk: + rdb.finish(flush=true) + let rc = AristoDb.init(BackendRocksDB,rdbPath) + if rc.isErr: + check rc.error == AristoError(0) + return + rdb = rc.value + count.inc + + check ndb.backend.isNil + check not mdb.backend.isNil + check doRdbOk or not rdb.backend.isNil + + when true and false: + noisy.say "***", "beCon(1) <", n, "/", list.len-1, ">", " groups=", count + + block: + let + rootVid = VertexID(1) + leafs = w.kvpLst.mapRootVid VertexID(1) # for merging it into main trie + + block: + let ndbOk = ndb.mergeData( + rootKey, rootVid, w.proof, leafs, noisy=false) + if not ndbOk: + check ndbOk + return + block: + let mdbOk = mdb.mergeData( + rootKey, rootVid, w.proof, leafs, noisy=false) + if not mdbOk: + check mdbOk + return + if doRdbOk: # optional + let rdbOk = rdb.mergeData( + rootKey, rootVid, w.proof, leafs, noisy=false) + if not rdbOk: + check rdbOk + return + + when true and false: + noisy.say "***", "beCon(2) <", n, "/", list.len-1, ">", + " groups=", count, + "\n cache dump\n ", ndb.pp, + "\n backend dump\n ", ndb.backend.AristoTypedBackendRef.pp(ndb), + "\n -------------", + "\n mdb cache\n ", mdb.pp, + "\n mdb backend\n ", mdb.to(MemBackendRef).pp(ndb), + "\n -------------", + "\n rdb cache\n ", rdb.pp, + "\n rdb backend\n ", rdb.to(RdbBackendRef).pp(ndb), + "\n -------------" + + when true and false: + noisy.say "***", "beCon(4) <", n, "/", list.len-1, ">", " groups=", count + + var + mdbPreSaveCache, mdbPreSaveBackend: string + rdbPreSaveCache, rdbPreSaveBackend: string + when true: # and false: + mdbPreSaveCache = mdb.pp + mdbPreSaveBackend = mdb.to(MemBackendRef).pp(ndb) + rdbPreSaveCache = rdb.pp + rdbPreSaveBackend = rdb.to(RdbBackendRef).pp(ndb) + + # Store onto backend database + let mdbHist = block: + #noisy.say "***", "db-dump\n ", mdb.pp + let rc = mdb.save + if rc.isErr: + check rc.error == AristoError(0) + return + rc.value + + if doRdbOk: + let rdbHist = block: + let rc = rdb.save + if rc.isErr: + check rc.error == AristoError(0) + return + rc.value + + if not ndb.top.verify(mdb.backend.MemBackendRef, noisy): + when true and false: + noisy.say "***", "beCon(4) <", n, "/", list.len-1, ">", + " groups=", count, + "\n ndb cache\n ", ndb.pp, + "\n ndb backend=", ndb.backend.isNil.not, + #"\n -------------", + #"\n mdb pre-save cache\n ", mdbPreSaveCache, + #"\n mdb pre-save backend\n ", mdbPreSaveBackend, + "\n -------------", + "\n mdb cache\n ", mdb.pp, + "\n mdb backend\n ", mdb.to(MemBackendRef).pp(ndb), + "\n -------------" + return + + if doRdbOk: + if not ndb.top.verify(rdb.backend.RdbBackendRef, noisy): + when true and false: + noisy.say "***", "beCon(4) <", n, "/", list.len-1, ">", + " groups=", count, + "\n ndb cache\n ", ndb.pp, + "\n ndb backend=", ndb.backend.isNil.not, + "\n -------------", + "\n rdb pre-save cache\n ", rdbPreSaveCache, + "\n rdb pre-save backend\n ", rdbPreSaveBackend, + "\n -------------", + "\n rdb cache\n ", rdb.pp, + "\n rdb backend\n ", rdb.to(RdbBackendRef).pp(ndb), + #"\n -------------", + #"\n mdb cache\n ", mdb.pp, + #"\n mdb backend\n ", mdb.to(MemBackendRef).pp(ndb), + "\n -------------" + return + + when true and false: + noisy.say "***", "beCon(9) <", n, "/", list.len-1, ">", " groups=", count + + true + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/tests/test_aristo/test_delete.nim b/tests/test_aristo/test_delete.nim index 6cd1c1ed88..6196a55d07 100644 --- a/tests/test_aristo/test_delete.nim +++ b/tests/test_aristo/test_delete.nim @@ -17,7 +17,8 @@ import stew/results, unittest2, ../../nimbus/db/aristo/[ - aristo_desc, aristo_delete, aristo_hashify, aristo_nearby, aristo_merge], + aristo_desc, aristo_delete, aristo_hashify, aristo_init, aristo_nearby, + aristo_merge], ./test_helpers type @@ -116,7 +117,7 @@ proc test_delete*( var td = TesterDesc.init 42 for n,w in list: let - db = AristoDb(top: AristoLayerRef()) + db = AristoDb.init BackendNone # (top: AristoLayerRef()) lstLen = list.len leafs = w.kvpLst.mapRootVid VertexID(1) # merge into main trie added = db.merge leafs diff --git a/tests/test_aristo/test_helpers.nim b/tests/test_aristo/test_helpers.nim index 2c0a69c73c..e17ed584f6 100644 --- a/tests/test_aristo/test_helpers.nim +++ b/tests/test_aristo/test_helpers.nim @@ -10,7 +10,7 @@ # distributed except according to those terms. import - std/sequtils, + std/[os, sequtils], eth/common, rocksdb, ../../nimbus/db/aristo/[ diff --git a/tests/test_aristo/test_merge.nim b/tests/test_aristo/test_merge.nim index 76c7019379..247b41fe21 100644 --- a/tests/test_aristo/test_merge.nim +++ b/tests/test_aristo/test_merge.nim @@ -17,7 +17,7 @@ import stew/results, unittest2, ../../nimbus/db/aristo/[ - aristo_desc, aristo_debug, aristo_get, aristo_hashify, + aristo_desc, aristo_debug, aristo_get, aristo_hashify, aristo_init, aristo_hike, aristo_merge], ./test_helpers @@ -126,7 +126,7 @@ proc test_mergeKvpList*( resetDb = false; ): bool = - var db = AristoDb(top: AristoLayerRef()) + var db = AristoDb.init BackendNone for n,w in list: if resetDb: db.top = AristoLayerRef() @@ -242,9 +242,6 @@ proc test_mergeProofAndKvpList*( check db.top.lTab.len == lTabLen + merged.merged check merged.merged + merged.dups == leafs.len - if w.proof.len == 0: - let vtx = db.getVtx VertexID(1) - block: if merged.error notin {AristoError(0), MergeLeafPathCachedAlready}: noisy.say "***", "<", n, "/", lstLen-1, ">\n ", db.pp @@ -277,9 +274,9 @@ proc test_mergeProofAndKvpList*( " testId=", testId, " groups=", count, "\n pre-DB", - "\n ", preDb, + "\n ", preDb, "\n --------", - "\n ", db.pp + "\n ", db.pp check rc.error == (VertexID(0),AristoError(0)) return diff --git a/tests/test_aristo/test_transcode.nim b/tests/test_aristo/test_transcode.nim index 2b03e3a1b3..569ea842af 100644 --- a/tests/test_aristo/test_transcode.nim +++ b/tests/test_aristo/test_transcode.nim @@ -18,7 +18,7 @@ import unittest2, ../../nimbus/db/kvstore_rocksdb, ../../nimbus/db/aristo/[ - aristo_desc, aristo_debug, aristo_transcode, aristo_vid], + aristo_desc, aristo_debug, aristo_init, aristo_transcode, aristo_vid], "."/[test_aristo_cache, test_helpers] type @@ -87,7 +87,7 @@ proc test_transcodeAccounts*( ) = ## Transcoder tests on accounts database var - adb = AristoDb(top: AristoLayerRef()) + adb = AristoDb.init BackendNone count = -1 for (n, key,value) in rocky.walkAllDb(): if stopAfter < n: @@ -139,7 +139,7 @@ proc test_transcodeAccounts*( # NIM object <-> DbRecord mapping let dbr = node.blobify.getOrEmpty(noisy) - var node1 = dbr.deblobify.asNode(adb) + var node1 = dbr.deblobify(VertexRef).asNode(adb) if node1.error != AristoError(0): check node1.error == AristoError(0) @@ -163,7 +163,7 @@ proc test_transcodeAccounts*( noisy.say "***", "count=", count, " dbr1=", dbr1.toHex # Serialise back as is - let dbr2 = dbr.deblobify.asNode(adb).blobify.getOrEmpty(noisy) + let dbr2 = dbr.deblobify(VertexRef).asNode(adb).blobify.getOrEmpty(noisy) block: if dbr != dbr2: check dbr == dbr2 @@ -176,7 +176,7 @@ proc test_transcodeAccounts*( proc test_transcodeVidRecycleLists*(noisy = true; seed = 42) = ## Transcode VID lists held in `AristoDb` descriptor var td = TesterDesc.init seed - let db = AristoDb(top: AristoLayerRef()) + let db = AristoDb.init BackendNone # Add some randum numbers block: @@ -198,14 +198,16 @@ proc test_transcodeVidRecycleLists*(noisy = true; seed = 42) = # Serialise/deserialise block: - let dbBlob = db.blobify + let dbBlob = db.top.vGen.blobify # Deserialise - let db1 = block: - let rc = dbBlob.deblobify AristoDb - if rc.isErr: - check rc.isOk - rc.get(otherwise = AristoDb(top: AristoLayerRef())) + let + db1 = AristoDb.init BackendNone + rc = dbBlob.deblobify seq[VertexID] + if rc.isErr: + check rc.error == AristoError(0) + else: + db1.top.vGen = rc.value check db.top.vGen == db1.top.vGen