Aristo db with storage backends (status-im#1603)

* Generalised Aristo DB constructor for any type of backend

details:
  * Records to be deleted are represented as key-void (rather than
    key-value) pairs by the put-function arguments
  * Allow direct driver access, iterators as example implementation and
    for testing.

* Provide backend storage interface

details:
  Stores the top layer onto backend tables

* Implemented Rocks DB backend

details:
  Transaction based `put()` functionality
  Iterators (based on direct RocksDB access)

nimbus/db/aristo/aristo_debug.nim

@@ -14,8 +14,8 @@ import
   std/[algorithm, sequtils, sets, strutils, tables],
   eth/[common, trie/nibbles],
   stew/byteutils,
-  "."/[aristo_constants, aristo_desc, aristo_hike, aristo_vid],
-  ./aristo_desc/aristo_types_private
+  "."/[aristo_constants, aristo_desc, aristo_hike, aristo_init, aristo_vid],
+  ./aristo_init/[aristo_memory, aristo_rocksdb]
 
 # ------------------------------------------------------------------------------
 # Ptivate functions
@@ -34,7 +34,7 @@ proc sortedKeys(pPrf: HashSet[VertexID]): seq[VertexID] =
   pPrf.toSeq.mapIt(it.uint64).sorted.mapIt(it.VertexID)
 
 proc toPfx(indent: int; offset = 0): string =
-  if 0 < indent: "\n" & " ".repeat(indent+offset) else: ""
+  if 0 < indent+offset: "\n" & " ".repeat(indent+offset) else: ""
 
 proc labelVidUpdate(db: var AristoDb, lbl: HashLabel, vid: VertexID): string =
   if lbl.key.isValid and vid.isValid:
@@ -70,8 +70,13 @@ proc squeeze(s: string; hex = false; ignLen = false): string =
 proc stripZeros(a: string): string =
   a.strip(leading=true, trailing=false, chars={'0'}).toLowerAscii
 
-proc ppVid(vid: VertexID): string =
-  if vid.isValid: "$" & vid.uint64.toHex.stripZeros.toLowerAscii else: "$ø"
+proc ppVid(vid: VertexID; pfx = true): string =
+  if pfx:
+    result = "$"
+  if vid.isValid:
+    result &= vid.uint64.toHex.stripZeros.toLowerAscii
+  else:
+    result &= "ø"
 
 proc vidCode(lbl: HashLabel, db: AristoDb): uint64 =
   if lbl.isValid:
@@ -109,11 +114,11 @@ proc ppLabel(lbl: HashLabel; db: AristoDb): string =
   if not db.top.isNil:
     let vid = db.top.pAmk.getOrVoid lbl
     if vid.isValid:
-      return "£" & rid & vid.ppVid
+      return "£" & rid & vid.ppVid(pfx=false)
   block:
     let vid = db.xMap.getOrVoid lbl
     if vid.isValid:
-      return "£" & rid & vid.ppVid
+      return "£" & rid & vid.ppVid(pfx=false)
 
   "%" & rid & lbl.key.ByteArray32
                      .mapIt(it.toHex(2)).join.tolowerAscii
@@ -192,17 +197,14 @@ proc ppXMap*(
                  .filterIt(1 < it[1]).toTable
 
   proc ppNtry(n: uint64): string =
+    var s = "(" & VertexID(n).ppVid
     let lbl = kMap.getOrVoid VertexID(n)
-    var s = "(" & VertexID(n).ppVid & ","
     if lbl.isValid:
-      s &= lbl.ppLabel(db)
-
       let vid = pAmk.getOrVoid lbl
-      if vid.isValid:
-        s &= ",ø"
+      if not vid.isValid:
+        s &= "," & lbl.ppLabel(db) & ",ø"
       elif vid != VertexID(n):
-        s &= "," & vid.ppVid
-
+        s &= "," & lbl.ppLabel(db) & "," & vid.ppVid
       let count = dups.getOrDefault(VertexID(n), 0)
       if 0 < count:
         s &= ",*" & $count
@@ -247,6 +249,20 @@ proc ppXMap*(
   else:
     result &= "}"
 
+proc ppBe[T](be: T; db: AristoDb; indent: int): string =
+  ## Walk over backend tables
+  let pfx = indent.toPfx
+  result = "<" & $be.kind & ">"
+  result &= pfx & "vGen" & pfx & " [" & be.walkIdg.toSeq.mapIt(
+      it[2].pp
+    ).join(",") & "]"
+  result &= pfx & "sTab" & pfx & " {" & be.walkVtx.toSeq.mapIt(
+      $(1+it[0]) & "(" & it[1].ppVid & "," & it[2].ppVtx(db,it[1]) & ")"
+    ).join("," & pfx & "  ") & "}"
+  result &= pfx & "kMap" & pfx & " {" & be.walkKey.toSeq.mapIt(
+      $(1+it[0]) & "(" & it[1].ppVid & "," & it[2].ppKey & ")"
+    ).join("," & pfx & "  ") & "}"
+
 # ------------------------------------------------------------------------------
 # Public functions
 # ------------------------------------------------------------------------------
@@ -276,8 +292,8 @@ proc pp*(lty: LeafTie, db = AristoDb()): string =
 proc pp*(vid: VertexID): string =
   vid.ppVid
 
-proc pp*(vid: openArray[VertexID]): string =
-  "[" & vid.mapIt(it.ppVid).join(",") & "]"
+proc pp*(vGen: openArray[VertexID]): string =
+  "[" & vGen.mapIt(it.ppVid).join(",") & "]"
 
 proc pp*(p: PayloadRef, db = AristoDb()): string =
   p.ppPayload(db)
@@ -337,18 +353,24 @@ proc pp*(lTab: Table[LeafTie,VertexID]; indent = 4): string =
             .mapIt("(" & it[0].ppLeafTie(db) & "," & it[1].ppVid & ")")
             .join("," & indent.toPfx(1)) & "}"
 
-proc pp*(vGen: seq[VertexID]): string =
-  "[" & vGen.mapIt(it.ppVid).join(",") & "]"
-
 proc pp*(pPrf: HashSet[VertexID]): string =
   "{" & pPrf.sortedKeys.mapIt(it.ppVid).join(",") & "}"
 
 proc pp*(leg: Leg; db = AristoDb()): string =
   result = "(" & leg.wp.vid.ppVid & ","
   if not db.top.isNil:
     let lbl = db.top.kMap.getOrVoid leg.wp.vid
-    result &= (if lbl.isValid: lbl.ppLabel(db) else: "ø")
-  result &= "," & $leg.nibble.ppNibble & "," & leg.wp.vtx.pp(db) & ")"
+    if not lbl.isValid:
+      result &= "ø"
+    elif leg.wp.vid != db.top.pAmk.getOrVoid lbl:
+      result &= lbl.ppLabel(db)
+  result &= ","
+  if leg.backend:
+    result &= "*"
+  result &= ","
+  if 0 <= leg.nibble:
+    result &= $leg.nibble.ppNibble
+  result &= "," & leg.wp.vtx.pp(db) & ")"
 
 proc pp*(hike: Hike; db = AristoDb(); indent = 4): string =
   let pfx = indent.toPfx(1)
@@ -388,52 +410,70 @@ proc pp*(kMap: Table[VertexID,Hashlabel]; db: AristoDb; indent = 4): string =
 proc pp*(pAmk: Table[Hashlabel,VertexID]; db: AristoDb; indent = 4): string =
   db.ppXMap(db.top.kMap, pAmk, indent)
 
+proc pp*(
+    be: MemBackendRef|RdbBackendRef;
+    db: AristoDb;
+    indent = 4;
+      ): string =
+  be.ppBe(db, indent)
+
 # ---------------------
 
 proc pp*(
     db: AristoDb;
     sTabOk = true;
     lTabOk = true;
     kMapOk = true;
-    dKeyOk = true;
     pPrfOk = true;
     indent = 4;
       ): string =
   let
-    pfx1 = max(indent-1,0).toPfx
-    pfx2 = indent.toPfx
-    labelOk = 1 < sTabOk.ord + lTabOk.ord + kMapOk.ord + dKeyOk.ord + pPrfOk.ord
+    pfx1 = indent.toPfx
+    pfx2 = indent.toPfx(1)
+    tagOk = 1 < sTabOk.ord + lTabOk.ord + kMapOk.ord + pPrfOk.ord
   var
-    pfy1 = ""
-    pfy2 = ""
+    pfy = ""
 
   proc doPrefix(s: string): string =
     var rc: string
-    if labelOk:
-      rc = pfy1 & s & pfx2
-      pfy1 = pfx1
+    if tagOk:
+      rc = pfy & s & pfx2
+      pfy = pfx1
     else:
-      rc = pfy2
-      pfy2 = pfx2
+      rc = pfy
+      pfy = pfx2
     rc
 
   if not db.top.isNil:
     if sTabOk:
       let info = "sTab(" & $db.top.sTab.len & ")"
-      result &= info.doPrefix & db.top.sTab.pp(db,indent)
+      result &= info.doPrefix & db.top.sTab.pp(db,indent+1)
     if lTabOk:
       let info = "lTab(" & $db.top.lTab.len & ")"
-      result &= info.doPrefix & db.top.lTab.pp(indent)
+      result &= info.doPrefix & db.top.lTab.pp(indent+1)
     if kMapOk:
       let info = "kMap(" & $db.top.kMap.len & "," & $db.top.pAmk.len & ")"
-      result &= info.doPrefix & db.ppXMap(db.top.kMap,db.top.pAmk,indent)
-    if dKeyOk:
-      let info = "dKey(" & $db.top.dkey.len & ")"
-      result &= info.doPrefix & db.top.dKey.pp
+      result &= info.doPrefix & db.ppXMap(db.top.kMap,db.top.pAmk,indent+1)
     if pPrfOk:
       let info = "pPrf(" & $db.top.pPrf.len & ")"
       result &= info.doPrefix & db.top.pPrf.pp
 
+proc pp*(
+  be: AristoTypedBackendRef;
+  db: AristoDb;
+  indent = 4;
+    ): string =
+
+  case (if be.isNil: BackendNone else: be.kind)
+  of BackendMemory:
+    be.MemBackendRef.ppBe(db, indent)
+
+  of BackendRocksDB:
+    be.RdbBackendRef.ppBe(db, indent)
+
+  of BackendNone:
+    "n/a"
+
 # ------------------------------------------------------------------------------
 # End
 # ------------------------------------------------------------------------------

nimbus/db/aristo/aristo_delete.nim

@@ -41,11 +41,11 @@ proc clearKey(db: AristoDb; vid: VertexID) =
     db.top.pAmk.del key
   elif db.getKeyBackend(vid).isOK:
     # Register for deleting on backend
-    db.top.dKey.incl vid
+    db.top.kMap[vid] = VOID_HASH_LABEL
+    db.top.pAmk.del key
 
 proc doneWith(db: AristoDb; vid: VertexID) =
   # Remove entry
-  db.top.dKey.excl vid # No need to register for deleting on backend
   db.vidDispose vid    # Will be propagated to backend
   db.top.sTab.del vid
   let key = db.top.kMap.getOrVoid vid
@@ -111,7 +111,7 @@ proc deleteImpl(
     # No need to keep it any longer
     db.top.lTab.del lty
   else:
-    # To be deleted in backend when it is updated
+    # To be recorded on change history
     db.top.lTab[lty] = VertexID(0)
 
   ok()

nimbus/db/aristo/aristo_desc.nim

@@ -41,7 +41,6 @@ type
     sTab*: Table[VertexID,VertexRef] ## Structural vertex table
     lTab*: Table[LeafTie,VertexID]   ## Direct access, path to leaf vertex
     kMap*: Table[VertexID,HashLabel] ## Merkle hash key mapping
-    dKey*: HashSet[VertexID]         ## Locally deleted Merkle hash keys
     pAmk*: Table[HashLabel,VertexID] ## Reverse mapper for data import
     pPrf*: HashSet[VertexID]         ## Locked vertices (proof nodes)
     vGen*: seq[VertexID]             ## Unique vertex ID generator
@@ -70,21 +69,38 @@ proc getOrVoid*[W](tab: Table[W,VertexID]; w: W): VertexID =
 
 # --------
 
-proc isValid*(vtx: VertexRef): bool =
+func isValid*(vtx: VertexRef): bool =
   vtx != VertexRef(nil) 
 
-proc isValid*(nd: NodeRef): bool =
+func isValid*(nd: NodeRef): bool =
   nd != NodeRef(nil)
 
-proc isValid*(key: HashKey): bool =
+func isValid*(key: HashKey): bool =
   key != VOID_HASH_KEY
 
-proc isValid*(lbl: HashLabel): bool =
+func isValid*(lbl: HashLabel): bool =
   lbl != VOID_HASH_LABEL
 
-proc isValid*(vid: VertexID): bool =
+func isValid*(vid: VertexID): bool =
   vid != VertexID(0)
 
+# ------------------------------------------------------------------------------
+# Public functions, miscellaneous
+# ------------------------------------------------------------------------------
+
+# Note that the below `init()` function cannot go into
+# `aristo_types_identifiers` as this would result in a circular import.
+
+func init*(key: var HashKey; data: openArray[byte]): bool =
+  ## Import argument `data` into `key` which must have length either `32`, or
+  ## `0`. The latter case is equivalent to an all zero byte array of size `32`.
+  if data.len == 32:
+    (addr key.ByteArray32[0]).copyMem(unsafeAddr data[0], data.len)
+    return true
+  if data.len == 0:
+    key = VOID_HASH_KEY
+    return true
+
 # ------------------------------------------------------------------------------
 # End
 # ------------------------------------------------------------------------------