Rewrite tree to use lazy-loaded cache with deferred commit

This introduces a largely rewritten implementation of the SMT which operates on a cached,
lazily-loaded tree structure which must be explicitly committed to DB, replacing the existing pattern
of performing DB reads and writes on for each operation. This gives a significant performance
improvement on all operations while preserving backwards compatibility of commitment hashes.

This also:

* refactors the hasher objects to allow tree paths and stored values to be hashed independently,
and Options to configure them. By passing an identity function as the path hasher, the raw key can
be used directly as the leaf.

* removes value storage from the tree; the caller must maintain their own value mapping.

* removes the DeepSubtree code supporting state-transition fraud proofs, as it was not compatible
as-is with this implementation (but there should be no technical block to adapting it to this pattern).

* renames various types and decouples the path and value hashing functions.

* expands tests to cover corner cases, e.g. orphan node removal.

README.md

@@ -19,21 +19,21 @@ import (
 )
 
 func main() {
-	// Initialise two new key-value store to store the nodes and values of the tree
+	// Initialise two new key-value store to store the nodes of the tree
+	// (Note: the tree only stores hashed values, not raw value data)
 	nodeStore := smt.NewSimpleMap()
-	valueStore := smt.NewSimpleMap()
 	// Initialise the tree
-	tree := smt.NewSparseMerkleTree(nodeStore, valueStore, sha256.New())
+	tree := smt.NewSMT(nodeStore, sha256.New())
 
 	// Update the key "foo" with the value "bar"
-	_, _ = tree.Update([]byte("foo"), []byte("bar"))
+	_ = tree.Update([]byte("foo"), []byte("bar"))
 
 	// Generate a Merkle proof for foo=bar
 	proof, _ := tree.Prove([]byte("foo"))
 	root := tree.Root() // We also need the current tree root for the proof
 
 	// Verify the Merkle proof for foo=bar
-	if smt.VerifyProof(proof, root, []byte("foo"), []byte("bar"), sha256.New()) {
+	if smt.VerifyProof(proof, root, []byte("foo"), []byte("bar"), tree.Spec()) {
 		fmt.Println("Proof verification succeeded.")
 	} else {
 		fmt.Println("Proof verification failed.")

bench_test.go

@@ -8,29 +8,29 @@ import (
 
 func BenchmarkSparseMerkleTree_Update(b *testing.B) {
 	smn, smv := NewSimpleMap(), NewSimpleMap()
-	smt := NewSparseMerkleTree(smn, smv, sha256.New())
+	smt := NewSMTWithStorage(smn, smv, sha256.New())
 
 	b.ResetTimer()
 	b.ReportAllocs()
 	for i := 0; i < b.N; i++ {
 		s := strconv.Itoa(i)
-		_, _ = smt.Update([]byte(s), []byte(s))
+		_ = smt.Update([]byte(s), []byte(s))
 	}
 }
 
 func BenchmarkSparseMerkleTree_Delete(b *testing.B) {
 	smn, smv := NewSimpleMap(), NewSimpleMap()
-	smt := NewSparseMerkleTree(smn, smv, sha256.New())
+	smt := NewSMTWithStorage(smn, smv, sha256.New())
 
 	for i := 0; i < 100000; i++ {
 		s := strconv.Itoa(i)
-		_, _ = smt.Update([]byte(s), []byte(s))
+		_ = smt.Update([]byte(s), []byte(s))
 	}
 
 	b.ResetTimer()
 	b.ReportAllocs()
 	for i := 0; i < b.N; i++ {
 		s := strconv.Itoa(i)
-		_, _ = smt.Delete([]byte(s))
+		_ = smt.Delete([]byte(s))
 	}
 }

bulk_test.go

@@ -4,29 +4,36 @@ import (
 	"bytes"
 	"crypto/sha256"
 	"math/rand"
-	"reflect"
 	"testing"
 )
 
+type opCounts struct{ ops, inserts, updates, deletes int }
+type bulkop struct{ key, val []byte }
+
 // Test all tree operations in bulk.
-func TestSparseMerkleTree(t *testing.T) {
-	for i := 0; i < 5; i++ {
+func TestBulkOperations(t *testing.T) {
+	rand.Seed(1)
+
+	cases := []opCounts{
 		// Test more inserts/updates than deletions.
-		bulkOperations(t, 200, 100, 100, 50)
-	}
-	for i := 0; i < 5; i++ {
+		{200, 100, 100, 50},
+		{1000, 100, 100, 50},
 		// Test extreme deletions.
-		bulkOperations(t, 200, 100, 100, 500)
+		{200, 100, 100, 500},
+		{1000, 100, 100, 500},
+	}
+	for _, tc := range cases {
+		bulkOperations(t, tc.ops, tc.inserts, tc.updates, tc.deletes)
 	}
 }
 
 // Test all tree operations in bulk, with specified ratio probabilities of insert, update and delete.
 func bulkOperations(t *testing.T, operations int, insert int, update int, delete int) {
 	smn, smv := NewSimpleMap(), NewSimpleMap()
-	smt := NewSparseMerkleTree(smn, smv, sha256.New())
+	smt := NewSMTWithStorage(smn, smv, sha256.New())
 
 	max := insert + update + delete
-	kv := make(map[string]string)
+	var kv []bulkop
 
 	for i := 0; i < operations; i++ {
 		n := rand.Intn(max)
@@ -39,98 +46,90 @@ func bulkOperations(t *testing.T, operations int, insert int, update int, delete
 			val := make([]byte, valLen)
 			rand.Read(val)
 
-			kv[string(key)] = string(val)
-			_, err := smt.Update(key, val)
+			err := smt.Update(key, val)
 			if err != nil {
-				t.Errorf("error: %v", err)
+				t.Fatalf("error: %v", err)
 			}
+			kv = append(kv, bulkop{key, val})
 		} else if n > insert && n < insert+update { // Update
-			keys := reflect.ValueOf(kv).MapKeys()
-			if len(keys) == 0 {
+			if len(kv) == 0 {
 				continue
 			}
-			key := []byte(keys[rand.Intn(len(keys))].Interface().(string))
-
+			ki := rand.Intn(len(kv))
 			valLen := 1 + rand.Intn(64)
 			val := make([]byte, valLen)
 			rand.Read(val)
 
-			kv[string(key)] = string(val)
-			_, err := smt.Update(key, val)
+			err := smt.Update(kv[ki].key, val)
 			if err != nil {
-				t.Errorf("error: %v", err)
+				t.Fatalf("error: %v", err)
 			}
+			kv[ki].val = val
 		} else { // Delete
-			keys := reflect.ValueOf(kv).MapKeys()
-			if len(keys) == 0 {
+			if len(kv) == 0 {
 				continue
 			}
-			key := []byte(keys[rand.Intn(len(keys))].Interface().(string))
+			ki := rand.Intn(len(kv))
 
-			kv[string(key)] = ""
-			_, err := smt.Update(key, defaultValue)
-			if err != nil {
-				t.Errorf("error: %v", err)
+			err := smt.Delete(kv[ki].key)
+			if err != nil && err != ErrKeyNotPresent {
+				t.Fatalf("error: %v", err)
 			}
+			kv[ki].val = defaultValue
 		}
-
-		bulkCheckAll(t, smt, &kv)
 	}
+	bulkCheckAll(t, smt, kv)
 }
 
-func bulkCheckAll(t *testing.T, smt *SparseMerkleTree, kv *map[string]string) {
-	for k, v := range *kv {
-		value, err := smt.Get([]byte(k))
+func bulkCheckAll(t *testing.T, smt *SMTWithStorage, kv []bulkop) {
+	for ki := range kv {
+		k, v := kv[ki].key, kv[ki].val
+
+		value, err := smt.GetValue([]byte(k))
 		if err != nil {
 			t.Errorf("error: %v", err)
 		}
 		if !bytes.Equal([]byte(v), value) {
-			t.Error("got incorrect value when bulk testing operations")
+			t.Errorf("Incorrect value (i=%d)", ki)
 		}
 
 		// Generate and verify a Merkle proof for this key.
 		proof, err := smt.Prove([]byte(k))
 		if err != nil {
 			t.Errorf("error: %v", err)
 		}
-		if !VerifyProof(proof, smt.Root(), []byte(k), []byte(v), smt.th.hasher) {
-			t.Error("Merkle proof failed to verify")
+		if !VerifyProof(proof, smt.Root(), []byte(k), []byte(v), smt.Spec()) {
+			t.Fatalf("Merkle proof failed to verify (i=%d): %v", ki, []byte(k))
 		}
-		compactProof, err := smt.ProveCompact([]byte(k))
+		compactProof, err := ProveCompact([]byte(k), smt)
 		if err != nil {
 			t.Errorf("error: %v", err)
 		}
-		if !VerifyCompactProof(compactProof, smt.Root(), []byte(k), []byte(v), smt.th.hasher) {
-			t.Error("Merkle proof failed to verify")
+		if !VerifyCompactProof(compactProof, smt.Root(), []byte(k), []byte(v), smt.Spec()) {
+			t.Fatalf("Compact Merkle proof failed to verify (i=%d): %v", ki, []byte(k))
 		}
 
-		if v == "" {
+		if v == nil {
 			continue
 		}
 
 		// Check that the key is at the correct height in the tree.
 		largestCommonPrefix := 0
-		for k2, v2 := range *kv {
-			if v2 == "" {
+		for ki2 := range kv {
+			k2, v2 := kv[ki2].key, kv[ki2].val
+			if v2 == nil {
 				continue
 			}
-			commonPrefix := countCommonPrefix(smt.th.path([]byte(k)), smt.th.path([]byte(k2)))
-			if commonPrefix != smt.depth() && commonPrefix > largestCommonPrefix {
+
+			ph := smt.Spec().ph
+			commonPrefix := countCommonPrefix(ph.Path([]byte(k)), ph.Path([]byte(k2)), 0)
+			if commonPrefix != smt.Spec().depth() && commonPrefix > largestCommonPrefix {
 				largestCommonPrefix = commonPrefix
 			}
 		}
-		sideNodes, _, _, _, err := smt.sideNodesForRoot(smt.th.path([]byte(k)), smt.Root(), false)
-		if err != nil {
-			t.Errorf("error: %v", err)
-		}
-		numSideNodes := 0
-		for _, v := range sideNodes {
-			if v != nil {
-				numSideNodes++
-			}
-		}
-		if numSideNodes != largestCommonPrefix+1 && (numSideNodes != 0 && largestCommonPrefix != 0) {
-			t.Error("leaf is at unexpected height")
+		if len(proof.SideNodes) != largestCommonPrefix+1 &&
+			(len(proof.SideNodes) != 0 && largestCommonPrefix != 0) {
+			t.Errorf("leaf is at unexpected height (ki=%d)", ki)
 		}
 	}
 }