diff --git a/merkle/simple_tree.go b/merkle/simple_tree.go new file mode 100644 index 000000000..8c6491f17 --- /dev/null +++ b/merkle/simple_tree.go @@ -0,0 +1,173 @@ +/* +Computes a deterministic minimal height merkle tree hash. +If the number of items is not a power of two, some leaves +will be at different levels, leaning left. + +Use this for shorter deterministic trees, such as the validator list. +For larger datasets, use IAVLTree. + + * + / \ + / \ + / \ + / \ + * * + / \ / \ + / \ / \ + / \ / \ + * * * h6 + / \ / \ / \ + h0 h1 h2 h3 h4 h5 + +*/ + +package merkle + +import ( + "bytes" + "crypto/sha256" + + . "github.com/tendermint/tendermint/binary" +) + +func HashFromTwoHashes(left []byte, right []byte) []byte { + var n int64 + var err error + var hasher = sha256.New() + WriteByteSlice(hasher, left, &n, &err) + WriteByteSlice(hasher, right, &n, &err) + if err != nil { + panic(err) + } + return hasher.Sum(nil) +} + +func HashFromHashes(hashes [][]byte) []byte { + // Recursive impl. + switch len(hashes) { + case 0: + panic("Cannot call HashFromHashes() with 0 length arg") + case 1: + return hashes[0] + default: + left := HashFromHashes(hashes[:(len(hashes)+1)/2]) + right := HashFromHashes(hashes[(len(hashes)+1)/2:]) + return HashFromTwoHashes(left, right) + } +} + +// Convenience for HashFromHashes. +func HashFromBinaries(items []Binary) []byte { + hashes := [][]byte{} + for _, item := range items { + hasher := sha256.New() + _, err := item.WriteTo(hasher) + if err != nil { + panic(err) + } + hash := hasher.Sum(nil) + hashes = append(hashes, hash) + } + return HashFromHashes(hashes) +} + +// Convenience for HashFromHashes. +func HashFromHashables(items []Hashable) []byte { + hashes := [][]byte{} + for _, item := range items { + hash := item.Hash() + hashes = append(hashes, hash) + } + return HashFromHashes(hashes) +} + +type HashTrail struct { + Hash []byte + Parent *HashTrail + Left *HashTrail + Right *HashTrail +} + +func (ht *HashTrail) Flatten() [][]byte { + // Nonrecursive impl. + trail := [][]byte{} + for ht != nil { + if ht.Left != nil { + trail = append(trail, ht.Left.Hash) + } else if ht.Right != nil { + trail = append(trail, ht.Right.Hash) + } else { + break + } + ht = ht.Parent + } + return trail +} + +// returned trails[0].Hash is the leaf hash. +// trails[0].Parent.Hash is the hash above that, etc. +func HashTrailsFromHashables(items []Hashable) (trails []*HashTrail, root *HashTrail) { + // Recursive impl. + switch len(items) { + case 0: + panic("Cannot call HashTrailsFromHashables() with 0 length arg") + case 1: + trail := &HashTrail{items[0].Hash(), nil, nil, nil} + return []*HashTrail{trail}, trail + default: + lefts, leftRoot := HashTrailsFromHashables(items[:(len(items)+1)/2]) + rights, rightRoot := HashTrailsFromHashables(items[(len(items)+1)/2:]) + rootHash := HashFromTwoHashes(leftRoot.Hash, rightRoot.Hash) + root := &HashTrail{rootHash, nil, nil, nil} + leftRoot.Parent = root + leftRoot.Right = rightRoot + rightRoot.Parent = root + rightRoot.Left = leftRoot + return append(lefts, rights...), root + } +} + +// Ensures that leafHash is part of rootHash. +func VerifyHashTrail(index uint, total uint, leafHash []byte, trail [][]byte, rootHash []byte) bool { + computedRoot := ComputeRootFromTrail(index, total, leafHash, trail) + if computedRoot == nil { + return false + } + return bytes.Equal(computedRoot, rootHash) +} + +// Use the leafHash and trail to get the root merkle hash. +// If the length of the trail slice isn't exactly correct, the result is nil. +func ComputeRootFromTrail(index uint, total uint, leafHash []byte, trail [][]byte) []byte { + // Recursive impl. + if index >= total { + return nil + } + switch total { + case 0: + panic("Cannot call ComputeRootFromTrail() with 0 total") + case 1: + if len(trail) != 0 { + return nil + } + return leafHash + default: + if len(trail) == 0 { + return nil + } + numLeft := (total + 1) / 2 + if index < numLeft { + leftRoot := ComputeRootFromTrail(index, numLeft, leafHash, trail[:len(trail)-1]) + if leftRoot == nil { + return nil + } + return HashFromTwoHashes(leftRoot, trail[len(trail)-1]) + } else { + rightRoot := ComputeRootFromTrail(index-numLeft, total-numLeft, leafHash, trail[:len(trail)-1]) + if rightRoot == nil { + return nil + } + return HashFromTwoHashes(trail[len(trail)-1], rightRoot) + } + } +} diff --git a/merkle/simple_tree_test.go b/merkle/simple_tree_test.go new file mode 100644 index 000000000..893a30ce9 --- /dev/null +++ b/merkle/simple_tree_test.go @@ -0,0 +1,74 @@ +package merkle + +import ( + . "github.com/tendermint/tendermint/common" + + "bytes" + "testing" +) + +type testItem []byte + +func (tI testItem) Hash() []byte { + return []byte(tI) +} + +func TestMerkleTrails(t *testing.T) { + + numItems := uint(100) + + items := make([]Hashable, numItems) + for i := uint(0); i < numItems; i++ { + items[i] = testItem(RandBytes(32)) + } + + root := HashFromHashables(items) + + trails, rootTrail := HashTrailsFromHashables(items) + + // Assert that HashFromHashables and HashTrailsFromHashables are compatible. + if !bytes.Equal(root, rootTrail.Hash) { + t.Errorf("Root mismatch:\n%X vs\n%X", root, rootTrail.Hash) + } + + // For each item, check the trail. + for i, item := range items { + itemHash := item.Hash() + flatTrail := trails[i].Flatten() + + // Verify success + ok := VerifyHashTrail(uint(i), numItems, itemHash, flatTrail, root) + if !ok { + t.Errorf("Verification failed for index %v.", i) + } + + // Wrong item index should make it fail + ok = VerifyHashTrail(uint(i)+1, numItems, itemHash, flatTrail, root) + if ok { + t.Errorf("Expected verification to fail for wrong index %v.", i) + } + + // Trail too long should make it fail + trail2 := append(flatTrail, RandBytes(32)) + ok = VerifyHashTrail(uint(i), numItems, itemHash, trail2, root) + if ok { + t.Errorf("Expected verification to fail for wrong trail length.") + } + + // Trail too short should make it fail + trail2 = flatTrail[:len(flatTrail)-1] + ok = VerifyHashTrail(uint(i), numItems, itemHash, trail2, root) + if ok { + t.Errorf("Expected verification to fail for wrong trail length.") + } + + // Mutating the itemHash should make it fail. + itemHash2 := make([]byte, len(itemHash)) + copy(itemHash2, itemHash) + itemHash2[0] += byte(0x01) + ok = VerifyHashTrail(uint(i), numItems, itemHash2, flatTrail, root) + if ok { + t.Errorf("Expected verification to fail for mutated leaf hash") + } + } +} diff --git a/merkle/util.go b/merkle/util.go index ab32740b9..edc3fbf74 100644 --- a/merkle/util.go +++ b/merkle/util.go @@ -1,230 +1,10 @@ package merkle import ( - "bytes" - "crypto/sha256" "fmt" - - . "github.com/tendermint/tendermint/binary" ) -func HashFromTwoHashes(left []byte, right []byte) []byte { - var n int64 - var err error - var hasher = sha256.New() - WriteByteSlice(hasher, left, &n, &err) - WriteByteSlice(hasher, right, &n, &err) - if err != nil { - panic(err) - } - return hasher.Sum(nil) -} - -/* -Computes a deterministic minimal height merkle tree hash. -If the number of items is not a power of two, some leaves -will be at different levels. - - * - / \ - / \ - / \ - / \ - * * - / \ / \ - / \ / \ - / \ / \ - * h2 * * - / \ / \ / \ - h0 h1 h3 h4 h5 h6 - -*/ -func HashFromHashes(hashes [][]byte) []byte { - switch len(hashes) { - case 0: - return nil - case 1: - return hashes[0] - default: - left := HashFromHashes(hashes[:len(hashes)/2]) - right := HashFromHashes(hashes[len(hashes)/2:]) - return HashFromTwoHashes(left, right) - } -} - -// Convenience for HashFromHashes. -func HashFromBinaries(items []Binary) []byte { - hashes := [][]byte{} - for _, item := range items { - hasher := sha256.New() - _, err := item.WriteTo(hasher) - if err != nil { - panic(err) - } - hash := hasher.Sum(nil) - hashes = append(hashes, hash) - } - return HashFromHashes(hashes) -} - -// Convenience for HashFromHashes. -func HashFromHashables(items []Hashable) []byte { - hashes := [][]byte{} - for _, item := range items { - hash := item.Hash() - hashes = append(hashes, hash) - } - return HashFromHashes(hashes) -} - -/* -Calculates an array of hashes, useful for deriving hash trails. - - 7 - / \ - / \ - / \ - / \ - 3 11 - / \ / \ - / \ / \ - / \ / \ - 1 5 9 13 - / \ / \ / \ / \ - 0 2 4 6 8 10 12 14 - h0 h1 h2 h3 h4 h5 h6 h7 - -(diagram and idea borrowed from libswift) - -The hashes provided get assigned to even indices. -The derived merkle hashes get assigned to odd indices. -If "hashes" is not of length power of 2, it is padded -with blank (zeroed) hashes. -*/ -func HashTreeFromHashes(hashes [][]byte) [][]byte { - - // Make length of "hashes" a power of 2 - hashesLen := uint32(len(hashes)) - fullLen := uint32(1) - for { - if fullLen >= hashesLen { - break - } else { - fullLen <<= 1 - } - } - blank := make([]byte, len(hashes[0])) - for i := hashesLen; i < fullLen; i++ { - hashes = append(hashes, blank) - } - - // The result is twice the length minus one. - res := make([][]byte, len(hashes)*2-1) - for i, hash := range hashes { - res[i*2] = hash - } - - // Fill all the hashes recursively. - fillTreeRoot(res, 0, len(res)-1) - return res -} - -// Fill in the blanks. -func fillTreeRoot(res [][]byte, start, end int) []byte { - if start == end { - return res[start] - } else { - mid := (start + end) / 2 - left := fillTreeRoot(res, start, mid-1) - right := fillTreeRoot(res, mid+1, end) - root := HashFromTwoHashes(left, right) - res[mid] = root - return root - } -} - -// Convenience for HashTreeFromHashes. -func HashTreeFromHashables(items []Hashable) [][]byte { - hashes := [][]byte{} - for _, item := range items { - hash := item.Hash() - hashes = append(hashes, hash) - } - return HashTreeFromHashes(hashes) -} - -/* -Given the original index of an item, -(e.g. for h5 in the diagram above, the index is 5, not 10) -returns a trail of hashes, which along with the index can be -used to calculate the merkle root. - -See VerifyHashTrailForIndex() -*/ -func HashTrailForIndex(hashTree [][]byte, index int) [][]byte { - trail := [][]byte{} - index *= 2 - - // We start from the leaf layer and work our way up. - // Notice the indices in the diagram: - // 0 2 4 ... offset 0, stride 2 - // 1 5 9 ... offset 1, stride 4 - // 3 11 19 ... offset 3, stride 8 - // 7 23 39 ... offset 7, stride 16 etc. - - offset := 0 - stride := 2 - - for { - // Calculate sibling of index. - var next int - if ((index-offset)/stride)%2 == 0 { - next = index + stride - } else { - next = index - stride - } - if next >= len(hashTree) { - break - } - // Insert sibling hash to trail. - trail = append(trail, hashTree[next]) - - index = (index + next) / 2 - offset += stride / 2 - stride *= 2 - } - - return trail -} - -// Ensures that leafHash is part of rootHash. -func VerifyHashTrailForIndex(index int, leafHash []byte, trail [][]byte, rootHash []byte) bool { - index *= 2 - offset := 0 - stride := 2 - - tempHash := make([]byte, len(leafHash)) - copy(tempHash, leafHash) - - for i := 0; i < len(trail); i++ { - var next int - if ((index-offset)/stride)%2 == 0 { - next = index + stride - tempHash = HashFromTwoHashes(tempHash, trail[i]) - } else { - next = index - stride - tempHash = HashFromTwoHashes(trail[i], tempHash) - } - index = (index + next) / 2 - offset += stride / 2 - stride *= 2 - } - - return bytes.Equal(rootHash, tempHash) -} - -//----------------------------------------------------------------------------- - +// Prints the in-memory children recursively. func PrintIAVLNode(node *IAVLNode) { fmt.Println("==== NODE") if node != nil { diff --git a/merkle/util_test.go b/merkle/util_test.go deleted file mode 100644 index a23ba67a9..000000000 --- a/merkle/util_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package merkle - -import ( - . "github.com/tendermint/tendermint/common" - - "testing" -) - -// TODO: Actually test. All this does is help debug. -// consensus/part_set tests some of this functionality. -func TestHashTreeMerkleTrail(t *testing.T) { - - numHashes := 5 - - // Make some fake "hashes". - hashes := make([][]byte, numHashes) - for i := 0; i < numHashes; i++ { - hashes[i] = RandBytes(32) - t.Logf("hash %v\t%X\n", i, hashes[i]) - } - - hashTree := HashTreeFromHashes(hashes) - for i := 0; i < len(hashTree); i++ { - t.Logf("tree %v\t%X\n", i, hashTree[i]) - } - - for i := 0; i < numHashes; i++ { - t.Logf("trail %v\n", i) - trail := HashTrailForIndex(hashTree, i) - for j := 0; j < len(trail); j++ { - t.Logf("index: %v, hash: %X\n", j, trail[j]) - } - } - -}