diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 66f2fdc1b..7e6a30968 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -23,6 +23,7 @@ BREAKING CHANGES: * [rpc/client] \#2298 `ABCIQueryOptions.Trusted` -> `ABCIQueryOptions.Prove` * [types] \#2298 Remove `Index` and `Total` fields from `TxProof`. * [crypto/merkle & lite] \#2298 Various changes to accomodate General Merkle trees + * [crypto/merkle] \#2595 Remove all Hasher objects in favor of byte slices * Blockchain Protocol * [types] \#2459 `Vote`/`Proposal`/`Heartbeat` use amino encoding instead of JSON in `SignBytes`. diff --git a/crypto/merkle/simple_map.go b/crypto/merkle/simple_map.go index ba4b9309a..bde442203 100644 --- a/crypto/merkle/simple_map.go +++ b/crypto/merkle/simple_map.go @@ -1,6 +1,9 @@ package merkle import ( + "bytes" + + amino "github.com/tendermint/go-amino" "github.com/tendermint/tendermint/crypto/tmhash" cmn "github.com/tendermint/tendermint/libs/common" ) @@ -20,14 +23,15 @@ func newSimpleMap() *simpleMap { } } -// Set hashes the key and value and appends it to the kv pairs. -func (sm *simpleMap) Set(key string, value Hasher) { +// Set creates a kv pair of the key and the hash of the value, +// and then appends it to simpleMap's kv pairs. +func (sm *simpleMap) Set(key string, value []byte) { sm.sorted = false // The value is hashed, so you can // check for equality with a cached value (say) // and make a determination to fetch or not. - vhash := value.Hash() + vhash := tmhash.Sum(value) sm.kvs = append(sm.kvs, cmn.KVPair{ Key: []byte(key), @@ -66,23 +70,25 @@ func (sm *simpleMap) KVPairs() cmn.KVPairs { // then hashed. type KVPair cmn.KVPair -func (kv KVPair) Hash() []byte { - hasher := tmhash.New() - err := encodeByteSlice(hasher, kv.Key) +// Bytes returns key || value, with both the +// key and value length prefixed. +func (kv KVPair) Bytes() []byte { + var b bytes.Buffer + err := amino.EncodeByteSlice(&b, kv.Key) if err != nil { panic(err) } - err = encodeByteSlice(hasher, kv.Value) + err = amino.EncodeByteSlice(&b, kv.Value) if err != nil { panic(err) } - return hasher.Sum(nil) + return b.Bytes() } func hashKVPairs(kvs cmn.KVPairs) []byte { - kvsH := make([]Hasher, len(kvs)) + kvsH := make([][]byte, len(kvs)) for i, kvp := range kvs { - kvsH[i] = KVPair(kvp) + kvsH[i] = KVPair(kvp).Bytes() } - return SimpleHashFromHashers(kvsH) + return SimpleHashFromByteSlices(kvsH) } diff --git a/crypto/merkle/simple_map_test.go b/crypto/merkle/simple_map_test.go index 34febcf16..bc095c003 100644 --- a/crypto/merkle/simple_map_test.go +++ b/crypto/merkle/simple_map_test.go @@ -5,50 +5,29 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/tendermint/tendermint/crypto/tmhash" ) -type strHasher string - -func (str strHasher) Hash() []byte { - return tmhash.Sum([]byte(str)) -} - func TestSimpleMap(t *testing.T) { - { - db := newSimpleMap() - db.Set("key1", strHasher("value1")) - assert.Equal(t, "fa9bc106ffd932d919bee935ceb6cf2b3dd72d8f", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") - } - { - db := newSimpleMap() - db.Set("key1", strHasher("value2")) - assert.Equal(t, "e00e7dcfe54e9fafef5111e813a587f01ba9c3e8", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") - } - { - db := newSimpleMap() - db.Set("key1", strHasher("value1")) - db.Set("key2", strHasher("value2")) - assert.Equal(t, "eff12d1c703a1022ab509287c0f196130123d786", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") - } - { - db := newSimpleMap() - db.Set("key2", strHasher("value2")) // NOTE: out of order - db.Set("key1", strHasher("value1")) - assert.Equal(t, "eff12d1c703a1022ab509287c0f196130123d786", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") - } - { - db := newSimpleMap() - db.Set("key1", strHasher("value1")) - db.Set("key2", strHasher("value2")) - db.Set("key3", strHasher("value3")) - assert.Equal(t, "b2c62a277c08dbd2ad73ca53cd1d6bfdf5830d26", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + tests := []struct { + keys []string + values []string // each string gets converted to []byte in test + want string + }{ + {[]string{"key1"}, []string{"value1"}, "fa9bc106ffd932d919bee935ceb6cf2b3dd72d8f"}, + {[]string{"key1"}, []string{"value2"}, "e00e7dcfe54e9fafef5111e813a587f01ba9c3e8"}, + // swap order with 2 keys + {[]string{"key1", "key2"}, []string{"value1", "value2"}, "eff12d1c703a1022ab509287c0f196130123d786"}, + {[]string{"key2", "key1"}, []string{"value2", "value1"}, "eff12d1c703a1022ab509287c0f196130123d786"}, + // swap order with 3 keys + {[]string{"key1", "key2", "key3"}, []string{"value1", "value2", "value3"}, "b2c62a277c08dbd2ad73ca53cd1d6bfdf5830d26"}, + {[]string{"key1", "key3", "key2"}, []string{"value1", "value3", "value2"}, "b2c62a277c08dbd2ad73ca53cd1d6bfdf5830d26"}, } - { + for i, tc := range tests { db := newSimpleMap() - db.Set("key2", strHasher("value2")) // NOTE: out of order - db.Set("key1", strHasher("value1")) - db.Set("key3", strHasher("value3")) - assert.Equal(t, "b2c62a277c08dbd2ad73ca53cd1d6bfdf5830d26", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + for i := 0; i < len(tc.keys); i++ { + db.Set(tc.keys[i], []byte(tc.values[i])) + } + got := db.Hash() + assert.Equal(t, tc.want, fmt.Sprintf("%x", got), "Hash didn't match on tc %d", i) } } diff --git a/crypto/merkle/simple_proof.go b/crypto/merkle/simple_proof.go index 306505fc2..d2cbb126f 100644 --- a/crypto/merkle/simple_proof.go +++ b/crypto/merkle/simple_proof.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" + "github.com/tendermint/tendermint/crypto/tmhash" cmn "github.com/tendermint/tendermint/libs/common" ) @@ -22,10 +23,10 @@ type SimpleProof struct { Aunts [][]byte `json:"aunts"` // Hashes from leaf's sibling to a root's child. } -// SimpleProofsFromHashers computes inclusion proof for given items. +// SimpleProofsFromByteSlices computes inclusion proof for given items. // proofs[0] is the proof for items[0]. -func SimpleProofsFromHashers(items []Hasher) (rootHash []byte, proofs []*SimpleProof) { - trails, rootSPN := trailsFromHashers(items) +func SimpleProofsFromByteSlices(items [][]byte) (rootHash []byte, proofs []*SimpleProof) { + trails, rootSPN := trailsFromByteSlices(items) rootHash = rootSPN.Hash proofs = make([]*SimpleProof, len(items)) for i, trail := range trails { @@ -42,19 +43,19 @@ func SimpleProofsFromHashers(items []Hasher) (rootHash []byte, proofs []*SimpleP // SimpleProofsFromMap generates proofs from a map. The keys/values of the map will be used as the keys/values // in the underlying key-value pairs. // The keys are sorted before the proofs are computed. -func SimpleProofsFromMap(m map[string]Hasher) (rootHash []byte, proofs map[string]*SimpleProof, keys []string) { +func SimpleProofsFromMap(m map[string][]byte) (rootHash []byte, proofs map[string]*SimpleProof, keys []string) { sm := newSimpleMap() for k, v := range m { sm.Set(k, v) } sm.Sort() kvs := sm.kvs - kvsH := make([]Hasher, 0, len(kvs)) - for _, kvp := range kvs { - kvsH = append(kvsH, KVPair(kvp)) + kvsBytes := make([][]byte, len(kvs)) + for i, kvp := range kvs { + kvsBytes[i] = KVPair(kvp).Bytes() } - rootHash, proofList := SimpleProofsFromHashers(kvsH) + rootHash, proofList := SimpleProofsFromByteSlices(kvsBytes) proofs = make(map[string]*SimpleProof) keys = make([]string, len(proofList)) for i, kvp := range kvs { @@ -175,17 +176,17 @@ func (spn *SimpleProofNode) FlattenAunts() [][]byte { // trails[0].Hash is the leaf hash for items[0]. // trails[i].Parent.Parent....Parent == root for all i. -func trailsFromHashers(items []Hasher) (trails []*SimpleProofNode, root *SimpleProofNode) { +func trailsFromByteSlices(items [][]byte) (trails []*SimpleProofNode, root *SimpleProofNode) { // Recursive impl. switch len(items) { case 0: return nil, nil case 1: - trail := &SimpleProofNode{items[0].Hash(), nil, nil, nil} + trail := &SimpleProofNode{tmhash.Sum(items[0]), nil, nil, nil} return []*SimpleProofNode{trail}, trail default: - lefts, leftRoot := trailsFromHashers(items[:(len(items)+1)/2]) - rights, rightRoot := trailsFromHashers(items[(len(items)+1)/2:]) + lefts, leftRoot := trailsFromByteSlices(items[:(len(items)+1)/2]) + rights, rightRoot := trailsFromByteSlices(items[(len(items)+1)/2:]) rootHash := SimpleHashFromTwoHashes(leftRoot.Hash, rightRoot.Hash) root := &SimpleProofNode{rootHash, nil, nil, nil} leftRoot.Parent = root diff --git a/crypto/merkle/simple_tree.go b/crypto/merkle/simple_tree.go index 46a075909..9677aef4e 100644 --- a/crypto/merkle/simple_tree.go +++ b/crypto/merkle/simple_tree.go @@ -18,11 +18,12 @@ func SimpleHashFromTwoHashes(left, right []byte) []byte { return hasher.Sum(nil) } -// SimpleHashFromHashers computes a Merkle tree from items that can be hashed. -func SimpleHashFromHashers(items []Hasher) []byte { +// SimpleHashFromByteSlices computes a Merkle tree where the leaves are the byte slice, +// in the provided order. +func SimpleHashFromByteSlices(items [][]byte) []byte { hashes := make([][]byte, len(items)) for i, item := range items { - hash := item.Hash() + hash := tmhash.Sum(item) hashes[i] = hash } return simpleHashFromHashes(hashes) @@ -32,7 +33,7 @@ func SimpleHashFromHashers(items []Hasher) []byte { // Like calling SimpleHashFromHashers with // `item = []byte(Hash(key) | Hash(value))`, // sorted by `item`. -func SimpleHashFromMap(m map[string]Hasher) []byte { +func SimpleHashFromMap(m map[string][]byte) []byte { sm := newSimpleMap() for k, v := range m { sm.Set(k, v) diff --git a/crypto/merkle/simple_tree_test.go b/crypto/merkle/simple_tree_test.go index b299aba78..32edc652e 100644 --- a/crypto/merkle/simple_tree_test.go +++ b/crypto/merkle/simple_tree_test.go @@ -21,20 +21,20 @@ func TestSimpleProof(t *testing.T) { total := 100 - items := make([]Hasher, total) + items := make([][]byte, total) for i := 0; i < total; i++ { items[i] = testItem(cmn.RandBytes(tmhash.Size)) } - rootHash := SimpleHashFromHashers(items) + rootHash := SimpleHashFromByteSlices(items) - rootHash2, proofs := SimpleProofsFromHashers(items) + rootHash2, proofs := SimpleProofsFromByteSlices(items) require.Equal(t, rootHash, rootHash2, "Unmatched root hashes: %X vs %X", rootHash, rootHash2) // For each item, check the trail. for i, item := range items { - itemHash := item.Hash() + itemHash := tmhash.Sum(item) proof := proofs[i] // Check total/index diff --git a/crypto/merkle/types.go b/crypto/merkle/types.go index 2fcb3f39d..97a47879b 100644 --- a/crypto/merkle/types.go +++ b/crypto/merkle/types.go @@ -25,11 +25,6 @@ type Tree interface { IterateRange(start []byte, end []byte, ascending bool, fx func(key []byte, value []byte) (stop bool)) (stopped bool) } -// Hasher represents a hashable piece of data which can be hashed in the Tree. -type Hasher interface { - Hash() []byte -} - //----------------------------------------------------------------------- // Uvarint length prefixed byteslice diff --git a/types/block.go b/types/block.go index 5610cc799..07a71ca82 100644 --- a/types/block.go +++ b/types/block.go @@ -9,7 +9,6 @@ import ( "time" "github.com/tendermint/tendermint/crypto/merkle" - "github.com/tendermint/tendermint/crypto/tmhash" cmn "github.com/tendermint/tendermint/libs/common" ) @@ -290,22 +289,22 @@ func (h *Header) Hash() cmn.HexBytes { if h == nil || len(h.ValidatorsHash) == 0 { return nil } - return merkle.SimpleHashFromMap(map[string]merkle.Hasher{ - "ChainID": aminoHasher(h.ChainID), - "Height": aminoHasher(h.Height), - "Time": aminoHasher(h.Time), - "NumTxs": aminoHasher(h.NumTxs), - "TotalTxs": aminoHasher(h.TotalTxs), - "LastBlockID": aminoHasher(h.LastBlockID), - "LastCommit": aminoHasher(h.LastCommitHash), - "Data": aminoHasher(h.DataHash), - "Validators": aminoHasher(h.ValidatorsHash), - "NextValidators": aminoHasher(h.NextValidatorsHash), - "App": aminoHasher(h.AppHash), - "Consensus": aminoHasher(h.ConsensusHash), - "Results": aminoHasher(h.LastResultsHash), - "Evidence": aminoHasher(h.EvidenceHash), - "Proposer": aminoHasher(h.ProposerAddress), + return merkle.SimpleHashFromMap(map[string][]byte{ + "ChainID": cdcEncode(h.ChainID), + "Height": cdcEncode(h.Height), + "Time": cdcEncode(h.Time), + "NumTxs": cdcEncode(h.NumTxs), + "TotalTxs": cdcEncode(h.TotalTxs), + "LastBlockID": cdcEncode(h.LastBlockID), + "LastCommit": cdcEncode(h.LastCommitHash), + "Data": cdcEncode(h.DataHash), + "Validators": cdcEncode(h.ValidatorsHash), + "NextValidators": cdcEncode(h.NextValidatorsHash), + "App": cdcEncode(h.AppHash), + "Consensus": cdcEncode(h.ConsensusHash), + "Results": cdcEncode(h.LastResultsHash), + "Evidence": cdcEncode(h.EvidenceHash), + "Proposer": cdcEncode(h.ProposerAddress), }) } @@ -480,11 +479,11 @@ func (commit *Commit) Hash() cmn.HexBytes { return nil } if commit.hash == nil { - bs := make([]merkle.Hasher, len(commit.Precommits)) + bs := make([][]byte, len(commit.Precommits)) for i, precommit := range commit.Precommits { - bs[i] = aminoHasher(precommit) + bs[i] = cdcEncode(precommit) } - commit.hash = merkle.SimpleHashFromHashers(bs) + commit.hash = merkle.SimpleHashFromByteSlices(bs) } return commit.hash } @@ -689,33 +688,3 @@ func (blockID BlockID) Key() string { func (blockID BlockID) String() string { return fmt.Sprintf(`%v:%v`, blockID.Hash, blockID.PartsHeader) } - -//------------------------------------------------------- - -type hasher struct { - item interface{} -} - -func (h hasher) Hash() []byte { - hasher := tmhash.New() - if h.item != nil && !cmn.IsTypedNil(h.item) && !cmn.IsEmpty(h.item) { - bz, err := cdc.MarshalBinaryBare(h.item) - if err != nil { - panic(err) - } - _, err = hasher.Write(bz) - if err != nil { - panic(err) - } - } - return hasher.Sum(nil) -} - -func aminoHash(item interface{}) []byte { - h := hasher{item} - return h.Hash() -} - -func aminoHasher(item interface{}) merkle.Hasher { - return hasher{item} -} diff --git a/types/encoding_helper.go b/types/encoding_helper.go new file mode 100644 index 000000000..f825de8a6 --- /dev/null +++ b/types/encoding_helper.go @@ -0,0 +1,14 @@ +package types + +import ( + cmn "github.com/tendermint/tendermint/libs/common" +) + +// cdcEncode returns nil if the input is nil, otherwise returns +// cdc.MustMarshalBinaryBare(item) +func cdcEncode(item interface{}) []byte { + if item != nil && !cmn.IsTypedNil(item) && !cmn.IsEmpty(item) { + return cdc.MustMarshalBinaryBare(item) + } + return nil +} diff --git a/types/evidence.go b/types/evidence.go index 916dd094f..00c46c593 100644 --- a/types/evidence.go +++ b/types/evidence.go @@ -4,6 +4,8 @@ import ( "bytes" "fmt" + "github.com/tendermint/tendermint/crypto/tmhash" + amino "github.com/tendermint/go-amino" "github.com/tendermint/tendermint/crypto" @@ -104,7 +106,7 @@ func (dve *DuplicateVoteEvidence) Address() []byte { // Hash returns the hash of the evidence. func (dve *DuplicateVoteEvidence) Hash() []byte { - return aminoHasher(dve).Hash() + return tmhash.Sum(cdcEncode(dve)) } // Verify returns an error if the two votes aren't conflicting. @@ -157,8 +159,8 @@ func (dve *DuplicateVoteEvidence) Equal(ev Evidence) bool { } // just check their hashes - dveHash := aminoHasher(dve).Hash() - evHash := aminoHasher(ev).Hash() + dveHash := tmhash.Sum(cdcEncode(dve)) + evHash := tmhash.Sum(cdcEncode(ev)) return bytes.Equal(dveHash, evHash) } diff --git a/types/params.go b/types/params.go index 014694ccb..129d47627 100644 --- a/types/params.go +++ b/types/params.go @@ -82,10 +82,10 @@ func (params *ConsensusParams) Validate() error { // Hash returns a merkle hash of the parameters to store in the block header func (params *ConsensusParams) Hash() []byte { - return merkle.SimpleHashFromMap(map[string]merkle.Hasher{ - "block_size_max_bytes": aminoHasher(params.BlockSize.MaxBytes), - "block_size_max_gas": aminoHasher(params.BlockSize.MaxGas), - "evidence_params_max_age": aminoHasher(params.EvidenceParams.MaxAge), + return merkle.SimpleHashFromMap(map[string][]byte{ + "block_size_max_bytes": cdcEncode(params.BlockSize.MaxBytes), + "block_size_max_gas": cdcEncode(params.BlockSize.MaxGas), + "evidence_params_max_age": cdcEncode(params.EvidenceParams.MaxAge), }) } diff --git a/types/part_set.go b/types/part_set.go index 8c8151ba8..812b1c2fd 100644 --- a/types/part_set.go +++ b/types/part_set.go @@ -88,7 +88,7 @@ func NewPartSetFromData(data []byte, partSize int) *PartSet { // divide data into 4kb parts. total := (len(data) + partSize - 1) / partSize parts := make([]*Part, total) - parts_ := make([]merkle.Hasher, total) + partsBytes := make([][]byte, total) partsBitArray := cmn.NewBitArray(total) for i := 0; i < total; i++ { part := &Part{ @@ -96,11 +96,11 @@ func NewPartSetFromData(data []byte, partSize int) *PartSet { Bytes: data[i*partSize : cmn.MinInt(len(data), (i+1)*partSize)], } parts[i] = part - parts_[i] = part + partsBytes[i] = part.Bytes partsBitArray.SetIndex(i, true) } // Compute merkle proofs - root, proofs := merkle.SimpleProofsFromHashers(parts_) + root, proofs := merkle.SimpleProofsFromByteSlices(partsBytes) for i := 0; i < total; i++ { parts[i].Proof = *proofs[i] } diff --git a/types/results.go b/types/results.go index 17d5891c3..6b5b82d27 100644 --- a/types/results.go +++ b/types/results.go @@ -3,6 +3,7 @@ package types import ( abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/crypto/merkle" + "github.com/tendermint/tendermint/crypto/tmhash" cmn "github.com/tendermint/tendermint/libs/common" ) @@ -17,10 +18,14 @@ type ABCIResult struct { // Hash returns the canonical hash of the ABCIResult func (a ABCIResult) Hash() []byte { - bz := aminoHash(a) + bz := tmhash.Sum(cdcEncode(a)) return bz } +func (a ABCIResult) Bytes() []byte { + return cdcEncode(a) +} + // ABCIResults wraps the deliver tx results to return a proof type ABCIResults []ABCIResult @@ -54,20 +59,20 @@ func (a ABCIResults) Bytes() []byte { func (a ABCIResults) Hash() []byte { // NOTE: we copy the impl of the merkle tree for txs - // we should be consistent and either do it for both or not. - return merkle.SimpleHashFromHashers(a.toHashers()) + return merkle.SimpleHashFromByteSlices(a.toByteSlices()) } // ProveResult returns a merkle proof of one result from the set func (a ABCIResults) ProveResult(i int) merkle.SimpleProof { - _, proofs := merkle.SimpleProofsFromHashers(a.toHashers()) + _, proofs := merkle.SimpleProofsFromByteSlices(a.toByteSlices()) return *proofs[i] } -func (a ABCIResults) toHashers() []merkle.Hasher { +func (a ABCIResults) toByteSlices() [][]byte { l := len(a) - hashers := make([]merkle.Hasher, l) + bzs := make([][]byte, l) for i := 0; i < l; i++ { - hashers[i] = a[i] + bzs[i] = a[i].Bytes() } - return hashers + return bzs } diff --git a/types/tx.go b/types/tx.go index 41fc310f1..ec42f3f13 100644 --- a/types/tx.go +++ b/types/tx.go @@ -70,11 +70,11 @@ func (txs Txs) IndexByHash(hash []byte) int { // TODO: optimize this! func (txs Txs) Proof(i int) TxProof { l := len(txs) - hashers := make([]merkle.Hasher, l) + bzs := make([][]byte, l) for i := 0; i < l; i++ { - hashers[i] = txs[i] + bzs[i] = txs[i] } - root, proofs := merkle.SimpleProofsFromHashers(hashers) + root, proofs := merkle.SimpleProofsFromByteSlices(bzs) return TxProof{ RootHash: root, diff --git a/types/validator.go b/types/validator.go index 46d1a7a9f..af3471848 100644 --- a/types/validator.go +++ b/types/validator.go @@ -4,6 +4,8 @@ import ( "bytes" "fmt" + "github.com/tendermint/tendermint/crypto/tmhash" + "github.com/tendermint/tendermint/crypto" cmn "github.com/tendermint/tendermint/libs/common" ) @@ -71,13 +73,21 @@ func (v *Validator) String() string { // Hash computes the unique ID of a validator with a given voting power. // It excludes the Accum value, which changes with every round. func (v *Validator) Hash() []byte { - return aminoHash(struct { + return tmhash.Sum(v.Bytes()) +} + +// Bytes computes the unique encoding of a validator with a given voting power. +// These are the bytes that gets hashed in consensus. It excludes pubkey +// as its redundant with the address. This also excludes accum which changes +// every round. +func (v *Validator) Bytes() []byte { + return cdcEncode((struct { Address Address VotingPower int64 }{ v.Address, v.VotingPower, - }) + })) } //---------------------------------------- diff --git a/types/validator_set.go b/types/validator_set.go index 4dab4d840..72ab68c08 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -176,11 +176,11 @@ func (vals *ValidatorSet) Hash() []byte { if len(vals.Validators) == 0 { return nil } - hashers := make([]merkle.Hasher, len(vals.Validators)) + bzs := make([][]byte, len(vals.Validators)) for i, val := range vals.Validators { - hashers[i] = val + bzs[i] = val.Bytes() } - return merkle.SimpleHashFromHashers(hashers) + return merkle.SimpleHashFromByteSlices(bzs) } // Add adds val to the validator set and returns true. It returns false if val