diff --git a/common/bytes.go b/common/bytes.go new file mode 100644 index 000000000..d9ede98df --- /dev/null +++ b/common/bytes.go @@ -0,0 +1,53 @@ +package common + +import ( + "encoding/hex" + "fmt" + "strings" +) + +// The main purpose of Bytes is to enable HEX-encoding for json/encoding. +type Bytes []byte + +// Marshal needed for protobuf compatibility +func (b Bytes) Marshal() ([]byte, error) { + return b, nil +} + +// Unmarshal needed for protobuf compatibility +func (b *Bytes) Unmarshal(data []byte) error { + *b = data + return nil +} + +// This is the point of Bytes. +func (b Bytes) MarshalJSON() ([]byte, error) { + s := strings.ToUpper(hex.EncodeToString(b)) + jb := make([]byte, len(s)+2) + jb[0] = '"' + copy(jb[1:], []byte(s)) + jb[1] = '"' + return jb, nil +} + +// This is the point of Bytes. +func (b *Bytes) UnmarshalJSON(data []byte) error { + if len(data) < 2 || data[0] != '"' || data[len(data)-1] != '"' { + return fmt.Errorf("Invalid hex string: %s", data) + } + bytes, err := hex.DecodeString(string(data[1 : len(data)-1])) + if err != nil { + return err + } + *b = bytes + return nil +} + +// Allow it to fulfill various interfaces in light-client, etc... +func (b Bytes) Bytes() []byte { + return b +} + +func (b Bytes) String() string { + return strings.ToUpper(hex.EncodeToString(b)) +} diff --git a/common/bytes_test.go b/common/bytes_test.go new file mode 100644 index 000000000..0c0eacc33 --- /dev/null +++ b/common/bytes_test.go @@ -0,0 +1,68 @@ +package common + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +// This is a trivial test for protobuf compatibility. +func TestMarshal(t *testing.T) { + assert := assert.New(t) + + b := []byte("hello world") + dataB := Bytes(b) + b2, err := dataB.Marshal() + assert.Nil(err) + assert.Equal(b, b2) + + var dataB2 Bytes + err = (&dataB2).Unmarshal(b) + assert.Nil(err) + assert.Equal(dataB, dataB2) +} + +// Test that the hex encoding works. +func TestJSONMarshal(t *testing.T) { + assert := assert.New(t) + + type TestStruct struct { + B1 []byte + B2 Bytes + } + + cases := []struct { + input []byte + expected string + }{ + {[]byte(``), `{"B1":"","B2":""}`}, + {[]byte(``), `{"B1":"","B2":""}`}, + {[]byte(``), `{"B1":"","B2":""}`}, + } + + for i, tc := range cases { + t.Run(fmt.Sprintf("Case %d", i), func(t *testing.T) { + ts := TestStruct{B1: tc.input, B2: tc.input} + + // Test that it marshals correctly to JSON. + jsonBytes, err := json.Marshal(ts) + if err != nil { + t.Fatal(err) + } + assert.Equal(string(jsonBytes), tc.expected) + + // TODO do fuzz testing to ensure that unmarshal fails + + // Test that unmarshaling works correctly. + ts2 := TestStruct{} + err = json.Unmarshal(jsonBytes, &ts2) + if err != nil { + t.Fatal(err) + } + assert.Equal(ts2.B1, tc.input) + assert.Equal(ts2.B2, Bytes(tc.input)) + }) + } +} diff --git a/common/kvpair.go b/common/kvpair.go new file mode 100644 index 000000000..b9e45733f --- /dev/null +++ b/common/kvpair.go @@ -0,0 +1,30 @@ +package common + +import ( + "bytes" + "sort" +) + +type KVPair struct { + Key Bytes + Value Bytes +} + +type KVPairs []KVPair + +// Sorting +func (kvs KVPairs) Len() int { return len(kvs) } +func (kvs KVPairs) Less(i, j int) bool { + switch bytes.Compare(kvs[i].Key, kvs[j].Key) { + case -1: + return true + case 0: + return bytes.Compare(kvs[i].Value, kvs[j].Value) < 0 + case 1: + return false + default: + panic("invalid comparison result") + } +} +func (kvs KVPairs) Swap(i, j int) { kvs[i], kvs[j] = kvs[j], kvs[i] } +func (kvs KVPairs) Sort() { sort.Sort(kvs) } diff --git a/glide.lock b/glide.lock index b0b3ff3c7..e87782d21 100644 --- a/glide.lock +++ b/glide.lock @@ -1,10 +1,10 @@ hash: 6efda1f3891a7211fc3dc1499c0079267868ced9739b781928af8e225420f867 -updated: 2017-08-11T20:28:34.550901198Z +updated: 2017-12-17T12:50:35.983353926-08:00 imports: - name: github.com/fsnotify/fsnotify version: 4da3e2cfbabc9f751898f250b49f2439785783a1 - name: github.com/go-kit/kit - version: 0873e56b0faeae3a1d661b10d629135508ea5504 + version: e3b2152e0063c5f05efea89ecbe297852af2a92d subpackages: - log - log/level @@ -12,17 +12,17 @@ imports: - name: github.com/go-logfmt/logfmt version: 390ab7935ee28ec6b286364bba9b4dd6410cb3d5 - name: github.com/go-playground/locales - version: 1e5f1161c6416a5ff48840eb8724a394e48cc534 + version: e4cbcb5d0652150d40ad0646651076b6bd2be4f6 subpackages: - currency - name: github.com/go-playground/universal-translator version: 71201497bace774495daed26a3874fd339e0b538 - name: github.com/go-stack/stack - version: 7a2f19628aabfe68f0766b59e74d6315f8347d22 + version: 259ab82a6cad3992b4e21ff5cac294ccb06474bc - name: github.com/golang/snappy version: 553a641470496b2327abcac10b36396bd98e45c9 - name: github.com/hashicorp/hcl - version: a4b07c25de5ff55ad3b8936cea69a79a3d95a855 + version: 23c074d0eceb2b8a5bfdbb271ab780cde70f05a8 subpackages: - hcl/ast - hcl/parser @@ -39,35 +39,33 @@ imports: - name: github.com/kr/logfmt version: b84e30acd515aadc4b783ad4ff83aff3299bdfe0 - name: github.com/magiconair/properties - version: 51463bfca2576e06c62a8504b5c0f06d61312647 + version: 49d762b9817ba1c2e9d0c69183c2b4a8b8f1d934 - name: github.com/mattn/go-colorable - version: ded68f7a9561c023e790de24279db7ebf473ea80 + version: 6fcc0c1fd9b620311d821b106a400b35dc95c497 - name: github.com/mattn/go-isatty - version: fc9e8d8ef48496124e79ae0df75490096eccf6fe + version: a5cdd64afdee435007ee3e9f6ed4684af949d568 - name: github.com/mitchellh/mapstructure - version: cc8532a8e9a55ea36402aa21efdf403a60d34096 -- name: github.com/pelletier/go-buffruneio - version: c37440a7cf42ac63b919c752ca73a85067e05992 + version: 06020f85339e21b2478f756a78e295255ffa4d6a - name: github.com/pelletier/go-toml - version: 97253b98df84f9eef872866d079e74b8265150f1 + version: 4e9e0ee19b60b13eb79915933f44d8ed5f268bdd - name: github.com/pkg/errors - version: c605e284fe17294bda444b34710735b29d1a9d90 + version: 645ef00459ed84a119197bfb8d8205042c6df63d - name: github.com/spf13/afero - version: 9be650865eab0c12963d8753212f4f9c66cdcf12 + version: 8d919cbe7e2627e417f3e45c3c0e489a5b7e2536 subpackages: - mem - name: github.com/spf13/cast version: acbeb36b902d72a7a4c18e8f3241075e7ab763e4 - name: github.com/spf13/cobra - version: db6b9a8b3f3f400c8ecb4a4d7d02245b8facad66 + version: 7b2c5ac9fc04fc5efafb60700713d4fa609b777b - name: github.com/spf13/jwalterweatherman - version: fa7ca7e836cf3a8bb4ebf799f472c12d7e903d66 + version: 12bd96e66386c1960ab0f74ced1362f66f552f7b - name: github.com/spf13/pflag - version: 80fe0fb4eba54167e2ccae1c6c950e72abf61b73 + version: 4c012f6dcd9546820e378d0bdda4d8fc772cdfea - name: github.com/spf13/viper - version: 0967fc9aceab2ce9da34061253ac10fb99bba5b2 + version: 25b30aa063fc18e48662b86996252eabdcf2f0c7 - name: github.com/syndtr/goleveldb - version: 8c81ea47d4c41a385645e133e15510fc6a2a74b4 + version: adf24ef3f94bd13ec4163060b21a5678f22b429b subpackages: - leveldb - leveldb/cache @@ -82,7 +80,7 @@ imports: - leveldb/table - leveldb/util - name: github.com/tendermint/go-wire - version: b53add0b622662731985485f3a19be7f684660b8 + version: b6fc872b42d41158a60307db4da051dd6f179415 subpackages: - data - data/base58 @@ -91,22 +89,22 @@ imports: subpackages: - term - name: golang.org/x/crypto - version: 5a033cc77e57eca05bdb50522851d29e03569cbe + version: 94eea52f7b742c7cbe0b03b22f0c4c8631ece122 subpackages: - ripemd160 - name: golang.org/x/sys - version: 9ccfe848b9db8435a24c424abbc07a921adf1df5 + version: 8b4580aae2a0dd0c231a45d3ccb8434ff533b840 subpackages: - unix - name: golang.org/x/text - version: 470f45bf29f4147d6fbd7dfd0a02a848e49f5bf4 + version: 75cc3cad82b5f47d3fb229ddda8c5167da14f294 subpackages: - transform - unicode/norm - name: gopkg.in/go-playground/validator.v9 - version: d529ee1b0f30352444f507cc6cdac96bfd12decc + version: 61caf9d3038e1af346dbf5c2e16f6678e1548364 - name: gopkg.in/yaml.v2 - version: cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b + version: 287cf08546ab5e7e37d55a84f7ed3fd1db036de5 testImports: - name: github.com/davecgh/go-spew version: 6d212800a42e8ab5c146b8ace3490ee17e5225f9 diff --git a/merkle/kvpairs.go b/merkle/kvpairs.go deleted file mode 100644 index 3d67049f2..000000000 --- a/merkle/kvpairs.go +++ /dev/null @@ -1,48 +0,0 @@ -package merkle - -import ( - "sort" - - wire "github.com/tendermint/go-wire" - "golang.org/x/crypto/ripemd160" -) - -// NOTE: Behavior is undefined with dup keys. -type KVPair struct { - Key string - Value interface{} // Can be Hashable or not. -} - -func (kv KVPair) Hash() []byte { - hasher, n, err := ripemd160.New(), new(int), new(error) - wire.WriteString(kv.Key, hasher, n, err) - if kvH, ok := kv.Value.(Hashable); ok { - wire.WriteByteSlice(kvH.Hash(), hasher, n, err) - } else { - wire.WriteBinary(kv.Value, hasher, n, err) - } - if *err != nil { - panic(*err) - } - return hasher.Sum(nil) -} - -type KVPairs []KVPair - -func (kvps KVPairs) Len() int { return len(kvps) } -func (kvps KVPairs) Less(i, j int) bool { return kvps[i].Key < kvps[j].Key } -func (kvps KVPairs) Swap(i, j int) { kvps[i], kvps[j] = kvps[j], kvps[i] } -func (kvps KVPairs) Sort() { sort.Sort(kvps) } - -func MakeSortedKVPairs(m map[string]interface{}) []Hashable { - kvPairs := make([]KVPair, 0, len(m)) - for k, v := range m { - kvPairs = append(kvPairs, KVPair{k, v}) - } - KVPairs(kvPairs).Sort() - kvPairsH := make([]Hashable, 0, len(kvPairs)) - for _, kvp := range kvPairs { - kvPairsH = append(kvPairsH, kvp) - } - return kvPairsH -} diff --git a/merkle/simple_map.go b/merkle/simple_map.go index 43dce990f..003c7cd42 100644 --- a/merkle/simple_map.go +++ b/merkle/simple_map.go @@ -1,26 +1,86 @@ package merkle +import ( + "github.com/tendermint/go-wire" + cmn "github.com/tendermint/tmlibs/common" + "golang.org/x/crypto/ripemd160" +) + type SimpleMap struct { - kvz KVPairs + kvs cmn.KVPairs + sorted bool } func NewSimpleMap() *SimpleMap { return &SimpleMap{ - kvz: nil, + kvs: nil, + sorted: false, } } -func (sm *SimpleMap) Set(k string, o interface{}) { - sm.kvz = append(sm.kvz, KVPair{Key: k, Value: o}) +func (sm *SimpleMap) Set(key string, value interface{}) { + sm.sorted = false + + // Is value Hashable? + var vBytes []byte + if hashable, ok := value.(Hashable); ok { + vBytes = hashable.Hash() + } else { + vBytes = wire.BinaryBytes(value) + } + + sm.kvs = append(sm.kvs, cmn.KVPair{ + Key: []byte(key), + Value: vBytes, + }) } // Merkle root hash of items sorted by key. // NOTE: Behavior is undefined when key is duplicate. func (sm *SimpleMap) Hash() []byte { - sm.kvz.Sort() - kvPairsH := make([]Hashable, 0, len(sm.kvz)) - for _, kvp := range sm.kvz { - kvPairsH = append(kvPairsH, kvp) + sm.Sort() + return hashKVPairs(sm.kvs) +} + +func (sm *SimpleMap) Sort() { + if sm.sorted { + return + } + sm.kvs.Sort() + sm.sorted = true +} + +// Returns a copy of sorted KVPairs. +// CONTRACT: The returned slice must not be mutated. +func (sm *SimpleMap) KVPairs() cmn.KVPairs { + sm.Sort() + kvs := make(cmn.KVPairs, len(sm.kvs)) + copy(kvs, sm.kvs) + return kvs +} + +//---------------------------------------- + +// A local extension to KVPair that can be hashed. +type kvPair cmn.KVPair + +func (kv kvPair) Hash() []byte { + hasher, n, err := ripemd160.New(), new(int), new(error) + wire.WriteByteSlice(kv.Key, hasher, n, err) + if *err != nil { + panic(*err) + } + wire.WriteByteSlice(kv.Value, hasher, n, err) + if *err != nil { + panic(*err) + } + return hasher.Sum(nil) +} + +func hashKVPairs(kvs cmn.KVPairs) []byte { + kvsH := make([]Hashable, 0, len(kvs)) + for _, kvp := range kvs { + kvsH = append(kvsH, kvPair(kvp)) } - return SimpleHashFromHashables(kvPairsH) + return SimpleHashFromHashables(kvsH) } diff --git a/merkle/simple_map_test.go b/merkle/simple_map_test.go index 5eb218274..8ba7ce66b 100644 --- a/merkle/simple_map_test.go +++ b/merkle/simple_map_test.go @@ -11,37 +11,37 @@ func TestSimpleMap(t *testing.T) { { db := NewSimpleMap() db.Set("key1", "value1") - assert.Equal(t, "376bf717ebe3659a34f68edb833dfdcf4a2d3c10", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + assert.Equal(t, "3bb53f017d2f5b4f144692aa829a5c245ac2b123", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") } { db := NewSimpleMap() db.Set("key1", "value2") - assert.Equal(t, "72fd3a7224674377952214cb10ef21753ec803eb", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + assert.Equal(t, "14a68db29e3f930ffaafeff5e07c17a439384f39", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") } { db := NewSimpleMap() db.Set("key1", "value1") db.Set("key2", "value2") - assert.Equal(t, "23a160bd4eea5b2fcc0755d722f9112a15999abc", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + assert.Equal(t, "275c6367f4be335f9c482b6ef72e49c84e3f8bda", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") } { db := NewSimpleMap() db.Set("key2", "value2") // NOTE: out of order db.Set("key1", "value1") - assert.Equal(t, "23a160bd4eea5b2fcc0755d722f9112a15999abc", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + assert.Equal(t, "275c6367f4be335f9c482b6ef72e49c84e3f8bda", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") } { db := NewSimpleMap() db.Set("key1", "value1") db.Set("key2", "value2") db.Set("key3", "value3") - assert.Equal(t, "40df7416429148d03544cfafa86e1080615cd2bc", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + assert.Equal(t, "48d60701cb4c96916f68a958b3368205ebe3809b", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") } { db := NewSimpleMap() db.Set("key2", "value2") // NOTE: out of order db.Set("key1", "value1") db.Set("key3", "value3") - assert.Equal(t, "40df7416429148d03544cfafa86e1080615cd2bc", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + assert.Equal(t, "48d60701cb4c96916f68a958b3368205ebe3809b", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") } } diff --git a/merkle/simple_tree.go b/merkle/simple_tree.go index d64082b43..3a82f4edc 100644 --- a/merkle/simple_tree.go +++ b/merkle/simple_tree.go @@ -88,6 +88,9 @@ func SimpleHashFromHashables(items []Hashable) []byte { // Convenience for SimpleHashFromHashes. func SimpleHashFromMap(m map[string]interface{}) []byte { - kpPairsH := MakeSortedKVPairs(m) - return SimpleHashFromHashables(kpPairsH) + sm := NewSimpleMap() + for k, v := range m { + sm.Set(k, v) + } + return sm.Hash() }