From 066fe82a927aef6f7f6431af78f0d5156cb2cdb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Vad=C3=A9e?= Date: Tue, 6 Mar 2018 08:29:18 +0100 Subject: [PATCH 01/40] pubsub implements service.OnReset (#156) --- pubsub/pubsub.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pubsub/pubsub.go b/pubsub/pubsub.go index 54a4b8aed..28e008ca6 100644 --- a/pubsub/pubsub.go +++ b/pubsub/pubsub.go @@ -209,6 +209,11 @@ func (s *Server) OnStart() error { return nil } +// OnReset implements Service.OnReset +func (s *Server) OnReset() error { + return nil +} + func (s *Server) loop(state state) { loop: for cmd := range s.cmds { From b1cc688a61c53e39b92ceb5df370e3c94b19da4c Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Sun, 11 Mar 2018 22:46:31 -0700 Subject: [PATCH 02/40] encodeByteSlice uses uvarint for length instead of varint (#161) --- events/README.md | 2 +- events/events.go | 2 +- merkle/simple_map_test.go | 12 ++++++------ merkle/types.go | 8 ++++---- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/events/README.md b/events/README.md index 7a00d79dc..d7469515e 100644 --- a/events/README.md +++ b/events/README.md @@ -95,7 +95,7 @@ type EventCallback func(data EventData) type EventData interface { } ``` -Generic event data can be typed and registered with tendermint/go-wire +Generic event data can be typed and registered with tendermint/go-amino via concrete implementation of this interface diff --git a/events/events.go b/events/events.go index 12aa07813..3bc349306 100644 --- a/events/events.go +++ b/events/events.go @@ -9,7 +9,7 @@ import ( . "github.com/tendermint/tmlibs/common" ) -// Generic event data can be typed and registered with tendermint/go-wire +// Generic event data can be typed and registered with tendermint/go-amino // via concrete implementation of this interface type EventData interface { //AssertIsEventData() diff --git a/merkle/simple_map_test.go b/merkle/simple_map_test.go index 61210132b..c9c871354 100644 --- a/merkle/simple_map_test.go +++ b/merkle/simple_map_test.go @@ -17,37 +17,37 @@ func TestSimpleMap(t *testing.T) { { db := NewSimpleMap() db.Set("key1", strHasher("value1")) - assert.Equal(t, "19618304d1ad2635c4238bce87f72331b22a11a1", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + assert.Equal(t, "acdb4f121bc6f25041eb263ab463f1cd79236a32", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") } { db := NewSimpleMap() db.Set("key1", strHasher("value2")) - assert.Equal(t, "51cb96d3d41e1714def72eb4bacc211de9ddf284", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + assert.Equal(t, "b8cbf5adee8c524e14f531da9b49adbbbd66fffa", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") } { db := NewSimpleMap() db.Set("key1", strHasher("value1")) db.Set("key2", strHasher("value2")) - assert.Equal(t, "58a0a99d5019fdcad4bcf55942e833b2dfab9421", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + assert.Equal(t, "1708aabc85bbe00242d3db8c299516aa54e48c38", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") } { db := NewSimpleMap() db.Set("key2", strHasher("value2")) // NOTE: out of order db.Set("key1", strHasher("value1")) - assert.Equal(t, "58a0a99d5019fdcad4bcf55942e833b2dfab9421", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + assert.Equal(t, "1708aabc85bbe00242d3db8c299516aa54e48c38", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") } { db := NewSimpleMap() db.Set("key1", strHasher("value1")) db.Set("key2", strHasher("value2")) db.Set("key3", strHasher("value3")) - assert.Equal(t, "cb56db3c7993e977f4c2789559ae3e5e468a6e9b", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + assert.Equal(t, "e728afe72ce351eed6aca65c5f78da19b9a6e214", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") } { db := NewSimpleMap() db.Set("key2", strHasher("value2")) // NOTE: out of order db.Set("key1", strHasher("value1")) db.Set("key3", strHasher("value3")) - assert.Equal(t, "cb56db3c7993e977f4c2789559ae3e5e468a6e9b", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + assert.Equal(t, "e728afe72ce351eed6aca65c5f78da19b9a6e214", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") } } diff --git a/merkle/types.go b/merkle/types.go index e0fe35fa8..a0c491a7e 100644 --- a/merkle/types.go +++ b/merkle/types.go @@ -28,10 +28,10 @@ type Hasher interface { } //----------------------------------------------------------------------- -// NOTE: these are duplicated from go-wire so we dont need go-wire as a dep +// NOTE: these are duplicated from go-amino so we dont need go-amino as a dep func encodeByteSlice(w io.Writer, bz []byte) (err error) { - err = encodeVarint(w, int64(len(bz))) + err = encodeUvarint(w, uint64(len(bz))) if err != nil { return } @@ -39,9 +39,9 @@ func encodeByteSlice(w io.Writer, bz []byte) (err error) { return } -func encodeVarint(w io.Writer, i int64) (err error) { +func encodeUvarint(w io.Writer, i uint64) (err error) { var buf [10]byte - n := binary.PutVarint(buf[:], i) + n := binary.PutUvarint(buf[:], i) _, err = w.Write(buf[0:n]) return } From d289c9286e816a37336289a75752752a751bc918 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Thu, 15 Mar 2018 09:43:23 -0700 Subject: [PATCH 03/40] Implement NewPrefixDB (#164) * encodeByteSlice uses uvarint for length instead of varint * Implemented NewPrefixDB * Fix flowrate test (#165) * Complete implementation and fix tests * Add tests for MemBatch Write[Sync] --- common/types.pb.go | 3 +- db/c_level_db.go | 8 ++ db/common_test.go | 182 +++++++++++++++++------------- db/db_test.go | 190 +++++++++++++++++++++++++++++++ db/go_level_db.go | 18 ++- db/mem_batch.go | 22 +++- db/mem_db.go | 22 ++-- db/prefix_db.go | 263 +++++++++++++++++++++++++++++++++++++++++++ db/prefix_db_test.go | 44 ++++++++ db/types.go | 4 +- db/util.go | 24 ++-- flowrate/io_test.go | 10 +- 12 files changed, 683 insertions(+), 107 deletions(-) create mode 100644 db/db_test.go create mode 100644 db/prefix_db.go create mode 100644 db/prefix_db_test.go diff --git a/common/types.pb.go b/common/types.pb.go index c301d28c0..047b7aee2 100644 --- a/common/types.pb.go +++ b/common/types.pb.go @@ -1,6 +1,5 @@ -// Code generated by protoc-gen-gogo. +// Code generated by protoc-gen-gogo. DO NOT EDIT. // source: common/types.proto -// DO NOT EDIT! /* Package common is a generated protocol buffer package. diff --git a/db/c_level_db.go b/db/c_level_db.go index a59137883..e3e6c1d5d 100644 --- a/db/c_level_db.go +++ b/db/c_level_db.go @@ -171,6 +171,14 @@ func (mBatch *cLevelDBBatch) Write() { } } +// Implements Batch. +func (mBatch *cLevelDBBatch) WriteSync() { + err := mBatch.db.db.Write(mBatch.db.woSync, mBatch.batch) + if err != nil { + panic(err) + } +} + //---------------------------------------- // Iterator // NOTE This is almost identical to db/go_level_db.Iterator diff --git a/db/common_test.go b/db/common_test.go index 1b0f00416..5afec28b3 100644 --- a/db/common_test.go +++ b/db/common_test.go @@ -2,6 +2,7 @@ package db import ( "fmt" + "sync" "testing" "github.com/stretchr/testify/assert" @@ -9,6 +10,14 @@ import ( cmn "github.com/tendermint/tmlibs/common" ) +//---------------------------------------- +// Helper functions. + +func checkValue(t *testing.T, db DB, key []byte, valueWanted []byte) { + valueGot := db.Get(key) + assert.Equal(t, valueWanted, valueGot) +} + func checkValid(t *testing.T, itr Iterator, expected bool) { valid := itr.Valid() require.Equal(t, expected, valid) @@ -46,110 +55,131 @@ func checkValuePanics(t *testing.T, itr Iterator) { } func newTempDB(t *testing.T, backend DBBackendType) (db DB) { - dir, dirname := cmn.Tempdir("test_go_iterator") + dir, dirname := cmn.Tempdir("db_common_test") db = NewDB("testdb", backend, dirname) dir.Close() return db } -func TestDBIteratorSingleKey(t *testing.T) { - for backend, _ := range backends { - t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { - db := newTempDB(t, backend) - db.SetSync(bz("1"), bz("value_1")) - itr := db.Iterator(nil, nil) +//---------------------------------------- +// mockDB - checkValid(t, itr, true) - checkNext(t, itr, false) - checkValid(t, itr, false) - checkNextPanics(t, itr) +// NOTE: not actually goroutine safe. +// If you want something goroutine safe, maybe you just want a MemDB. +type mockDB struct { + mtx sync.Mutex + calls map[string]int +} - // Once invalid... - checkInvalid(t, itr) - }) +func newMockDB() *mockDB { + return &mockDB{ + calls: make(map[string]int), } } -func TestDBIteratorTwoKeys(t *testing.T) { - for backend, _ := range backends { - t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { - db := newTempDB(t, backend) - db.SetSync(bz("1"), bz("value_1")) - db.SetSync(bz("2"), bz("value_1")) +func (mdb *mockDB) Mutex() *sync.Mutex { + return &(mdb.mtx) +} - { // Fail by calling Next too much - itr := db.Iterator(nil, nil) - checkValid(t, itr, true) +func (mdb *mockDB) Get([]byte) []byte { + mdb.calls["Get"] += 1 + return nil +} - checkNext(t, itr, true) - checkValid(t, itr, true) +func (mdb *mockDB) Has([]byte) bool { + mdb.calls["Has"] += 1 + return false +} - checkNext(t, itr, false) - checkValid(t, itr, false) +func (mdb *mockDB) Set([]byte, []byte) { + mdb.calls["Set"] += 1 +} - checkNextPanics(t, itr) +func (mdb *mockDB) SetSync([]byte, []byte) { + mdb.calls["SetSync"] += 1 +} - // Once invalid... - checkInvalid(t, itr) - } - }) - } +func (mdb *mockDB) SetNoLock([]byte, []byte) { + mdb.calls["SetNoLock"] += 1 } -func TestDBIteratorMany(t *testing.T) { - for backend, _ := range backends { - t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { - db := newTempDB(t, backend) +func (mdb *mockDB) SetNoLockSync([]byte, []byte) { + mdb.calls["SetNoLockSync"] += 1 +} - keys := make([][]byte, 100) - for i := 0; i < 100; i++ { - keys[i] = []byte{byte(i)} - } +func (mdb *mockDB) Delete([]byte, []byte) { + mdb.calls["Delete"] += 1 +} - value := []byte{5} - for _, k := range keys { - db.Set(k, value) - } +func (mdb *mockDB) DeleteSync([]byte, []byte) { + mdb.calls["DeleteSync"] += 1 +} - itr := db.Iterator(nil, nil) - defer itr.Close() - for ; itr.Valid(); itr.Next() { - assert.Equal(t, db.Get(itr.Key()), itr.Value()) - } - }) - } +func (mdb *mockDB) DeleteNoLock([]byte) { + mdb.calls["DeleteNoLock"] += 1 } -func TestDBIteratorEmpty(t *testing.T) { - for backend, _ := range backends { - t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { - db := newTempDB(t, backend) - itr := db.Iterator(nil, nil) +func (mdb *mockDB) DeleteNoLockSync([]byte) { + mdb.calls["DeleteNoLockSync"] += 1 +} - checkInvalid(t, itr) - }) - } +func (mdb *mockDB) Iterator(start, end []byte) Iterator { + mdb.calls["Iterator"] += 1 + return &mockIterator{} } -func TestDBIteratorEmptyBeginAfter(t *testing.T) { - for backend, _ := range backends { - t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { - db := newTempDB(t, backend) - itr := db.Iterator(bz("1"), nil) +func (mdb *mockDB) ReverseIterator(start, end []byte) Iterator { + mdb.calls["ReverseIterator"] += 1 + return &mockIterator{} +} - checkInvalid(t, itr) - }) - } +func (mdb *mockDB) Close() { + mdb.calls["Close"] += 1 } -func TestDBIteratorNonemptyBeginAfter(t *testing.T) { - for backend, _ := range backends { - t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { - db := newTempDB(t, backend) - db.SetSync(bz("1"), bz("value_1")) - itr := db.Iterator(bz("2"), nil) +func (mdb *mockDB) NewBatch() Batch { + mdb.calls["NewBatch"] += 1 + return &memBatch{db: mdb} +} + +func (mdb *mockDB) Print() { + mdb.calls["Print"] += 1 + fmt.Sprintf("mockDB{%v}", mdb.Stats()) +} - checkInvalid(t, itr) - }) +func (mdb *mockDB) Stats() map[string]string { + mdb.calls["Stats"] += 1 + + res := make(map[string]string) + for key, count := range mdb.calls { + res[key] = fmt.Sprintf("%d", count) } + return res +} + +//---------------------------------------- +// mockIterator + +type mockIterator struct{} + +func (_ mockIterator) Domain() (start []byte, end []byte) { + return nil, nil +} + +func (_ mockIterator) Valid() bool { + return false +} + +func (_ mockIterator) Next() { +} + +func (_ mockIterator) Key() []byte { + return nil +} + +func (_ mockIterator) Value() []byte { + return nil +} + +func (_ mockIterator) Close() { } diff --git a/db/db_test.go b/db/db_test.go new file mode 100644 index 000000000..8884cea2d --- /dev/null +++ b/db/db_test.go @@ -0,0 +1,190 @@ +package db + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDBIteratorSingleKey(t *testing.T) { + for backend, _ := range backends { + t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { + db := newTempDB(t, backend) + db.SetSync(bz("1"), bz("value_1")) + itr := db.Iterator(nil, nil) + + checkValid(t, itr, true) + checkNext(t, itr, false) + checkValid(t, itr, false) + checkNextPanics(t, itr) + + // Once invalid... + checkInvalid(t, itr) + }) + } +} + +func TestDBIteratorTwoKeys(t *testing.T) { + for backend, _ := range backends { + t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { + db := newTempDB(t, backend) + db.SetSync(bz("1"), bz("value_1")) + db.SetSync(bz("2"), bz("value_1")) + + { // Fail by calling Next too much + itr := db.Iterator(nil, nil) + checkValid(t, itr, true) + + checkNext(t, itr, true) + checkValid(t, itr, true) + + checkNext(t, itr, false) + checkValid(t, itr, false) + + checkNextPanics(t, itr) + + // Once invalid... + checkInvalid(t, itr) + } + }) + } +} + +func TestDBIteratorMany(t *testing.T) { + for backend, _ := range backends { + t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { + db := newTempDB(t, backend) + + keys := make([][]byte, 100) + for i := 0; i < 100; i++ { + keys[i] = []byte{byte(i)} + } + + value := []byte{5} + for _, k := range keys { + db.Set(k, value) + } + + itr := db.Iterator(nil, nil) + defer itr.Close() + for ; itr.Valid(); itr.Next() { + assert.Equal(t, db.Get(itr.Key()), itr.Value()) + } + }) + } +} + +func TestDBIteratorEmpty(t *testing.T) { + for backend, _ := range backends { + t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { + db := newTempDB(t, backend) + itr := db.Iterator(nil, nil) + + checkInvalid(t, itr) + }) + } +} + +func TestDBIteratorEmptyBeginAfter(t *testing.T) { + for backend, _ := range backends { + t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { + db := newTempDB(t, backend) + itr := db.Iterator(bz("1"), nil) + + checkInvalid(t, itr) + }) + } +} + +func TestDBIteratorNonemptyBeginAfter(t *testing.T) { + for backend, _ := range backends { + t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { + db := newTempDB(t, backend) + db.SetSync(bz("1"), bz("value_1")) + itr := db.Iterator(bz("2"), nil) + + checkInvalid(t, itr) + }) + } +} + +func TestDBBatchWrite1(t *testing.T) { + mdb := newMockDB() + batch := mdb.NewBatch() + + batch.Set(bz("1"), bz("1")) + batch.Set(bz("2"), bz("2")) + batch.Delete(bz("3")) + batch.Set(bz("4"), bz("4")) + batch.Write() + + assert.Equal(t, 0, mdb.calls["Set"]) + assert.Equal(t, 0, mdb.calls["SetSync"]) + assert.Equal(t, 3, mdb.calls["SetNoLock"]) + assert.Equal(t, 0, mdb.calls["SetNoLockSync"]) + assert.Equal(t, 0, mdb.calls["Delete"]) + assert.Equal(t, 0, mdb.calls["DeleteSync"]) + assert.Equal(t, 1, mdb.calls["DeleteNoLock"]) + assert.Equal(t, 0, mdb.calls["DeleteNoLockSync"]) +} + +func TestDBBatchWrite2(t *testing.T) { + mdb := newMockDB() + batch := mdb.NewBatch() + + batch.Set(bz("1"), bz("1")) + batch.Set(bz("2"), bz("2")) + batch.Set(bz("4"), bz("4")) + batch.Delete(bz("3")) + batch.Write() + + assert.Equal(t, 0, mdb.calls["Set"]) + assert.Equal(t, 0, mdb.calls["SetSync"]) + assert.Equal(t, 3, mdb.calls["SetNoLock"]) + assert.Equal(t, 0, mdb.calls["SetNoLockSync"]) + assert.Equal(t, 0, mdb.calls["Delete"]) + assert.Equal(t, 0, mdb.calls["DeleteSync"]) + assert.Equal(t, 1, mdb.calls["DeleteNoLock"]) + assert.Equal(t, 0, mdb.calls["DeleteNoLockSync"]) +} + +func TestDBBatchWriteSync1(t *testing.T) { + mdb := newMockDB() + batch := mdb.NewBatch() + + batch.Set(bz("1"), bz("1")) + batch.Set(bz("2"), bz("2")) + batch.Delete(bz("3")) + batch.Set(bz("4"), bz("4")) + batch.WriteSync() + + assert.Equal(t, 0, mdb.calls["Set"]) + assert.Equal(t, 0, mdb.calls["SetSync"]) + assert.Equal(t, 2, mdb.calls["SetNoLock"]) + assert.Equal(t, 1, mdb.calls["SetNoLockSync"]) + assert.Equal(t, 0, mdb.calls["Delete"]) + assert.Equal(t, 0, mdb.calls["DeleteSync"]) + assert.Equal(t, 1, mdb.calls["DeleteNoLock"]) + assert.Equal(t, 0, mdb.calls["DeleteNoLockSync"]) +} + +func TestDBBatchWriteSync2(t *testing.T) { + mdb := newMockDB() + batch := mdb.NewBatch() + + batch.Set(bz("1"), bz("1")) + batch.Set(bz("2"), bz("2")) + batch.Set(bz("4"), bz("4")) + batch.Delete(bz("3")) + batch.WriteSync() + + assert.Equal(t, 0, mdb.calls["Set"]) + assert.Equal(t, 0, mdb.calls["SetSync"]) + assert.Equal(t, 3, mdb.calls["SetNoLock"]) + assert.Equal(t, 0, mdb.calls["SetNoLockSync"]) + assert.Equal(t, 0, mdb.calls["Delete"]) + assert.Equal(t, 0, mdb.calls["DeleteSync"]) + assert.Equal(t, 0, mdb.calls["DeleteNoLock"]) + assert.Equal(t, 1, mdb.calls["DeleteNoLockSync"]) +} diff --git a/db/go_level_db.go b/db/go_level_db.go index 9fed329bf..55ca36c39 100644 --- a/db/go_level_db.go +++ b/db/go_level_db.go @@ -110,10 +110,10 @@ func (db *GoLevelDB) Print() { str, _ := db.db.GetProperty("leveldb.stats") fmt.Printf("%v\n", str) - iter := db.db.NewIterator(nil, nil) - for iter.Next() { - key := iter.Key() - value := iter.Value() + itr := db.db.NewIterator(nil, nil) + for itr.Next() { + key := itr.Key() + value := itr.Value() fmt.Printf("[%X]:\t[%X]\n", key, value) } } @@ -167,7 +167,15 @@ func (mBatch *goLevelDBBatch) Delete(key []byte) { // Implements Batch. func (mBatch *goLevelDBBatch) Write() { - err := mBatch.db.db.Write(mBatch.batch, nil) + err := mBatch.db.db.Write(mBatch.batch, &opt.WriteOptions{Sync: false}) + if err != nil { + panic(err) + } +} + +// Implements Batch. +func (mBatch *goLevelDBBatch) WriteSync() { + err := mBatch.db.db.Write(mBatch.batch, &opt.WriteOptions{Sync: true}) if err != nil { panic(err) } diff --git a/db/mem_batch.go b/db/mem_batch.go index 7072d931a..756798ded 100644 --- a/db/mem_batch.go +++ b/db/mem_batch.go @@ -5,7 +5,9 @@ import "sync" type atomicSetDeleter interface { Mutex() *sync.Mutex SetNoLock(key, value []byte) + SetNoLockSync(key, value []byte) DeleteNoLock(key []byte) + DeleteNoLockSync(key []byte) } type memBatch struct { @@ -35,16 +37,34 @@ func (mBatch *memBatch) Delete(key []byte) { } func (mBatch *memBatch) Write() { + mBatch.write(false) +} + +func (mBatch *memBatch) WriteSync() { + mBatch.write(true) +} + +func (mBatch *memBatch) write(doSync bool) { mtx := mBatch.db.Mutex() mtx.Lock() defer mtx.Unlock() - for _, op := range mBatch.ops { + for i, op := range mBatch.ops { + if doSync && i == (len(mBatch.ops)-1) { + switch op.opType { + case opTypeSet: + mBatch.db.SetNoLockSync(op.key, op.value) + case opTypeDelete: + mBatch.db.DeleteNoLockSync(op.key) + } + break // we're done. + } switch op.opType { case opTypeSet: mBatch.db.SetNoLock(op.key, op.value) case opTypeDelete: mBatch.db.DeleteNoLock(op.key) } + } } diff --git a/db/mem_db.go b/db/mem_db.go index f2c484fa7..5439d6789 100644 --- a/db/mem_db.go +++ b/db/mem_db.go @@ -26,6 +26,11 @@ func NewMemDB() *MemDB { return database } +// Implements atomicSetDeleter. +func (db *MemDB) Mutex() *sync.Mutex { + return &(db.mtx) +} + // Implements DB. func (db *MemDB) Get(key []byte) []byte { db.mtx.Lock() @@ -63,6 +68,11 @@ func (db *MemDB) SetSync(key []byte, value []byte) { // Implements atomicSetDeleter. func (db *MemDB) SetNoLock(key []byte, value []byte) { + db.SetNoLockSync(key, value) +} + +// Implements atomicSetDeleter. +func (db *MemDB) SetNoLockSync(key []byte, value []byte) { key = nonNilBytes(key) value = nonNilBytes(value) @@ -87,6 +97,11 @@ func (db *MemDB) DeleteSync(key []byte) { // Implements atomicSetDeleter. func (db *MemDB) DeleteNoLock(key []byte) { + db.DeleteNoLockSync(key) +} + +// Implements atomicSetDeleter. +func (db *MemDB) DeleteNoLockSync(key []byte) { key = nonNilBytes(key) delete(db.db, string(key)) @@ -122,9 +137,6 @@ func (db *MemDB) Stats() map[string]string { return stats } -//---------------------------------------- -// Batch - // Implements DB. func (db *MemDB) NewBatch() Batch { db.mtx.Lock() @@ -133,10 +145,6 @@ func (db *MemDB) NewBatch() Batch { return &memBatch{db, nil} } -func (db *MemDB) Mutex() *sync.Mutex { - return &(db.mtx) -} - //---------------------------------------- // Iterator diff --git a/db/prefix_db.go b/db/prefix_db.go new file mode 100644 index 000000000..5947e7fce --- /dev/null +++ b/db/prefix_db.go @@ -0,0 +1,263 @@ +package db + +import ( + "bytes" + "fmt" + "sync" +) + +// IteratePrefix is a convenience function for iterating over a key domain +// restricted by prefix. +func IteratePrefix(db DB, prefix []byte) Iterator { + var start, end []byte + if len(prefix) == 0 { + start = nil + end = nil + } else { + start = cp(prefix) + end = cpIncr(prefix) + } + return db.Iterator(start, end) +} + +/* +TODO: Make test, maybe rename. +// Like IteratePrefix but the iterator strips the prefix from the keys. +func IteratePrefixStripped(db DB, prefix []byte) Iterator { + return newUnprefixIterator(prefix, IteratePrefix(db, prefix)) +} +*/ + +//---------------------------------------- +// prefixDB + +type prefixDB struct { + mtx sync.Mutex + prefix []byte + db DB +} + +// NewPrefixDB lets you namespace multiple DBs within a single DB. +func NewPrefixDB(db DB, prefix []byte) *prefixDB { + return &prefixDB{ + prefix: prefix, + db: db, + } +} + +// Implements atomicSetDeleter. +func (pdb *prefixDB) Mutex() *sync.Mutex { + return &(pdb.mtx) +} + +// Implements DB. +func (pdb *prefixDB) Get(key []byte) []byte { + pdb.mtx.Lock() + defer pdb.mtx.Unlock() + + return pdb.db.Get(pdb.prefixed(key)) +} + +// Implements DB. +func (pdb *prefixDB) Has(key []byte) bool { + pdb.mtx.Lock() + defer pdb.mtx.Unlock() + + return pdb.db.Has(pdb.prefixed(key)) +} + +// Implements DB. +func (pdb *prefixDB) Set(key []byte, value []byte) { + pdb.mtx.Lock() + defer pdb.mtx.Unlock() + + pdb.db.Set(pdb.prefixed(key), value) +} + +// Implements DB. +func (pdb *prefixDB) SetSync(key []byte, value []byte) { + pdb.mtx.Lock() + defer pdb.mtx.Unlock() + + pdb.db.SetSync(pdb.prefixed(key), value) +} + +// Implements atomicSetDeleter. +func (pdb *prefixDB) SetNoLock(key []byte, value []byte) { + pdb.db.Set(pdb.prefixed(key), value) +} + +// Implements atomicSetDeleter. +func (pdb *prefixDB) SetNoLockSync(key []byte, value []byte) { + pdb.db.SetSync(pdb.prefixed(key), value) +} + +// Implements DB. +func (pdb *prefixDB) Delete(key []byte) { + pdb.mtx.Lock() + defer pdb.mtx.Unlock() + + pdb.db.Delete(pdb.prefixed(key)) +} + +// Implements DB. +func (pdb *prefixDB) DeleteSync(key []byte) { + pdb.mtx.Lock() + defer pdb.mtx.Unlock() + + pdb.db.DeleteSync(pdb.prefixed(key)) +} + +// Implements atomicSetDeleter. +func (pdb *prefixDB) DeleteNoLock(key []byte) { + pdb.db.Delete(pdb.prefixed(key)) +} + +// Implements atomicSetDeleter. +func (pdb *prefixDB) DeleteNoLockSync(key []byte) { + pdb.db.DeleteSync(pdb.prefixed(key)) +} + +// Implements DB. +func (pdb *prefixDB) Iterator(start, end []byte) Iterator { + pdb.mtx.Lock() + defer pdb.mtx.Unlock() + + pstart := append([]byte(pdb.prefix), start...) + pend := []byte(nil) + if end != nil { + pend = append([]byte(pdb.prefix), end...) + } + return newUnprefixIterator( + pdb.prefix, + pdb.db.Iterator( + pstart, + pend, + ), + ) +} + +// Implements DB. +func (pdb *prefixDB) ReverseIterator(start, end []byte) Iterator { + pdb.mtx.Lock() + defer pdb.mtx.Unlock() + + pstart := []byte(nil) + if start != nil { + pstart = append([]byte(pdb.prefix), start...) + } + pend := []byte(nil) + if end != nil { + pend = append([]byte(pdb.prefix), end...) + } + return newUnprefixIterator( + pdb.prefix, + pdb.db.ReverseIterator( + pstart, + pend, + ), + ) +} + +// Implements DB. +func (pdb *prefixDB) NewBatch() Batch { + pdb.mtx.Lock() + defer pdb.mtx.Unlock() + + return &memBatch{pdb, nil} +} + +// Implements DB. +func (pdb *prefixDB) Close() { + pdb.mtx.Lock() + defer pdb.mtx.Unlock() + + pdb.db.Close() +} + +// Implements DB. +func (pdb *prefixDB) Print() { + fmt.Printf("prefix: %X\n", pdb.prefix) + + itr := pdb.Iterator(nil, nil) + defer itr.Close() + for ; itr.Valid(); itr.Next() { + key := itr.Key() + value := itr.Value() + fmt.Printf("[%X]:\t[%X]\n", key, value) + } +} + +// Implements DB. +func (pdb *prefixDB) Stats() map[string]string { + stats := make(map[string]string) + stats["prefixdb.prefix.string"] = string(pdb.prefix) + stats["prefixdb.prefix.hex"] = fmt.Sprintf("%X", pdb.prefix) + source := pdb.db.Stats() + for key, value := range source { + stats["prefixdb.source."+key] = value + } + return stats +} + +func (pdb *prefixDB) prefixed(key []byte) []byte { + return append([]byte(pdb.prefix), key...) +} + +//---------------------------------------- + +// Strips prefix while iterating from Iterator. +type unprefixIterator struct { + prefix []byte + source Iterator +} + +func newUnprefixIterator(prefix []byte, source Iterator) unprefixIterator { + return unprefixIterator{ + prefix: prefix, + source: source, + } +} + +func (itr unprefixIterator) Domain() (start []byte, end []byte) { + start, end = itr.source.Domain() + if len(start) > 0 { + start = stripPrefix(start, itr.prefix) + } + if len(end) > 0 { + end = stripPrefix(end, itr.prefix) + } + return +} + +func (itr unprefixIterator) Valid() bool { + return itr.source.Valid() +} + +func (itr unprefixIterator) Next() { + itr.source.Next() +} + +func (itr unprefixIterator) Key() (key []byte) { + return stripPrefix(itr.source.Key(), itr.prefix) +} + +func (itr unprefixIterator) Value() (value []byte) { + return itr.source.Value() +} + +func (itr unprefixIterator) Close() { + itr.source.Close() +} + +//---------------------------------------- + +func stripPrefix(key []byte, prefix []byte) (stripped []byte) { + if len(key) < len(prefix) { + panic("should not happen") + } + if !bytes.Equal(key[:len(prefix)], prefix) { + panic("should not happne") + } + return key[len(prefix):] +} diff --git a/db/prefix_db_test.go b/db/prefix_db_test.go new file mode 100644 index 000000000..fd44a7ec8 --- /dev/null +++ b/db/prefix_db_test.go @@ -0,0 +1,44 @@ +package db + +import "testing" + +func TestIteratePrefix(t *testing.T) { + db := NewMemDB() + // Under "key" prefix + db.Set(bz("key"), bz("value")) + db.Set(bz("key1"), bz("value1")) + db.Set(bz("key2"), bz("value2")) + db.Set(bz("key3"), bz("value3")) + db.Set(bz("something"), bz("else")) + db.Set(bz(""), bz("")) + db.Set(bz("k"), bz("val")) + db.Set(bz("ke"), bz("valu")) + db.Set(bz("kee"), bz("valuu")) + xitr := db.Iterator(nil, nil) + xitr.Key() + + pdb := NewPrefixDB(db, bz("key")) + checkValue(t, pdb, bz("key"), nil) + checkValue(t, pdb, bz(""), bz("value")) + checkValue(t, pdb, bz("key1"), nil) + checkValue(t, pdb, bz("1"), bz("value1")) + checkValue(t, pdb, bz("key2"), nil) + checkValue(t, pdb, bz("2"), bz("value2")) + checkValue(t, pdb, bz("key3"), nil) + checkValue(t, pdb, bz("3"), bz("value3")) + checkValue(t, pdb, bz("something"), nil) + checkValue(t, pdb, bz("k"), nil) + checkValue(t, pdb, bz("ke"), nil) + checkValue(t, pdb, bz("kee"), nil) + + itr := pdb.Iterator(nil, nil) + itr.Key() + checkItem(t, itr, bz(""), bz("value")) + checkNext(t, itr, true) + checkItem(t, itr, bz("1"), bz("value1")) + checkNext(t, itr, true) + checkItem(t, itr, bz("2"), bz("value2")) + checkNext(t, itr, true) + checkItem(t, itr, bz("3"), bz("value3")) + itr.Close() +} diff --git a/db/types.go b/db/types.go index 07858087a..45146942e 100644 --- a/db/types.go +++ b/db/types.go @@ -1,5 +1,6 @@ package db +// DBs are goroutine safe. type DB interface { // Get returns nil iff key doesn't exist. @@ -35,7 +36,7 @@ type DB interface { // Iterate over a domain of keys in descending order. End is exclusive. // Start must be greater than end, or the Iterator is invalid. // If start is nil, iterates from the last/greatest item (inclusive). - // If end is nil, iterates up to the first/least item (iclusive). + // If end is nil, iterates up to the first/least item (inclusive). // CONTRACT: No writes may happen within a domain while an iterator exists over it. // CONTRACT: start, end readonly []byte ReverseIterator(start, end []byte) Iterator @@ -59,6 +60,7 @@ type DB interface { type Batch interface { SetDeleter Write() + WriteSync() } type SetDeleter interface { diff --git a/db/util.go b/db/util.go index b0ab7f6ad..ecb392dd6 100644 --- a/db/util.go +++ b/db/util.go @@ -4,28 +4,20 @@ import ( "bytes" ) -func IteratePrefix(db DB, prefix []byte) Iterator { - var start, end []byte - if len(prefix) == 0 { - start = nil - end = nil - } else { - start = cp(prefix) - end = cpIncr(prefix) - } - return db.Iterator(start, end) -} - -//---------------------------------------- - func cp(bz []byte) (ret []byte) { ret = make([]byte, len(bz)) copy(ret, bz) return ret } +// Returns a slice of the same length (big endian) +// except incremented by one. +// Returns nil on overflow (e.g. if bz bytes are all 0xFF) // CONTRACT: len(bz) > 0 func cpIncr(bz []byte) (ret []byte) { + if len(bz) == 0 { + panic("cpIncr expects non-zero bz length") + } ret = cp(bz) for i := len(bz) - 1; i >= 0; i-- { if ret[i] < byte(0xFF) { @@ -33,6 +25,10 @@ func cpIncr(bz []byte) (ret []byte) { return } else { ret[i] = byte(0x00) + if i == 0 { + // Overflow + return nil + } } } return nil diff --git a/flowrate/io_test.go b/flowrate/io_test.go index db40337c9..c84029d5e 100644 --- a/flowrate/io_test.go +++ b/flowrate/io_test.go @@ -121,7 +121,15 @@ func TestWriter(t *testing.T) { w.SetBlocking(true) if n, err := w.Write(b[20:]); n != 80 || err != nil { t.Fatalf("w.Write(b[20:]) expected 80 (); got %v (%v)", n, err) - } else if rt := time.Since(start); rt < _400ms { + } else if rt := time.Since(start); rt < _300ms { + // Explanation for `rt < _300ms` (as opposed to `< _400ms`) + // + // |<-- start | | + // epochs: -----0ms|---100ms|---200ms|---300ms|---400ms + // sends: 20|20 |20 |20 |20# + // + // NOTE: The '#' symbol can thus happen before 400ms is up. + // Thus, we can only panic if rt < _300ms. t.Fatalf("w.Write(b[20:]) returned ahead of time (%v)", rt) } From 536c27de8eefbf32b1738e0685401a7944dd76c1 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Sat, 17 Mar 2018 04:28:53 -0700 Subject: [PATCH 04/40] common/random.go supports seeding and *Rand (#121) * common/random.go supports seeding and *Rand * Ensure determinism --- common/random.go | 236 ++++++++++++++++++++++++++++++------------ common/random_test.go | 47 +++++---- 2 files changed, 194 insertions(+), 89 deletions(-) diff --git a/common/random.go b/common/random.go index ca71b6143..b945a88eb 100644 --- a/common/random.go +++ b/common/random.go @@ -13,34 +13,138 @@ const ( // pseudo random number generator. // seeded with OS randomness (crand) -var prng struct { + +type Rand struct { sync.Mutex *mrand.Rand } -func reset() { - b := cRandBytes(8) +var grand *Rand + +func init() { + grand = New() + grand.init() +} + +func New() *Rand { + rand := &Rand{} + rand.init() + return rand +} + +func (r *Rand) init() { + bz := cRandBytes(8) var seed uint64 for i := 0; i < 8; i++ { - seed |= uint64(b[i]) + seed |= uint64(bz[i]) seed <<= 8 } - prng.Lock() - prng.Rand = mrand.New(mrand.NewSource(int64(seed))) - prng.Unlock() + r.reset(int64(seed)) } -func init() { - reset() +func (r *Rand) reset(seed int64) { + r.Rand = mrand.New(mrand.NewSource(seed)) +} + +//---------------------------------------- +// Global functions + +func Seed(seed int64) { + grand.Seed(seed) +} + +func RandStr(length int) string { + return grand.RandStr(length) +} + +func RandUint16() uint16 { + return grand.RandUint16() +} + +func RandUint32() uint32 { + return grand.RandUint32() +} + +func RandUint64() uint64 { + return grand.RandUint64() +} + +func RandUint() uint { + return grand.RandUint() +} + +func RandInt16() int16 { + return grand.RandInt16() +} + +func RandInt32() int32 { + return grand.RandInt32() +} + +func RandInt64() int64 { + return grand.RandInt64() +} + +func RandInt() int { + return grand.RandInt() +} + +func RandInt31() int32 { + return grand.RandInt31() +} + +func RandInt63() int64 { + return grand.RandInt63() +} + +func RandUint16Exp() uint16 { + return grand.RandUint16Exp() +} + +func RandUint32Exp() uint32 { + return grand.RandUint32Exp() +} + +func RandUint64Exp() uint64 { + return grand.RandUint64Exp() +} + +func RandFloat32() float32 { + return grand.RandFloat32() +} + +func RandTime() time.Time { + return grand.RandTime() +} + +func RandBytes(n int) []byte { + return grand.RandBytes(n) +} + +func RandIntn(n int) int { + return grand.RandIntn(n) +} + +func RandPerm(n int) []int { + return grand.RandPerm(n) +} + +//---------------------------------------- +// Rand methods + +func (r *Rand) Seed(seed int64) { + r.Lock() + r.reset(seed) + r.Unlock() } // Constructs an alphanumeric string of given length. // It is not safe for cryptographic usage. -func RandStr(length int) string { +func (r *Rand) RandStr(length int) string { chars := []byte{} MAIN_LOOP: for { - val := RandInt63() + val := r.RandInt63() for i := 0; i < 10; i++ { v := int(val & 0x3f) // rightmost 6 bits if v >= 62 { // only 62 characters in strChars @@ -60,127 +164,127 @@ MAIN_LOOP: } // It is not safe for cryptographic usage. -func RandUint16() uint16 { - return uint16(RandUint32() & (1<<16 - 1)) +func (r *Rand) RandUint16() uint16 { + return uint16(r.RandUint32() & (1<<16 - 1)) } // It is not safe for cryptographic usage. -func RandUint32() uint32 { - prng.Lock() - u32 := prng.Uint32() - prng.Unlock() +func (r *Rand) RandUint32() uint32 { + r.Lock() + u32 := r.Uint32() + r.Unlock() return u32 } // It is not safe for cryptographic usage. -func RandUint64() uint64 { - return uint64(RandUint32())<<32 + uint64(RandUint32()) +func (r *Rand) RandUint64() uint64 { + return uint64(r.RandUint32())<<32 + uint64(r.RandUint32()) } // It is not safe for cryptographic usage. -func RandUint() uint { - prng.Lock() - i := prng.Int() - prng.Unlock() +func (r *Rand) RandUint() uint { + r.Lock() + i := r.Int() + r.Unlock() return uint(i) } // It is not safe for cryptographic usage. -func RandInt16() int16 { - return int16(RandUint32() & (1<<16 - 1)) +func (r *Rand) RandInt16() int16 { + return int16(r.RandUint32() & (1<<16 - 1)) } // It is not safe for cryptographic usage. -func RandInt32() int32 { - return int32(RandUint32()) +func (r *Rand) RandInt32() int32 { + return int32(r.RandUint32()) } // It is not safe for cryptographic usage. -func RandInt64() int64 { - return int64(RandUint64()) +func (r *Rand) RandInt64() int64 { + return int64(r.RandUint64()) } // It is not safe for cryptographic usage. -func RandInt() int { - prng.Lock() - i := prng.Int() - prng.Unlock() +func (r *Rand) RandInt() int { + r.Lock() + i := r.Int() + r.Unlock() return i } // It is not safe for cryptographic usage. -func RandInt31() int32 { - prng.Lock() - i31 := prng.Int31() - prng.Unlock() +func (r *Rand) RandInt31() int32 { + r.Lock() + i31 := r.Int31() + r.Unlock() return i31 } // It is not safe for cryptographic usage. -func RandInt63() int64 { - prng.Lock() - i63 := prng.Int63() - prng.Unlock() +func (r *Rand) RandInt63() int64 { + r.Lock() + i63 := r.Int63() + r.Unlock() return i63 } // Distributed pseudo-exponentially to test for various cases // It is not safe for cryptographic usage. -func RandUint16Exp() uint16 { - bits := RandUint32() % 16 +func (r *Rand) RandUint16Exp() uint16 { + bits := r.RandUint32() % 16 if bits == 0 { return 0 } n := uint16(1 << (bits - 1)) - n += uint16(RandInt31()) & ((1 << (bits - 1)) - 1) + n += uint16(r.RandInt31()) & ((1 << (bits - 1)) - 1) return n } // Distributed pseudo-exponentially to test for various cases // It is not safe for cryptographic usage. -func RandUint32Exp() uint32 { - bits := RandUint32() % 32 +func (r *Rand) RandUint32Exp() uint32 { + bits := r.RandUint32() % 32 if bits == 0 { return 0 } n := uint32(1 << (bits - 1)) - n += uint32(RandInt31()) & ((1 << (bits - 1)) - 1) + n += uint32(r.RandInt31()) & ((1 << (bits - 1)) - 1) return n } // Distributed pseudo-exponentially to test for various cases // It is not safe for cryptographic usage. -func RandUint64Exp() uint64 { - bits := RandUint32() % 64 +func (r *Rand) RandUint64Exp() uint64 { + bits := r.RandUint32() % 64 if bits == 0 { return 0 } n := uint64(1 << (bits - 1)) - n += uint64(RandInt63()) & ((1 << (bits - 1)) - 1) + n += uint64(r.RandInt63()) & ((1 << (bits - 1)) - 1) return n } // It is not safe for cryptographic usage. -func RandFloat32() float32 { - prng.Lock() - f32 := prng.Float32() - prng.Unlock() +func (r *Rand) RandFloat32() float32 { + r.Lock() + f32 := r.Float32() + r.Unlock() return f32 } // It is not safe for cryptographic usage. -func RandTime() time.Time { - return time.Unix(int64(RandUint64Exp()), 0) +func (r *Rand) RandTime() time.Time { + return time.Unix(int64(r.RandUint64Exp()), 0) } // RandBytes returns n random bytes from the OS's source of entropy ie. via crypto/rand. // It is not safe for cryptographic usage. -func RandBytes(n int) []byte { +func (r *Rand) RandBytes(n int) []byte { // cRandBytes isn't guaranteed to be fast so instead // use random bytes generated from the internal PRNG bs := make([]byte, n) for i := 0; i < len(bs); i++ { - bs[i] = byte(RandInt() & 0xFF) + bs[i] = byte(r.RandInt() & 0xFF) } return bs } @@ -188,19 +292,19 @@ func RandBytes(n int) []byte { // RandIntn returns, as an int, a non-negative pseudo-random number in [0, n). // It panics if n <= 0. // It is not safe for cryptographic usage. -func RandIntn(n int) int { - prng.Lock() - i := prng.Intn(n) - prng.Unlock() +func (r *Rand) RandIntn(n int) int { + r.Lock() + i := r.Intn(n) + r.Unlock() return i } // RandPerm returns a pseudo-random permutation of n integers in [0, n). // It is not safe for cryptographic usage. -func RandPerm(n int) []int { - prng.Lock() - perm := prng.Perm(n) - prng.Unlock() +func (r *Rand) RandPerm(n int) []int { + r.Lock() + perm := r.Perm(n) + r.Unlock() return perm } diff --git a/common/random_test.go b/common/random_test.go index 216f2f8bc..b58b4a13a 100644 --- a/common/random_test.go +++ b/common/random_test.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/json" "fmt" - "io" mrand "math/rand" "sync" "testing" @@ -33,37 +32,38 @@ func TestRandIntn(t *testing.T) { } } -// It is essential that these tests run and never repeat their outputs -// lest we've been pwned and the behavior of our randomness is controlled. -// See Issues: -// * https://github.com/tendermint/tmlibs/issues/99 -// * https://github.com/tendermint/tendermint/issues/973 -func TestUniqueRng(t *testing.T) { - buf := new(bytes.Buffer) - outputs := make(map[string][]int) +// Test to make sure that we never call math.rand(). +// We do this by ensuring that outputs are deterministic. +func TestDeterminism(t *testing.T) { + var firstOutput string + + // Set math/rand's seed for the sake of debugging this test. + // (It isn't strictly necessary). + mrand.Seed(1) + for i := 0; i < 100; i++ { - testThemAll(buf) - output := buf.String() - buf.Reset() - runs, seen := outputs[output] - if seen { - t.Errorf("Run #%d's output was already seen in previous runs: %v", i, runs) + output := testThemAll() + if i == 0 { + firstOutput = output + } else { + if firstOutput != output { + t.Errorf("Run #%d's output was different from first run.\nfirst: %v\nlast: %v", + i, firstOutput, output) + } } - outputs[output] = append(outputs[output], i) } } -func testThemAll(out io.Writer) { - // Reset the internal PRNG - reset() +func testThemAll() string { - // Set math/rand's Seed so that any direct invocations - // of math/rand will reveal themselves. - mrand.Seed(1) + // Such determinism. + grand.reset(1) + + // Use it. + out := new(bytes.Buffer) perm := RandPerm(10) blob, _ := json.Marshal(perm) fmt.Fprintf(out, "perm: %s\n", blob) - fmt.Fprintf(out, "randInt: %d\n", RandInt()) fmt.Fprintf(out, "randUint: %d\n", RandUint()) fmt.Fprintf(out, "randIntn: %d\n", RandIntn(97)) @@ -76,6 +76,7 @@ func testThemAll(out io.Writer) { fmt.Fprintf(out, "randUint16Exp: %d\n", RandUint16Exp()) fmt.Fprintf(out, "randUint32Exp: %d\n", RandUint32Exp()) fmt.Fprintf(out, "randUint64Exp: %d\n", RandUint64Exp()) + return out.String() } func TestRngConcurrencySafety(t *testing.T) { From 6d61ca3bb5354b3c5104e1b112a74e2f4009797d Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Sat, 17 Mar 2018 12:34:23 +0100 Subject: [PATCH 05/40] New -> NewRand --- common/random.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/random.go b/common/random.go index b945a88eb..f70d6477f 100644 --- a/common/random.go +++ b/common/random.go @@ -22,11 +22,11 @@ type Rand struct { var grand *Rand func init() { - grand = New() + grand = NewRand() grand.init() } -func New() *Rand { +func NewRand() *Rand { rand := &Rand{} rand.init() return rand From 90cd89eab08dec6f9b45808371475dbf715d4632 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Sat, 17 Mar 2018 05:18:22 -0700 Subject: [PATCH 06/40] Unexpose r.rand (#167) --- common/random.go | 124 +++++++++++++++++++++++------------------------ 1 file changed, 62 insertions(+), 62 deletions(-) diff --git a/common/random.go b/common/random.go index f70d6477f..a2237487b 100644 --- a/common/random.go +++ b/common/random.go @@ -16,7 +16,7 @@ const ( type Rand struct { sync.Mutex - *mrand.Rand + rand *mrand.Rand } var grand *Rand @@ -43,7 +43,7 @@ func (r *Rand) init() { } func (r *Rand) reset(seed int64) { - r.Rand = mrand.New(mrand.NewSource(seed)) + r.rand = mrand.New(mrand.NewSource(seed)) } //---------------------------------------- @@ -54,79 +54,79 @@ func Seed(seed int64) { } func RandStr(length int) string { - return grand.RandStr(length) + return grand.Str(length) } func RandUint16() uint16 { - return grand.RandUint16() + return grand.Uint16() } func RandUint32() uint32 { - return grand.RandUint32() + return grand.Uint32() } func RandUint64() uint64 { - return grand.RandUint64() + return grand.Uint64() } func RandUint() uint { - return grand.RandUint() + return grand.Uint() } func RandInt16() int16 { - return grand.RandInt16() + return grand.Int16() } func RandInt32() int32 { - return grand.RandInt32() + return grand.Int32() } func RandInt64() int64 { - return grand.RandInt64() + return grand.Int64() } func RandInt() int { - return grand.RandInt() + return grand.Int() } func RandInt31() int32 { - return grand.RandInt31() + return grand.Int31() } func RandInt63() int64 { - return grand.RandInt63() + return grand.Int63() } func RandUint16Exp() uint16 { - return grand.RandUint16Exp() + return grand.Uint16Exp() } func RandUint32Exp() uint32 { - return grand.RandUint32Exp() + return grand.Uint32Exp() } func RandUint64Exp() uint64 { - return grand.RandUint64Exp() + return grand.Uint64Exp() } func RandFloat32() float32 { - return grand.RandFloat32() + return grand.Float32() } func RandTime() time.Time { - return grand.RandTime() + return grand.Time() } func RandBytes(n int) []byte { - return grand.RandBytes(n) + return grand.Bytes(n) } func RandIntn(n int) int { - return grand.RandIntn(n) + return grand.Intn(n) } func RandPerm(n int) []int { - return grand.RandPerm(n) + return grand.Perm(n) } //---------------------------------------- @@ -140,11 +140,11 @@ func (r *Rand) Seed(seed int64) { // Constructs an alphanumeric string of given length. // It is not safe for cryptographic usage. -func (r *Rand) RandStr(length int) string { +func (r *Rand) Str(length int) string { chars := []byte{} MAIN_LOOP: for { - val := r.RandInt63() + val := r.Int63() for i := 0; i < 10; i++ { v := int(val & 0x3f) // rightmost 6 bits if v >= 62 { // only 62 characters in strChars @@ -164,127 +164,127 @@ MAIN_LOOP: } // It is not safe for cryptographic usage. -func (r *Rand) RandUint16() uint16 { - return uint16(r.RandUint32() & (1<<16 - 1)) +func (r *Rand) Uint16() uint16 { + return uint16(r.rand.Uint32() & (1<<16 - 1)) } // It is not safe for cryptographic usage. -func (r *Rand) RandUint32() uint32 { +func (r *Rand) Uint32() uint32 { r.Lock() - u32 := r.Uint32() + u32 := r.rand.Uint32() r.Unlock() return u32 } // It is not safe for cryptographic usage. -func (r *Rand) RandUint64() uint64 { - return uint64(r.RandUint32())<<32 + uint64(r.RandUint32()) +func (r *Rand) Uint64() uint64 { + return uint64(r.rand.Uint32())<<32 + uint64(r.rand.Uint32()) } // It is not safe for cryptographic usage. -func (r *Rand) RandUint() uint { +func (r *Rand) Uint() uint { r.Lock() - i := r.Int() + i := r.rand.Int() r.Unlock() return uint(i) } // It is not safe for cryptographic usage. -func (r *Rand) RandInt16() int16 { - return int16(r.RandUint32() & (1<<16 - 1)) +func (r *Rand) Int16() int16 { + return int16(r.rand.Uint32() & (1<<16 - 1)) } // It is not safe for cryptographic usage. -func (r *Rand) RandInt32() int32 { - return int32(r.RandUint32()) +func (r *Rand) Int32() int32 { + return int32(r.rand.Uint32()) } // It is not safe for cryptographic usage. -func (r *Rand) RandInt64() int64 { - return int64(r.RandUint64()) +func (r *Rand) Int64() int64 { + return int64(r.rand.Uint64()) } // It is not safe for cryptographic usage. -func (r *Rand) RandInt() int { +func (r *Rand) Int() int { r.Lock() - i := r.Int() + i := r.rand.Int() r.Unlock() return i } // It is not safe for cryptographic usage. -func (r *Rand) RandInt31() int32 { +func (r *Rand) Int31() int32 { r.Lock() - i31 := r.Int31() + i31 := r.rand.Int31() r.Unlock() return i31 } // It is not safe for cryptographic usage. -func (r *Rand) RandInt63() int64 { +func (r *Rand) Int63() int64 { r.Lock() - i63 := r.Int63() + i63 := r.rand.Int63() r.Unlock() return i63 } // Distributed pseudo-exponentially to test for various cases // It is not safe for cryptographic usage. -func (r *Rand) RandUint16Exp() uint16 { - bits := r.RandUint32() % 16 +func (r *Rand) Uint16Exp() uint16 { + bits := r.rand.Uint32() % 16 if bits == 0 { return 0 } n := uint16(1 << (bits - 1)) - n += uint16(r.RandInt31()) & ((1 << (bits - 1)) - 1) + n += uint16(r.rand.Int31()) & ((1 << (bits - 1)) - 1) return n } // Distributed pseudo-exponentially to test for various cases // It is not safe for cryptographic usage. -func (r *Rand) RandUint32Exp() uint32 { - bits := r.RandUint32() % 32 +func (r *Rand) Uint32Exp() uint32 { + bits := r.rand.Uint32() % 32 if bits == 0 { return 0 } n := uint32(1 << (bits - 1)) - n += uint32(r.RandInt31()) & ((1 << (bits - 1)) - 1) + n += uint32(r.rand.Int31()) & ((1 << (bits - 1)) - 1) return n } // Distributed pseudo-exponentially to test for various cases // It is not safe for cryptographic usage. -func (r *Rand) RandUint64Exp() uint64 { - bits := r.RandUint32() % 64 +func (r *Rand) Uint64Exp() uint64 { + bits := r.rand.Uint32() % 64 if bits == 0 { return 0 } n := uint64(1 << (bits - 1)) - n += uint64(r.RandInt63()) & ((1 << (bits - 1)) - 1) + n += uint64(r.rand.Int63()) & ((1 << (bits - 1)) - 1) return n } // It is not safe for cryptographic usage. -func (r *Rand) RandFloat32() float32 { +func (r *Rand) Float32() float32 { r.Lock() - f32 := r.Float32() + f32 := r.rand.Float32() r.Unlock() return f32 } // It is not safe for cryptographic usage. -func (r *Rand) RandTime() time.Time { - return time.Unix(int64(r.RandUint64Exp()), 0) +func (r *Rand) Time() time.Time { + return time.Unix(int64(r.Uint64Exp()), 0) } // RandBytes returns n random bytes from the OS's source of entropy ie. via crypto/rand. // It is not safe for cryptographic usage. -func (r *Rand) RandBytes(n int) []byte { +func (r *Rand) Bytes(n int) []byte { // cRandBytes isn't guaranteed to be fast so instead // use random bytes generated from the internal PRNG bs := make([]byte, n) for i := 0; i < len(bs); i++ { - bs[i] = byte(r.RandInt() & 0xFF) + bs[i] = byte(r.rand.Int() & 0xFF) } return bs } @@ -292,18 +292,18 @@ func (r *Rand) RandBytes(n int) []byte { // RandIntn returns, as an int, a non-negative pseudo-random number in [0, n). // It panics if n <= 0. // It is not safe for cryptographic usage. -func (r *Rand) RandIntn(n int) int { +func (r *Rand) Intn(n int) int { r.Lock() - i := r.Intn(n) + i := r.rand.Intn(n) r.Unlock() return i } // RandPerm returns a pseudo-random permutation of n integers in [0, n). // It is not safe for cryptographic usage. -func (r *Rand) RandPerm(n int) []int { +func (r *Rand) Perm(n int) []int { r.Lock() - perm := r.Perm(n) + perm := r.rand.Perm(n) r.Unlock() return perm } From de36bfe31c1052a34b631284f130aab86b8534ae Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Sat, 17 Mar 2018 13:56:39 +0100 Subject: [PATCH 07/40] Add TMLibs/Error from Cosmos-SDK/Error --- common/errors.go | 93 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 91 insertions(+), 2 deletions(-) diff --git a/common/errors.go b/common/errors.go index 4710b9ee0..4a15d0ee6 100644 --- a/common/errors.go +++ b/common/errors.go @@ -2,8 +2,96 @@ package common import ( "fmt" + "runtime" ) +//---------------------------------------- +// Error & cmnError + +type Error interface { + Error() string + Trace(msg string) Error + TraceCause(cause error, msg string) Error + Cause() error +} + +func NewError(msg string) Error { + return newError(msg) +} + +type traceItem struct { + msg string + filename string + lineno int +} + +func (ti traceItem) String() string { + return fmt.Sprintf("%v:%v %v", ti.filename, ti.lineno, ti.msg) +} + +type cmnError struct { + msg string + cause error + traces []traceItem +} + +func newError(msg string) *cmnError { + return &cmnError{ + msg: msg, + cause: nil, + traces: nil, + } +} + +func (err *cmnError) Error() string { + return fmt.Sprintf("Error{%s,%v,%v}", err.msg, err.cause, len(err.traces)) +} + +// Add tracing information with msg. +func (err *cmnError) Trace(msg string) Error { + return err.doTrace(msg, 2) +} + +// Add tracing information with cause and msg. +// If a cause was already set before, it is overwritten. +func (err *cmnError) TraceCause(cause error, msg string) Error { + err.cause = cause + return err.doTrace(msg, 2) +} + +func (err *cmnError) doTrace(msg string, n int) Error { + _, fn, line, ok := runtime.Caller(n) + if !ok { + if fn == "" { + fn = "" + } + if line <= 0 { + line = -1 + } + } + // Include file & line number & msg. + // Do not include the whole stack trace. + err.traces = append(err.traces, traceItem{ + filename: fn, + lineno: line, + msg: msg, + }) + return err +} + +// Return last known cause. +// NOTE: The meaning of "cause" is left for the caller to define. +// There exists to canonical definition of "cause". +// Instead of blaming, try to handle-or-organize it. +func (err *cmnError) Cause() error { + return err.cause +} + +//---------------------------------------- +// StackError + +// NOTE: Used by Tendermint p2p upon recovery. +// Err could be "Reason", since it isn't an error type. type StackError struct { Err interface{} Stack []byte @@ -17,8 +105,9 @@ func (se StackError) Error() string { return se.String() } -//-------------------------------------------------------------------------------------------------- -// panic wrappers +//---------------------------------------- +// Panic wrappers +// XXX DEPRECATED // A panic resulting from a sanity check means there is a programmer error // and some guarantee is not satisfied. From bb875303c29d675c0934bfae026642a8c2a29358 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Sat, 17 Mar 2018 14:42:08 +0100 Subject: [PATCH 08/40] Add NewErrorWithCause() --- common/errors.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/common/errors.go b/common/errors.go index 4a15d0ee6..4e2591a9f 100644 --- a/common/errors.go +++ b/common/errors.go @@ -16,7 +16,11 @@ type Error interface { } func NewError(msg string) Error { - return newError(msg) + return newError(msg, nil) +} + +func NewErrorWithCause(cause error, msg string) Error { + return newError(msg, cause) } type traceItem struct { @@ -35,10 +39,10 @@ type cmnError struct { traces []traceItem } -func newError(msg string) *cmnError { +func newError(msg string, cause error) *cmnError { return &cmnError{ msg: msg, - cause: nil, + cause: cause, traces: nil, } } From 99437a96fb1a074f9bce622cfcfab87a8f41313f Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Sat, 17 Mar 2018 15:23:03 +0100 Subject: [PATCH 09/40] Add efficient implementation of fmt and use for errors. --- common/errors.go | 18 ++++++++++++------ common/string.go | 10 ++++++++-- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/common/errors.go b/common/errors.go index 4e2591a9f..4becd0446 100644 --- a/common/errors.go +++ b/common/errors.go @@ -10,16 +10,19 @@ import ( type Error interface { Error() string - Trace(msg string) Error - TraceCause(cause error, msg string) Error + Trace(format string, a ...interface{}) Error + TraceCause(cause error, format string, a ...interface{}) Error Cause() error } -func NewError(msg string) Error { +func NewError(format string, a ...interface{}) Error { + msg := Fmt(format, a...) return newError(msg, nil) + } -func NewErrorWithCause(cause error, msg string) Error { +func NewErrorWithCause(cause error, format string, a ...interface{}) Error { + msg := Fmt(format, a...) return newError(msg, cause) } @@ -39,6 +42,7 @@ type cmnError struct { traces []traceItem } +// NOTE: Do not expose, it's not very friendly. func newError(msg string, cause error) *cmnError { return &cmnError{ msg: msg, @@ -52,13 +56,15 @@ func (err *cmnError) Error() string { } // Add tracing information with msg. -func (err *cmnError) Trace(msg string) Error { +func (err *cmnError) Trace(format string, a ...interface{}) Error { + msg := Fmt(format, a...) return err.doTrace(msg, 2) } // Add tracing information with cause and msg. // If a cause was already set before, it is overwritten. -func (err *cmnError) TraceCause(cause error, msg string) Error { +func (err *cmnError) TraceCause(cause error, format string, a ...interface{}) Error { + msg := Fmt(format, a...) err.cause = cause return err.doTrace(msg, 2) } diff --git a/common/string.go b/common/string.go index a6895eb25..64484921f 100644 --- a/common/string.go +++ b/common/string.go @@ -6,8 +6,14 @@ import ( "strings" ) -// Fmt shorthand, XXX DEPRECATED -var Fmt = fmt.Sprintf +// Like fmt.Sprintf, but skips formatting if args are empty. +var Fmt = func(format string, a ...interface{}) string { + if len(a) == 0 { + return format + } else { + return fmt.Sprintf(format, a...) + } +} // RightPadString adds spaces to the right of a string to make it length totalLength func RightPadString(s string, totalLength int) string { From 9b9a9e7f8c73f6d0ae8672438a271f32060eebcb Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Sat, 17 Mar 2018 16:32:49 +0100 Subject: [PATCH 10/40] Add Error Type for switching --- common/errors.go | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/common/errors.go b/common/errors.go index 4becd0446..8e4b02283 100644 --- a/common/errors.go +++ b/common/errors.go @@ -13,17 +13,27 @@ type Error interface { Trace(format string, a ...interface{}) Error TraceCause(cause error, format string, a ...interface{}) Error Cause() error + Type() interface{} + WithType(t interface{}) Error } +// New Error with no cause where the type is the format string of the message.. func NewError(format string, a ...interface{}) Error { msg := Fmt(format, a...) - return newError(msg, nil) + return newError(msg, nil, format) } +// New Error with cause where the type is the cause, with message.. func NewErrorWithCause(cause error, format string, a ...interface{}) Error { msg := Fmt(format, a...) - return newError(msg, cause) + return newError(msg, cause, cause) +} + +// New Error with specified type and message. +func NewErrorWithType(type_ interface{}, format string, a ...interface{}) Error { + msg := Fmt(format, a...) + return newError(msg, nil, type_) } type traceItem struct { @@ -39,20 +49,22 @@ func (ti traceItem) String() string { type cmnError struct { msg string cause error + type_ interface{} traces []traceItem } // NOTE: Do not expose, it's not very friendly. -func newError(msg string, cause error) *cmnError { +func newError(msg string, cause error, type_ interface{}) *cmnError { return &cmnError{ msg: msg, cause: cause, + type_: type_, traces: nil, } } func (err *cmnError) Error() string { - return fmt.Sprintf("Error{%s,%v,%v}", err.msg, err.cause, len(err.traces)) + return fmt.Sprintf("Error{%v:%s,%v,%v}", err.type_, err.msg, err.cause, len(err.traces)) } // Add tracing information with msg. @@ -69,6 +81,18 @@ func (err *cmnError) TraceCause(cause error, format string, a ...interface{}) Er return err.doTrace(msg, 2) } +// Return the "type" of this message, primarily for switching +// to handle this error. +func (err *cmnError) Type() interface{} { + return err.type_ +} + +// Overwrites the error's type. +func (err *cmnError) WithType(type_ interface{}) Error { + err.type_ = type_ + return err +} + func (err *cmnError) doTrace(msg string, n int) Error { _, fn, line, ok := runtime.Caller(n) if !ok { From 34125870360c5f0b8d585f90c97ef8acbb66d647 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Sun, 18 Mar 2018 01:50:15 +0100 Subject: [PATCH 11/40] Fix race condition in random.go --- common/random.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/common/random.go b/common/random.go index a2237487b..af531992e 100644 --- a/common/random.go +++ b/common/random.go @@ -165,7 +165,7 @@ MAIN_LOOP: // It is not safe for cryptographic usage. func (r *Rand) Uint16() uint16 { - return uint16(r.rand.Uint32() & (1<<16 - 1)) + return uint16(r.Uint32() & (1<<16 - 1)) } // It is not safe for cryptographic usage. @@ -178,7 +178,7 @@ func (r *Rand) Uint32() uint32 { // It is not safe for cryptographic usage. func (r *Rand) Uint64() uint64 { - return uint64(r.rand.Uint32())<<32 + uint64(r.rand.Uint32()) + return uint64(r.Uint32())<<32 + uint64(r.Uint32()) } // It is not safe for cryptographic usage. @@ -191,17 +191,17 @@ func (r *Rand) Uint() uint { // It is not safe for cryptographic usage. func (r *Rand) Int16() int16 { - return int16(r.rand.Uint32() & (1<<16 - 1)) + return int16(r.Uint32() & (1<<16 - 1)) } // It is not safe for cryptographic usage. func (r *Rand) Int32() int32 { - return int32(r.rand.Uint32()) + return int32(r.Uint32()) } // It is not safe for cryptographic usage. func (r *Rand) Int64() int64 { - return int64(r.rand.Uint64()) + return int64(r.Uint64()) } // It is not safe for cryptographic usage. @@ -231,36 +231,36 @@ func (r *Rand) Int63() int64 { // Distributed pseudo-exponentially to test for various cases // It is not safe for cryptographic usage. func (r *Rand) Uint16Exp() uint16 { - bits := r.rand.Uint32() % 16 + bits := r.Uint32() % 16 if bits == 0 { return 0 } n := uint16(1 << (bits - 1)) - n += uint16(r.rand.Int31()) & ((1 << (bits - 1)) - 1) + n += uint16(r.Int31()) & ((1 << (bits - 1)) - 1) return n } // Distributed pseudo-exponentially to test for various cases // It is not safe for cryptographic usage. func (r *Rand) Uint32Exp() uint32 { - bits := r.rand.Uint32() % 32 + bits := r.Uint32() % 32 if bits == 0 { return 0 } n := uint32(1 << (bits - 1)) - n += uint32(r.rand.Int31()) & ((1 << (bits - 1)) - 1) + n += uint32(r.Int31()) & ((1 << (bits - 1)) - 1) return n } // Distributed pseudo-exponentially to test for various cases // It is not safe for cryptographic usage. func (r *Rand) Uint64Exp() uint64 { - bits := r.rand.Uint32() % 64 + bits := r.Uint32() % 64 if bits == 0 { return 0 } n := uint64(1 << (bits - 1)) - n += uint64(r.rand.Int63()) & ((1 << (bits - 1)) - 1) + n += uint64(r.Int63()) & ((1 << (bits - 1)) - 1) return n } @@ -284,7 +284,7 @@ func (r *Rand) Bytes(n int) []byte { // use random bytes generated from the internal PRNG bs := make([]byte, n) for i := 0; i < len(bs); i++ { - bs[i] = byte(r.rand.Int() & 0xFF) + bs[i] = byte(r.Int() & 0xFF) } return bs } From b0e0dc5de387ccdf96cb4c3e0637f07c73e5e9e0 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Sun, 18 Mar 2018 01:52:28 +0100 Subject: [PATCH 12/40] Implement DebugDB (#166) --- db/common_test.go | 6 +- db/db_test.go | 12 ++- db/debug_db.go | 216 ++++++++++++++++++++++++++++++++++++++++++ db/mem_batch.go | 7 +- db/prefix_db.go | 10 +- merkle/simple_tree.go | 3 + 6 files changed, 239 insertions(+), 15 deletions(-) create mode 100644 db/debug_db.go diff --git a/db/common_test.go b/db/common_test.go index 5afec28b3..7f9d10e9b 100644 --- a/db/common_test.go +++ b/db/common_test.go @@ -107,11 +107,11 @@ func (mdb *mockDB) SetNoLockSync([]byte, []byte) { mdb.calls["SetNoLockSync"] += 1 } -func (mdb *mockDB) Delete([]byte, []byte) { +func (mdb *mockDB) Delete([]byte) { mdb.calls["Delete"] += 1 } -func (mdb *mockDB) DeleteSync([]byte, []byte) { +func (mdb *mockDB) DeleteSync([]byte) { mdb.calls["DeleteSync"] += 1 } @@ -144,7 +144,7 @@ func (mdb *mockDB) NewBatch() Batch { func (mdb *mockDB) Print() { mdb.calls["Print"] += 1 - fmt.Sprintf("mockDB{%v}", mdb.Stats()) + fmt.Printf("mockDB{%v}", mdb.Stats()) } func (mdb *mockDB) Stats() map[string]string { diff --git a/db/db_test.go b/db/db_test.go index 8884cea2d..3d6ac38c4 100644 --- a/db/db_test.go +++ b/db/db_test.go @@ -111,7 +111,8 @@ func TestDBIteratorNonemptyBeginAfter(t *testing.T) { func TestDBBatchWrite1(t *testing.T) { mdb := newMockDB() - batch := mdb.NewBatch() + ddb := NewDebugDB(t.Name(), mdb) + batch := ddb.NewBatch() batch.Set(bz("1"), bz("1")) batch.Set(bz("2"), bz("2")) @@ -131,7 +132,8 @@ func TestDBBatchWrite1(t *testing.T) { func TestDBBatchWrite2(t *testing.T) { mdb := newMockDB() - batch := mdb.NewBatch() + ddb := NewDebugDB(t.Name(), mdb) + batch := ddb.NewBatch() batch.Set(bz("1"), bz("1")) batch.Set(bz("2"), bz("2")) @@ -151,7 +153,8 @@ func TestDBBatchWrite2(t *testing.T) { func TestDBBatchWriteSync1(t *testing.T) { mdb := newMockDB() - batch := mdb.NewBatch() + ddb := NewDebugDB(t.Name(), mdb) + batch := ddb.NewBatch() batch.Set(bz("1"), bz("1")) batch.Set(bz("2"), bz("2")) @@ -171,7 +174,8 @@ func TestDBBatchWriteSync1(t *testing.T) { func TestDBBatchWriteSync2(t *testing.T) { mdb := newMockDB() - batch := mdb.NewBatch() + ddb := NewDebugDB(t.Name(), mdb) + batch := ddb.NewBatch() batch.Set(bz("1"), bz("1")) batch.Set(bz("2"), bz("2")) diff --git a/db/debug_db.go b/db/debug_db.go new file mode 100644 index 000000000..7a15bc294 --- /dev/null +++ b/db/debug_db.go @@ -0,0 +1,216 @@ +package db + +import ( + "fmt" + "sync" +) + +//---------------------------------------- +// debugDB + +type debugDB struct { + label string + db DB +} + +// For printing all operationgs to the console for debugging. +func NewDebugDB(label string, db DB) debugDB { + return debugDB{ + label: label, + db: db, + } +} + +// Implements atomicSetDeleter. +func (ddb debugDB) Mutex() *sync.Mutex { return nil } + +// Implements DB. +func (ddb debugDB) Get(key []byte) (value []byte) { + defer fmt.Printf("%v.Get(%X) %X\n", ddb.label, key, value) + value = ddb.db.Get(key) + return +} + +// Implements DB. +func (ddb debugDB) Has(key []byte) (has bool) { + defer fmt.Printf("%v.Has(%X) %v\n", ddb.label, key, has) + return ddb.db.Has(key) +} + +// Implements DB. +func (ddb debugDB) Set(key []byte, value []byte) { + fmt.Printf("%v.Set(%X, %X)\n", ddb.label, key, value) + ddb.db.Set(key, value) +} + +// Implements DB. +func (ddb debugDB) SetSync(key []byte, value []byte) { + fmt.Printf("%v.SetSync(%X, %X)\n", ddb.label, key, value) + ddb.db.SetSync(key, value) +} + +// Implements atomicSetDeleter. +func (ddb debugDB) SetNoLock(key []byte, value []byte) { + fmt.Printf("%v.SetNoLock(%X, %X)\n", ddb.label, key, value) + ddb.db.Set(key, value) +} + +// Implements atomicSetDeleter. +func (ddb debugDB) SetNoLockSync(key []byte, value []byte) { + fmt.Printf("%v.SetNoLockSync(%X, %X)\n", ddb.label, key, value) + ddb.db.SetSync(key, value) +} + +// Implements DB. +func (ddb debugDB) Delete(key []byte) { + fmt.Printf("%v.Delete(%X)\n", ddb.label, key) + ddb.db.Delete(key) +} + +// Implements DB. +func (ddb debugDB) DeleteSync(key []byte) { + fmt.Printf("%v.DeleteSync(%X)\n", ddb.label, key) + ddb.db.DeleteSync(key) +} + +// Implements atomicSetDeleter. +func (ddb debugDB) DeleteNoLock(key []byte) { + fmt.Printf("%v.DeleteNoLock(%X)\n", ddb.label, key) + ddb.db.Delete(key) +} + +// Implements atomicSetDeleter. +func (ddb debugDB) DeleteNoLockSync(key []byte) { + fmt.Printf("%v.DeleteNoLockSync(%X)\n", ddb.label, key) + ddb.db.DeleteSync(key) +} + +// Implements DB. +func (ddb debugDB) Iterator(start, end []byte) Iterator { + fmt.Printf("%v.Iterator(%X, %X)\n", ddb.label, start, end) + return NewDebugIterator(ddb.label, ddb.db.Iterator(start, end)) +} + +// Implements DB. +func (ddb debugDB) ReverseIterator(start, end []byte) Iterator { + fmt.Printf("%v.ReverseIterator(%X, %X)\n", ddb.label, start, end) + return NewDebugIterator(ddb.label, ddb.db.ReverseIterator(start, end)) +} + +// Implements DB. +func (ddb debugDB) NewBatch() Batch { + fmt.Printf("%v.NewBatch()\n", ddb.label) + return NewDebugBatch(ddb.label, ddb.db.NewBatch()) +} + +// Implements DB. +func (ddb debugDB) Close() { + fmt.Printf("%v.Close()\n", ddb.label) + ddb.db.Close() +} + +// Implements DB. +func (ddb debugDB) Print() { + ddb.db.Print() +} + +// Implements DB. +func (ddb debugDB) Stats() map[string]string { + return ddb.db.Stats() +} + +//---------------------------------------- +// debugIterator + +type debugIterator struct { + label string + itr Iterator +} + +// For printing all operationgs to the console for debugging. +func NewDebugIterator(label string, itr Iterator) debugIterator { + return debugIterator{ + label: label, + itr: itr, + } +} + +// Implements Iterator. +func (ditr debugIterator) Domain() (start []byte, end []byte) { + defer fmt.Printf("%v.itr.Domain() (%X,%X)\n", ditr.label, start, end) + start, end = ditr.itr.Domain() + return +} + +// Implements Iterator. +func (ditr debugIterator) Valid() (ok bool) { + defer fmt.Printf("%v.itr.Valid() %v\n", ditr.label, ok) + ok = ditr.itr.Valid() + return +} + +// Implements Iterator. +func (ditr debugIterator) Next() { + fmt.Printf("%v.itr.Next()\n", ditr.label) + ditr.itr.Next() +} + +// Implements Iterator. +func (ditr debugIterator) Key() (key []byte) { + fmt.Printf("%v.itr.Key() %X\n", ditr.label, key) + key = ditr.itr.Key() + return +} + +// Implements Iterator. +func (ditr debugIterator) Value() (value []byte) { + fmt.Printf("%v.itr.Value() %X\n", ditr.label, value) + value = ditr.itr.Value() + return +} + +// Implements Iterator. +func (ditr debugIterator) Close() { + fmt.Printf("%v.itr.Close()\n", ditr.label) + ditr.itr.Close() +} + +//---------------------------------------- +// debugBatch + +type debugBatch struct { + label string + bch Batch +} + +// For printing all operationgs to the console for debugging. +func NewDebugBatch(label string, bch Batch) debugBatch { + return debugBatch{ + label: label, + bch: bch, + } +} + +// Implements Batch. +func (dbch debugBatch) Set(key, value []byte) { + fmt.Printf("%v.batch.Set(%X, %X)\n", dbch.label, key, value) + dbch.bch.Set(key, value) +} + +// Implements Batch. +func (dbch debugBatch) Delete(key []byte) { + fmt.Printf("%v.batch.Delete(%X)\n", dbch.label, key) + dbch.bch.Delete(key) +} + +// Implements Batch. +func (dbch debugBatch) Write() { + fmt.Printf("%v.batch.Write()\n", dbch.label) + dbch.bch.Write() +} + +// Implements Batch. +func (dbch debugBatch) WriteSync() { + fmt.Printf("%v.batch.WriteSync()\n", dbch.label) + dbch.bch.WriteSync() +} diff --git a/db/mem_batch.go b/db/mem_batch.go index 756798ded..81a63d62b 100644 --- a/db/mem_batch.go +++ b/db/mem_batch.go @@ -45,9 +45,10 @@ func (mBatch *memBatch) WriteSync() { } func (mBatch *memBatch) write(doSync bool) { - mtx := mBatch.db.Mutex() - mtx.Lock() - defer mtx.Unlock() + if mtx := mBatch.db.Mutex(); mtx != nil { + mtx.Lock() + defer mtx.Unlock() + } for i, op := range mBatch.ops { if doSync && i == (len(mBatch.ops)-1) { diff --git a/db/prefix_db.go b/db/prefix_db.go index 5947e7fce..4381ce070 100644 --- a/db/prefix_db.go +++ b/db/prefix_db.go @@ -123,10 +123,10 @@ func (pdb *prefixDB) Iterator(start, end []byte) Iterator { pdb.mtx.Lock() defer pdb.mtx.Unlock() - pstart := append([]byte(pdb.prefix), start...) + pstart := append(pdb.prefix, start...) pend := []byte(nil) if end != nil { - pend = append([]byte(pdb.prefix), end...) + pend = append(pdb.prefix, end...) } return newUnprefixIterator( pdb.prefix, @@ -144,11 +144,11 @@ func (pdb *prefixDB) ReverseIterator(start, end []byte) Iterator { pstart := []byte(nil) if start != nil { - pstart = append([]byte(pdb.prefix), start...) + pstart = append(pdb.prefix, start...) } pend := []byte(nil) if end != nil { - pend = append([]byte(pdb.prefix), end...) + pend = append(pdb.prefix, end...) } return newUnprefixIterator( pdb.prefix, @@ -201,7 +201,7 @@ func (pdb *prefixDB) Stats() map[string]string { } func (pdb *prefixDB) prefixed(key []byte) []byte { - return append([]byte(pdb.prefix), key...) + return append(pdb.prefix, key...) } //---------------------------------------- diff --git a/merkle/simple_tree.go b/merkle/simple_tree.go index a363ea8e8..9bdf52cb2 100644 --- a/merkle/simple_tree.go +++ b/merkle/simple_tree.go @@ -31,6 +31,9 @@ import ( func SimpleHashFromTwoHashes(left []byte, right []byte) []byte { var hasher = ripemd160.New() err := encodeByteSlice(hasher, left) + if err != nil { + panic(err) + } err = encodeByteSlice(hasher, right) if err != nil { panic(err) From b1c9b825311d642be437397759e0e93d7f368cf5 Mon Sep 17 00:00:00 2001 From: Emmanuel T Odeke Date: Sun, 18 Mar 2018 04:17:11 -0700 Subject: [PATCH 13/40] common: NewBitArray never crashes on negatives (#170) Fixes #169 Fixes https://github.com/tendermint/tendermint/issues/1322 The previous code was very trusting assuming that rational actors will use this code. However, Byzantine actors don't care and in the case of the linked issue negative lengths can be sent to this code unfettered having been received from a peer. This code is essentially just a sign change from `==` to `<=` and we've gutted out that attack by being more defensive. --- common/bit_array.go | 2 +- common/bit_array_test.go | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/common/bit_array.go b/common/bit_array.go index 7cc84705e..a3a87ccab 100644 --- a/common/bit_array.go +++ b/common/bit_array.go @@ -15,7 +15,7 @@ type BitArray struct { // There is no BitArray whose Size is 0. Use nil instead. func NewBitArray(bits int) *BitArray { - if bits == 0 { + if bits <= 0 { return nil } return &BitArray{ diff --git a/common/bit_array_test.go b/common/bit_array_test.go index 94a312b7e..fbc438cd1 100644 --- a/common/bit_array_test.go +++ b/common/bit_array_test.go @@ -208,3 +208,10 @@ func TestUpdateNeverPanics(t *testing.T) { b.Update(a) } } + +func TestNewBitArrayNeverCrashesOnNegatives(t *testing.T) { + bitList := []int{-127, -128, -1<<31} + for _, bits := range bitList { + _ = NewBitArray(bits) + } +} From 4b0058dd6422eef7b8939fead2423aa692fecd2c Mon Sep 17 00:00:00 2001 From: Emmanuel T Odeke Date: Sun, 18 Mar 2018 04:19:23 -0700 Subject: [PATCH 14/40] common: remove {Left, Right}PadString (#168) Fixes #134 Those functions are unused in the whole Tendermint Github organization plus they were unnecessariy verbose and could have been concisely replaced with ```go func RightPadString(s string, totalLength uint) string { return fmt.Sprintf("% *s", totalLength, s) } func LeftPadString(s string, totalLength uint) string { return fmt.Sprintf("% -*s", totalLength, s) } ``` delete them anyways --- common/string.go | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/common/string.go b/common/string.go index 64484921f..ae140c9f4 100644 --- a/common/string.go +++ b/common/string.go @@ -15,24 +15,6 @@ var Fmt = func(format string, a ...interface{}) string { } } -// RightPadString adds spaces to the right of a string to make it length totalLength -func RightPadString(s string, totalLength int) string { - remaining := totalLength - len(s) - if remaining > 0 { - s = s + strings.Repeat(" ", remaining) - } - return s -} - -// LeftPadString adds spaces to the left of a string to make it length totalLength -func LeftPadString(s string, totalLength int) string { - remaining := totalLength - len(s) - if remaining > 0 { - s = strings.Repeat(" ", remaining) + s - } - return s -} - // IsHex returns true for non-empty hex-string prefixed with "0x" func IsHex(s string) bool { if len(s) > 2 && strings.EqualFold(s[:2], "0x") { From bb65f097fdb0f0ba9652bb65682676243b255aea Mon Sep 17 00:00:00 2001 From: Alexander Simmerl Date: Mon, 19 Mar 2018 09:38:28 +0100 Subject: [PATCH 15/40] Simplify WriteFileAtomic We can make the implementation more robust by adjusting our assumptions and leverage explicit file modes for syncing. Additionally we going to assume that we want to clean up and can't really recover if thos operations (file close and removal) fail. * utilise file mode for majority of concerns * improve test coverage by covering more assumptions * signature parity with ioutil.WriteFile * always clean up Replaces #160 --- common/os.go | 44 ++++++++++++++++++++++---------------------- common/os_test.go | 34 ++++++++++++++++++++++++++++------ 2 files changed, 50 insertions(+), 28 deletions(-) diff --git a/common/os.go b/common/os.go index 36fc969fa..f1e07115c 100644 --- a/common/os.go +++ b/common/os.go @@ -124,32 +124,32 @@ func MustWriteFile(filePath string, contents []byte, mode os.FileMode) { } } -// WriteFileAtomic writes newBytes to temp and atomically moves to filePath -// when everything else succeeds. -func WriteFileAtomic(filePath string, newBytes []byte, mode os.FileMode) error { - dir := filepath.Dir(filePath) - f, err := ioutil.TempFile(dir, "") +// WriteFileAtomic creates a temporary file with data and the perm given and +// swaps it atomically with filename if successful. +func WriteFileAtomic(filename string, data []byte, perm os.FileMode) error { + var ( + dir = filepath.Dir(filename) + tempFile = filepath.Join(dir, "write-file-atomic-"+RandStr(32)) + // Override in case it does exist, create in case it doesn't and force kernel + // flush, which still leaves the potential of lingering disk cache. + flag = os.O_WRONLY | os.O_CREATE | os.O_SYNC | os.O_TRUNC + ) + + f, err := os.OpenFile(tempFile, flag, perm) if err != nil { return err } - _, err = f.Write(newBytes) - if err == nil { - err = f.Sync() - } - if closeErr := f.Close(); err == nil { - err = closeErr - } - if permErr := os.Chmod(f.Name(), mode); err == nil { - err = permErr - } - if err == nil { - err = os.Rename(f.Name(), filePath) - } - // any err should result in full cleanup - if err != nil { - os.Remove(f.Name()) + // Clean up in any case. Defer stacking order is last-in-first-out. + defer os.Remove(f.Name()) + defer f.Close() + + if n, err := f.Write(data); err != nil { + return err + } else if n < len(data) { + return io.ErrShortWrite } - return err + + return os.Rename(f.Name(), filename) } //-------------------------------------------------------------------------------- diff --git a/common/os_test.go b/common/os_test.go index 126723aa6..97ad672b5 100644 --- a/common/os_test.go +++ b/common/os_test.go @@ -2,30 +2,52 @@ package common import ( "bytes" - "fmt" "io/ioutil" + "math/rand" "os" "testing" "time" ) func TestWriteFileAtomic(t *testing.T) { - data := []byte("Becatron") - fname := fmt.Sprintf("/tmp/write-file-atomic-test-%v.txt", time.Now().UnixNano()) - err := WriteFileAtomic(fname, data, 0664) + var ( + seed = rand.New(rand.NewSource(time.Now().UnixNano())) + data = []byte(RandStr(seed.Intn(2048))) + old = RandBytes(seed.Intn(2048)) + perm os.FileMode = 0600 + ) + + f, err := ioutil.TempFile("/tmp", "write-atomic-test-") if err != nil { t.Fatal(err) } - rData, err := ioutil.ReadFile(fname) + defer os.Remove(f.Name()) + + if err := ioutil.WriteFile(f.Name(), old, 0664); err != nil { + t.Fatal(err) + } + + if err := WriteFileAtomic(f.Name(), data, perm); err != nil { + t.Fatal(err) + } + + rData, err := ioutil.ReadFile(f.Name()) if err != nil { t.Fatal(err) } + if !bytes.Equal(data, rData) { t.Fatalf("data mismatch: %v != %v", data, rData) } - if err := os.Remove(fname); err != nil { + + stat, err := os.Stat(f.Name()) + if err != nil { t.Fatal(err) } + + if have, want := stat.Mode().Perm(), perm; have != want { + t.Errorf("have %v, want %v", have, want) + } } func TestGoPath(t *testing.T) { From bf24f2dcc529772dbb943043eca319441108796c Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Tue, 20 Mar 2018 19:24:18 +0100 Subject: [PATCH 16/40] Implement better Parallel (#174) * Implement better Parallel --- common/async.go | 67 ++++++++++++++++---- common/async_test.go | 145 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 201 insertions(+), 11 deletions(-) create mode 100644 common/async_test.go diff --git a/common/async.go b/common/async.go index 1d302c344..23d1a42b3 100644 --- a/common/async.go +++ b/common/async.go @@ -1,15 +1,60 @@ package common -import "sync" - -func Parallel(tasks ...func()) { - var wg sync.WaitGroup - wg.Add(len(tasks)) - for _, task := range tasks { - go func(task func()) { - task() - wg.Done() - }(task) +// val: the value returned after task execution. +// err: the error returned during task completion. +// abort: tells Parallel to return, whether or not all tasks have completed. +type Task func(i int) (val interface{}, err error, abort bool) + +type TaskResult struct { + Value interface{} + Error error + Panic interface{} +} + +type TaskResultCh <-chan TaskResult + +// Run tasks in parallel, with ability to abort early. +// NOTE: Do not implement quit features here. Instead, provide convenient +// concurrent quit-like primitives, passed implicitly via Task closures. (e.g. +// it's not Parallel's concern how you quit/abort your tasks). +func Parallel(tasks ...Task) []TaskResultCh { + var taskResultChz = make([]TaskResultCh, len(tasks)) // To return. + var taskDoneCh = make(chan bool, len(tasks)) // A "wait group" channel, early abort if any true received. + + // Start all tasks in parallel in separate goroutines. + // When the task is complete, it will appear in the + // respective taskResultCh (associated by task index). + for i, task := range tasks { + var taskResultCh = make(chan TaskResult, 1) // Capacity for 1 result. + taskResultChz[i] = taskResultCh + go func(i int, task Task, taskResultCh chan TaskResult) { + // Recovery + defer func() { + if pnk := recover(); pnk != nil { + taskResultCh <- TaskResult{nil, nil, pnk} + taskDoneCh <- false + } + }() + // Run the task. + var val, err, abort = task(i) + // Send val/err to taskResultCh. + // NOTE: Below this line, nothing must panic/ + taskResultCh <- TaskResult{val, err, nil} + // Decrement waitgroup. + taskDoneCh <- abort + }(i, task, taskResultCh) + } + + // Wait until all tasks are done, or until abort. + for i := 0; i < len(tasks); i++ { + abort := <-taskDoneCh + if abort { + break + } } - wg.Wait() + + // Caller can use this however they want. + // TODO: implement convenience functions to + // make sense of this structure safely. + return taskResultChz } diff --git a/common/async_test.go b/common/async_test.go new file mode 100644 index 000000000..1d6b0e7b0 --- /dev/null +++ b/common/async_test.go @@ -0,0 +1,145 @@ +package common + +import ( + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestParallel(t *testing.T) { + + // Create tasks. + var counter = new(int32) + var tasks = make([]Task, 100*1000) + for i := 0; i < len(tasks); i++ { + tasks[i] = func(i int) (res interface{}, err error, abort bool) { + atomic.AddInt32(counter, 1) + return -1 * i, nil, false + } + } + + // Run in parallel. + var taskResultChz = Parallel(tasks...) + + // Verify. + assert.Equal(t, int(*counter), len(tasks), "Each task should have incremented the counter already") + var failedTasks int + for i := 0; i < len(tasks); i++ { + select { + case taskResult := <-taskResultChz[i]: + if taskResult.Error != nil { + assert.Fail(t, "Task should not have errored but got %v", taskResult.Error) + failedTasks += 1 + } else if !assert.Equal(t, -1*i, taskResult.Value.(int)) { + failedTasks += 1 + } else { + // Good! + } + default: + failedTasks += 1 + } + } + assert.Equal(t, failedTasks, 0, "No task should have failed") + +} + +func TestParallelAbort(t *testing.T) { + + var flow1 = make(chan struct{}, 1) + var flow2 = make(chan struct{}, 1) + var flow3 = make(chan struct{}, 1) // Cap must be > 0 to prevent blocking. + var flow4 = make(chan struct{}, 1) + + // Create tasks. + var tasks = []Task{ + func(i int) (res interface{}, err error, abort bool) { + assert.Equal(t, i, 0) + flow1 <- struct{}{} + return 0, nil, false + }, + func(i int) (res interface{}, err error, abort bool) { + assert.Equal(t, i, 1) + flow2 <- <-flow1 + return 1, errors.New("some error"), false + }, + func(i int) (res interface{}, err error, abort bool) { + assert.Equal(t, i, 2) + flow3 <- <-flow2 + return 2, nil, true + }, + func(i int) (res interface{}, err error, abort bool) { + assert.Equal(t, i, 3) + <-flow4 + return 3, nil, false + }, + } + + // Run in parallel. + var taskResultChz = Parallel(tasks...) + + // Verify task #3. + // Initially taskResultCh[3] sends nothing since flow4 didn't send. + waitTimeout(t, taskResultChz[3], "Task #3") + + // Now let the last task (#3) complete after abort. + flow4 <- <-flow3 + + // Verify task #0, #1, #2. + waitFor(t, taskResultChz[0], "Task #0", 0, nil, nil) + waitFor(t, taskResultChz[1], "Task #1", 1, errors.New("some error"), nil) + waitFor(t, taskResultChz[2], "Task #2", 2, nil, nil) +} + +func TestParallelRecover(t *testing.T) { + + // Create tasks. + var tasks = []Task{ + func(i int) (res interface{}, err error, abort bool) { + return 0, nil, false + }, + func(i int) (res interface{}, err error, abort bool) { + return 1, errors.New("some error"), false + }, + func(i int) (res interface{}, err error, abort bool) { + panic(2) + }, + } + + // Run in parallel. + var taskResultChz = Parallel(tasks...) + + // Verify task #0, #1, #2. + waitFor(t, taskResultChz[0], "Task #0", 0, nil, nil) + waitFor(t, taskResultChz[1], "Task #1", 1, errors.New("some error"), nil) + waitFor(t, taskResultChz[2], "Task #2", nil, nil, 2) +} + +// Wait for result +func waitFor(t *testing.T, taskResultCh TaskResultCh, taskName string, val interface{}, err error, pnk interface{}) { + select { + case taskResult, ok := <-taskResultCh: + assert.True(t, ok, "TaskResultCh unexpectedly closed for %v", taskName) + assert.Equal(t, val, taskResult.Value, taskName) + assert.Equal(t, err, taskResult.Error, taskName) + assert.Equal(t, pnk, taskResult.Panic, taskName) + default: + assert.Fail(t, "Failed to receive result for %v", taskName) + } +} + +// Wait for timeout (no result) +func waitTimeout(t *testing.T, taskResultCh TaskResultCh, taskName string) { + select { + case _, ok := <-taskResultCh: + if !ok { + assert.Fail(t, "TaskResultCh unexpectedly closed (%v)", taskName) + } else { + assert.Fail(t, "TaskResultCh unexpectedly returned for %v", taskName) + } + case <-time.After(1 * time.Second): // TODO use deterministic time? + // Good! + } +} From db48010e813ca5b527ef6081c00fb11e325eedb0 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Tue, 20 Mar 2018 19:58:05 +0100 Subject: [PATCH 17/40] Add return parameter to Parallel --- common/async.go | 18 ++++++++++++++++-- common/async_test.go | 9 ++++++--- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/common/async.go b/common/async.go index 23d1a42b3..e7bc71b1a 100644 --- a/common/async.go +++ b/common/async.go @@ -1,5 +1,9 @@ package common +import ( + "sync/atomic" +) + // val: the value returned after task execution. // err: the error returned during task completion. // abort: tells Parallel to return, whether or not all tasks have completed. @@ -14,12 +18,15 @@ type TaskResult struct { type TaskResultCh <-chan TaskResult // Run tasks in parallel, with ability to abort early. +// Returns ok=false iff any of the tasks returned abort=true. // NOTE: Do not implement quit features here. Instead, provide convenient // concurrent quit-like primitives, passed implicitly via Task closures. (e.g. // it's not Parallel's concern how you quit/abort your tasks). -func Parallel(tasks ...Task) []TaskResultCh { +func Parallel(tasks ...Task) (chz []TaskResultCh, ok bool) { var taskResultChz = make([]TaskResultCh, len(tasks)) // To return. var taskDoneCh = make(chan bool, len(tasks)) // A "wait group" channel, early abort if any true received. + var numPanics = new(int32) // Keep track of panics to set ok=false later. + ok = true // We will set it to false iff any tasks panic'd or returned abort. // Start all tasks in parallel in separate goroutines. // When the task is complete, it will appear in the @@ -31,6 +38,7 @@ func Parallel(tasks ...Task) []TaskResultCh { // Recovery defer func() { if pnk := recover(); pnk != nil { + atomic.AddInt32(numPanics, 1) taskResultCh <- TaskResult{nil, nil, pnk} taskDoneCh <- false } @@ -46,15 +54,21 @@ func Parallel(tasks ...Task) []TaskResultCh { } // Wait until all tasks are done, or until abort. + // DONE_LOOP: for i := 0; i < len(tasks); i++ { abort := <-taskDoneCh if abort { + ok = false break } } + // Ok is also false if there were any panics. + // We must do this check here (after DONE_LOOP). + ok = ok && (atomic.LoadInt32(numPanics) == 0) + // Caller can use this however they want. // TODO: implement convenience functions to // make sense of this structure safely. - return taskResultChz + return taskResultChz, ok } diff --git a/common/async_test.go b/common/async_test.go index 1d6b0e7b0..f2a83d56d 100644 --- a/common/async_test.go +++ b/common/async_test.go @@ -22,7 +22,8 @@ func TestParallel(t *testing.T) { } // Run in parallel. - var taskResultChz = Parallel(tasks...) + var taskResultChz, ok = Parallel(tasks...) + assert.True(t, ok) // Verify. assert.Equal(t, int(*counter), len(tasks), "Each task should have incremented the counter already") @@ -78,7 +79,8 @@ func TestParallelAbort(t *testing.T) { } // Run in parallel. - var taskResultChz = Parallel(tasks...) + var taskResultChz, ok = Parallel(tasks...) + assert.False(t, ok, "ok should be false since we aborted task #2.") // Verify task #3. // Initially taskResultCh[3] sends nothing since flow4 didn't send. @@ -109,7 +111,8 @@ func TestParallelRecover(t *testing.T) { } // Run in parallel. - var taskResultChz = Parallel(tasks...) + var taskResultChz, ok = Parallel(tasks...) + assert.False(t, ok, "ok should be false since we panic'd in task #2.") // Verify task #0, #1, #2. waitFor(t, taskResultChz[0], "Task #0", 0, nil, nil) From 4caf943f49759c5429a1228e390330e622a43738 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Tue, 20 Mar 2018 21:43:58 +0100 Subject: [PATCH 18/40] Parallel returns a TaskResultSet --- common/async.go | 95 +++++++++++++++++++++++++++++++++++++++++--- common/async_test.go | 24 +++++------ 2 files changed, 102 insertions(+), 17 deletions(-) diff --git a/common/async.go b/common/async.go index e7bc71b1a..64e320760 100644 --- a/common/async.go +++ b/common/async.go @@ -4,6 +4,9 @@ import ( "sync/atomic" ) +//---------------------------------------- +// Task + // val: the value returned after task execution. // err: the error returned during task completion. // abort: tells Parallel to return, whether or not all tasks have completed. @@ -17,12 +20,97 @@ type TaskResult struct { type TaskResultCh <-chan TaskResult +type taskResultOK struct { + TaskResult + OK bool +} + +type TaskResultSet struct { + chz []TaskResultCh + results []taskResultOK +} + +func newTaskResultSet(chz []TaskResultCh) *TaskResultSet { + return &TaskResultSet{ + chz: chz, + results: nil, + } +} + +func (trs *TaskResultSet) Channels() []TaskResultCh { + return trs.chz +} + +func (trs *TaskResultSet) LastResult(index int) (TaskResult, bool) { + if len(trs.results) <= index { + return TaskResult{}, false + } + resultOK := trs.results[index] + return resultOK.TaskResult, resultOK.OK +} + +// NOTE: Not concurrency safe. +func (trs *TaskResultSet) Reap() { + if trs.results == nil { + trs.results = make([]taskResultOK, len(trs.chz)) + } + for i := 0; i < len(trs.results); i++ { + var trch = trs.chz[i] + select { + case result := <-trch: + // Overwrite result. + trs.results[i] = taskResultOK{ + TaskResult: result, + OK: true, + } + default: + // Do nothing. + } + } +} + +// Returns the firstmost (by task index) error as +// discovered by all previous Reap() calls. +func (trs *TaskResultSet) FirstValue() interface{} { + for _, result := range trs.results { + if result.Value != nil { + return result.Value + } + } + return nil +} + +// Returns the firstmost (by task index) error as +// discovered by all previous Reap() calls. +func (trs *TaskResultSet) FirstError() error { + for _, result := range trs.results { + if result.Error != nil { + return result.Error + } + } + return nil +} + +// Returns the firstmost (by task index) panic as +// discovered by all previous Reap() calls. +func (trs *TaskResultSet) FirstPanic() interface{} { + for _, result := range trs.results { + if result.Panic != nil { + return result.Panic + } + } + return nil +} + +//---------------------------------------- +// Parallel + // Run tasks in parallel, with ability to abort early. // Returns ok=false iff any of the tasks returned abort=true. // NOTE: Do not implement quit features here. Instead, provide convenient // concurrent quit-like primitives, passed implicitly via Task closures. (e.g. // it's not Parallel's concern how you quit/abort your tasks). -func Parallel(tasks ...Task) (chz []TaskResultCh, ok bool) { +func Parallel(tasks ...Task) (trs *TaskResultSet, ok bool) { var taskResultChz = make([]TaskResultCh, len(tasks)) // To return. var taskDoneCh = make(chan bool, len(tasks)) // A "wait group" channel, early abort if any true received. var numPanics = new(int32) // Keep track of panics to set ok=false later. @@ -67,8 +155,5 @@ func Parallel(tasks ...Task) (chz []TaskResultCh, ok bool) { // We must do this check here (after DONE_LOOP). ok = ok && (atomic.LoadInt32(numPanics) == 0) - // Caller can use this however they want. - // TODO: implement convenience functions to - // make sense of this structure safely. - return taskResultChz, ok + return newTaskResultSet(taskResultChz), ok } diff --git a/common/async_test.go b/common/async_test.go index f2a83d56d..3b47c3fa2 100644 --- a/common/async_test.go +++ b/common/async_test.go @@ -22,7 +22,7 @@ func TestParallel(t *testing.T) { } // Run in parallel. - var taskResultChz, ok = Parallel(tasks...) + var trs, ok = Parallel(tasks...) assert.True(t, ok) // Verify. @@ -30,7 +30,7 @@ func TestParallel(t *testing.T) { var failedTasks int for i := 0; i < len(tasks); i++ { select { - case taskResult := <-taskResultChz[i]: + case taskResult := <-trs.chz[i]: if taskResult.Error != nil { assert.Fail(t, "Task should not have errored but got %v", taskResult.Error) failedTasks += 1 @@ -79,20 +79,20 @@ func TestParallelAbort(t *testing.T) { } // Run in parallel. - var taskResultChz, ok = Parallel(tasks...) + var taskResultSet, ok = Parallel(tasks...) assert.False(t, ok, "ok should be false since we aborted task #2.") // Verify task #3. - // Initially taskResultCh[3] sends nothing since flow4 didn't send. - waitTimeout(t, taskResultChz[3], "Task #3") + // Initially taskResultSet.chz[3] sends nothing since flow4 didn't send. + waitTimeout(t, taskResultSet.chz[3], "Task #3") // Now let the last task (#3) complete after abort. flow4 <- <-flow3 // Verify task #0, #1, #2. - waitFor(t, taskResultChz[0], "Task #0", 0, nil, nil) - waitFor(t, taskResultChz[1], "Task #1", 1, errors.New("some error"), nil) - waitFor(t, taskResultChz[2], "Task #2", 2, nil, nil) + waitFor(t, taskResultSet.chz[0], "Task #0", 0, nil, nil) + waitFor(t, taskResultSet.chz[1], "Task #1", 1, errors.New("some error"), nil) + waitFor(t, taskResultSet.chz[2], "Task #2", 2, nil, nil) } func TestParallelRecover(t *testing.T) { @@ -111,13 +111,13 @@ func TestParallelRecover(t *testing.T) { } // Run in parallel. - var taskResultChz, ok = Parallel(tasks...) + var taskResultSet, ok = Parallel(tasks...) assert.False(t, ok, "ok should be false since we panic'd in task #2.") // Verify task #0, #1, #2. - waitFor(t, taskResultChz[0], "Task #0", 0, nil, nil) - waitFor(t, taskResultChz[1], "Task #1", 1, errors.New("some error"), nil) - waitFor(t, taskResultChz[2], "Task #2", nil, nil, 2) + waitFor(t, taskResultSet.chz[0], "Task #0", 0, nil, nil) + waitFor(t, taskResultSet.chz[1], "Task #1", 1, errors.New("some error"), nil) + waitFor(t, taskResultSet.chz[2], "Task #2", nil, nil, 2) } // Wait for result From 4e5c655944c9a636eaed549e6ad8fd8011fb4d42 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Tue, 20 Mar 2018 23:08:51 +0100 Subject: [PATCH 19/40] Parallel reaps automatically before returning --- common/async.go | 7 +++--- common/async_test.go | 59 +++++++++++++++++++++++--------------------- 2 files changed, 35 insertions(+), 31 deletions(-) diff --git a/common/async.go b/common/async.go index 64e320760..31dc2e968 100644 --- a/common/async.go +++ b/common/async.go @@ -41,7 +41,7 @@ func (trs *TaskResultSet) Channels() []TaskResultCh { return trs.chz } -func (trs *TaskResultSet) LastResult(index int) (TaskResult, bool) { +func (trs *TaskResultSet) LatestResult(index int) (TaskResult, bool) { if len(trs.results) <= index { return TaskResult{}, false } @@ -50,7 +50,7 @@ func (trs *TaskResultSet) LastResult(index int) (TaskResult, bool) { } // NOTE: Not concurrency safe. -func (trs *TaskResultSet) Reap() { +func (trs *TaskResultSet) Reap() *TaskResultSet { if trs.results == nil { trs.results = make([]taskResultOK, len(trs.chz)) } @@ -67,6 +67,7 @@ func (trs *TaskResultSet) Reap() { // Do nothing. } } + return trs } // Returns the firstmost (by task index) error as @@ -155,5 +156,5 @@ func Parallel(tasks ...Task) (trs *TaskResultSet, ok bool) { // We must do this check here (after DONE_LOOP). ok = ok && (atomic.LoadInt32(numPanics) == 0) - return newTaskResultSet(taskResultChz), ok + return newTaskResultSet(taskResultChz).Reap(), ok } diff --git a/common/async_test.go b/common/async_test.go index 3b47c3fa2..2e8db26eb 100644 --- a/common/async_test.go +++ b/common/async_test.go @@ -2,6 +2,7 @@ package common import ( "errors" + "fmt" "sync/atomic" "testing" "time" @@ -29,22 +30,27 @@ func TestParallel(t *testing.T) { assert.Equal(t, int(*counter), len(tasks), "Each task should have incremented the counter already") var failedTasks int for i := 0; i < len(tasks); i++ { - select { - case taskResult := <-trs.chz[i]: - if taskResult.Error != nil { - assert.Fail(t, "Task should not have errored but got %v", taskResult.Error) - failedTasks += 1 - } else if !assert.Equal(t, -1*i, taskResult.Value.(int)) { - failedTasks += 1 - } else { - // Good! - } - default: + taskResult, ok := trs.LatestResult(i) + if !ok { + assert.Fail(t, "Task #%v did not complete.", i) + failedTasks += 1 + } else if taskResult.Error != nil { + assert.Fail(t, "Task should not have errored but got %v", taskResult.Error) + failedTasks += 1 + } else if taskResult.Panic != nil { + assert.Fail(t, "Task should not have panic'd but got %v", taskResult.Panic) + failedTasks += 1 + } else if !assert.Equal(t, -1*i, taskResult.Value.(int)) { + assert.Fail(t, "Task should have returned %v but got %v", -1*i, taskResult.Value.(int)) failedTasks += 1 + } else { + // Good! } } assert.Equal(t, failedTasks, 0, "No task should have failed") - + assert.Nil(t, trs.FirstError(), "There should be no errors") + assert.Nil(t, trs.FirstPanic(), "There should be no panics") + assert.Equal(t, 0, trs.FirstValue(), "First value should be 0") } func TestParallelAbort(t *testing.T) { @@ -90,9 +96,9 @@ func TestParallelAbort(t *testing.T) { flow4 <- <-flow3 // Verify task #0, #1, #2. - waitFor(t, taskResultSet.chz[0], "Task #0", 0, nil, nil) - waitFor(t, taskResultSet.chz[1], "Task #1", 1, errors.New("some error"), nil) - waitFor(t, taskResultSet.chz[2], "Task #2", 2, nil, nil) + checkResult(t, taskResultSet, 0, 0, nil, nil) + checkResult(t, taskResultSet, 1, 1, errors.New("some error"), nil) + checkResult(t, taskResultSet, 2, 2, nil, nil) } func TestParallelRecover(t *testing.T) { @@ -115,22 +121,19 @@ func TestParallelRecover(t *testing.T) { assert.False(t, ok, "ok should be false since we panic'd in task #2.") // Verify task #0, #1, #2. - waitFor(t, taskResultSet.chz[0], "Task #0", 0, nil, nil) - waitFor(t, taskResultSet.chz[1], "Task #1", 1, errors.New("some error"), nil) - waitFor(t, taskResultSet.chz[2], "Task #2", nil, nil, 2) + checkResult(t, taskResultSet, 0, 0, nil, nil) + checkResult(t, taskResultSet, 1, 1, errors.New("some error"), nil) + checkResult(t, taskResultSet, 2, nil, nil, 2) } // Wait for result -func waitFor(t *testing.T, taskResultCh TaskResultCh, taskName string, val interface{}, err error, pnk interface{}) { - select { - case taskResult, ok := <-taskResultCh: - assert.True(t, ok, "TaskResultCh unexpectedly closed for %v", taskName) - assert.Equal(t, val, taskResult.Value, taskName) - assert.Equal(t, err, taskResult.Error, taskName) - assert.Equal(t, pnk, taskResult.Panic, taskName) - default: - assert.Fail(t, "Failed to receive result for %v", taskName) - } +func checkResult(t *testing.T, taskResultSet *TaskResultSet, index int, val interface{}, err error, pnk interface{}) { + taskResult, ok := taskResultSet.LatestResult(index) + taskName := fmt.Sprintf("Task #%v", index) + assert.True(t, ok, "TaskResultCh unexpectedly closed for %v", taskName) + assert.Equal(t, val, taskResult.Value, taskName) + assert.Equal(t, err, taskResult.Error, taskName) + assert.Equal(t, pnk, taskResult.Panic, taskName) } // Wait for timeout (no result) From b6400af7ac9f1ac0d8c5a956ca0f21e05938f46f Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 21 Mar 2018 05:15:30 +0100 Subject: [PATCH 20/40] update version, changelog --- CHANGELOG.md | 19 +++++++++++++++++++ version/version.go | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3a305b20..2fbf64873 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Changelog +## 0.7.1 (TBD) + +BREAKING: + + - [merkle] PutVarint->PutUvarint in encodeByteSlice + - [db] batch.WriteSync() + +FEATURES: + + - [db] NewPrefixDB for a DB with all keys prefixed + +IMPROVEMENTS: + + - glide -> dep + +BUG FIXES: + + + ## 0.7.0 (February 20, 2018) BREAKING: diff --git a/version/version.go b/version/version.go index 2c0474fa8..c683dd245 100644 --- a/version/version.go +++ b/version/version.go @@ -1,3 +1,3 @@ package version -const Version = "0.7.0" +const Version = "0.8.0-dev" From dc1042eb5f429665d7fd9dd2e2720e255c4e2e99 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 22 Mar 2018 13:55:55 -0400 Subject: [PATCH 21/40] finish changelog --- CHANGELOG.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fbf64873..7d3e789a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,20 @@ # Changelog -## 0.7.1 (TBD) +## 0.8.0 (March 22, 2018) BREAKING: - - [merkle] PutVarint->PutUvarint in encodeByteSlice + - [merkle] `PutVarint->PutUvarint` in encodeByteSlice - [db] batch.WriteSync() + - [common] Refactored and fixed `Parallel` function + - [common] Refactored `Rand` functionality + - [common] Remove unused `Right/LeftPadString` functions FEATURES: - [db] NewPrefixDB for a DB with all keys prefixed + - [db] NewDebugDB prints everything during operation + - [common] Error interface (so we don't use pkg/errors) IMPROVEMENTS: @@ -17,7 +22,8 @@ IMPROVEMENTS: BUG FIXES: - + - [common] Fix panic in NewBitArray for negative bits + - [common] Fix and simplify WriteFileAtomic so it cleans up properly ## 0.7.0 (February 20, 2018) From 87c0473730a7fd6f97e9f8be3cfeeaa21102fa41 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Sat, 24 Mar 2018 22:19:44 +0100 Subject: [PATCH 22/40] New Error (#180) * New Error can capture Stacktrace * Intelligent ErrorWrap * Review fixes --- common/errors.go | 248 ++++++++++++++++++++++++++++-------------- common/errors_test.go | 107 ++++++++++++++++++ 2 files changed, 276 insertions(+), 79 deletions(-) create mode 100644 common/errors_test.go diff --git a/common/errors.go b/common/errors.go index 8e4b02283..52c45c799 100644 --- a/common/errors.go +++ b/common/errors.go @@ -5,138 +5,228 @@ import ( "runtime" ) +//---------------------------------------- +// Convenience methods + +// ErrorWrap will just call .TraceFrom(), or create a new *cmnError. +func ErrorWrap(cause interface{}, format string, args ...interface{}) Error { + msg := Fmt(format, args...) + if causeCmnError, ok := cause.(*cmnError); ok { + return causeCmnError.TraceFrom(1, msg) + } else { + // NOTE: cause may be nil. + // NOTE: do not use causeCmnError here, not the same as nil. + return newError(msg, cause, cause).Stacktrace() + } +} + //---------------------------------------- // Error & cmnError +/* +Usage: + +```go + // Error construction + var someT = errors.New("Some err type") + var err1 error = NewErrorWithT(someT, "my message") + ... + // Wrapping + var err2 error = ErrorWrap(err1, "another message") + if (err1 != err2) { panic("should be the same") + ... + // Error handling + switch err2.T() { + case someT: ... + default: ... + } +``` + +*/ type Error interface { Error() string - Trace(format string, a ...interface{}) Error - TraceCause(cause error, format string, a ...interface{}) Error - Cause() error - Type() interface{} - WithType(t interface{}) Error + Message() string + Stacktrace() Error + Trace(format string, args ...interface{}) Error + TraceFrom(offset int, format string, args ...interface{}) Error + Cause() interface{} + WithT(t interface{}) Error + T() interface{} + Format(s fmt.State, verb rune) } // New Error with no cause where the type is the format string of the message.. -func NewError(format string, a ...interface{}) Error { - msg := Fmt(format, a...) +func NewError(format string, args ...interface{}) Error { + msg := Fmt(format, args...) return newError(msg, nil, format) } -// New Error with cause where the type is the cause, with message.. -func NewErrorWithCause(cause error, format string, a ...interface{}) Error { - msg := Fmt(format, a...) - return newError(msg, cause, cause) -} - // New Error with specified type and message. -func NewErrorWithType(type_ interface{}, format string, a ...interface{}) Error { - msg := Fmt(format, a...) - return newError(msg, nil, type_) +func NewErrorWithT(t interface{}, format string, args ...interface{}) Error { + msg := Fmt(format, args...) + return newError(msg, nil, t) } -type traceItem struct { - msg string - filename string - lineno int -} +// NOTE: The name of a function "NewErrorWithCause()" implies that you are +// creating a new Error, yet, if the cause is an Error, creating a new Error to +// hold a ref to the old Error is probably *not* what you want to do. +// So, use ErrorWrap(cause, format, a...) instead, which returns the same error +// if cause is an Error. +// IF you must set an Error as the cause of an Error, +// then you can use the WithCauser interface to do so manually. +// e.g. (error).(tmlibs.WithCauser).WithCause(causeError) -func (ti traceItem) String() string { - return fmt.Sprintf("%v:%v %v", ti.filename, ti.lineno, ti.msg) +type WithCauser interface { + WithCause(cause interface{}) Error } type cmnError struct { - msg string - cause error - type_ interface{} - traces []traceItem + msg string // first msg which also appears in msg + cause interface{} // underlying cause (or panic object) + t interface{} // for switching on error + msgtraces []msgtraceItem // all messages traced + stacktrace []uintptr // first stack trace } -// NOTE: Do not expose, it's not very friendly. -func newError(msg string, cause error, type_ interface{}) *cmnError { +var _ WithCauser = &cmnError{} +var _ Error = &cmnError{} + +// NOTE: do not expose. +func newError(msg string, cause interface{}, t interface{}) *cmnError { return &cmnError{ - msg: msg, - cause: cause, - type_: type_, - traces: nil, + msg: msg, + cause: cause, + t: t, + msgtraces: nil, + stacktrace: nil, } } +func (err *cmnError) Message() string { + return err.msg +} + func (err *cmnError) Error() string { - return fmt.Sprintf("Error{%v:%s,%v,%v}", err.type_, err.msg, err.cause, len(err.traces)) + return fmt.Sprintf("%v", err) +} + +// Captures a stacktrace if one was not already captured. +func (err *cmnError) Stacktrace() Error { + if err.stacktrace == nil { + var offset = 3 + var depth = 32 + err.stacktrace = captureStacktrace(offset, depth) + } + return err } // Add tracing information with msg. -func (err *cmnError) Trace(format string, a ...interface{}) Error { - msg := Fmt(format, a...) - return err.doTrace(msg, 2) +func (err *cmnError) Trace(format string, args ...interface{}) Error { + msg := Fmt(format, args...) + return err.doTrace(msg, 0) } -// Add tracing information with cause and msg. -// If a cause was already set before, it is overwritten. -func (err *cmnError) TraceCause(cause error, format string, a ...interface{}) Error { - msg := Fmt(format, a...) - err.cause = cause - return err.doTrace(msg, 2) +// Same as Trace, but traces the line `offset` calls out. +// If n == 0, the behavior is identical to Trace(). +func (err *cmnError) TraceFrom(offset int, format string, args ...interface{}) Error { + msg := Fmt(format, args...) + return err.doTrace(msg, offset) } -// Return the "type" of this message, primarily for switching -// to handle this error. -func (err *cmnError) Type() interface{} { - return err.type_ +// Return last known cause. +// NOTE: The meaning of "cause" is left for the caller to define. +// There exists no "canonical" definition of "cause". +// Instead of blaming, try to handle it, or organize it. +func (err *cmnError) Cause() interface{} { + return err.cause +} + +// Overwrites the Error's cause. +func (err *cmnError) WithCause(cause interface{}) Error { + err.cause = cause + return err } -// Overwrites the error's type. -func (err *cmnError) WithType(type_ interface{}) Error { - err.type_ = type_ +// Overwrites the Error's type. +func (err *cmnError) WithT(t interface{}) Error { + err.t = t return err } +// Return the "type" of this message, primarily for switching +// to handle this Error. +func (err *cmnError) T() interface{} { + return err.t +} + func (err *cmnError) doTrace(msg string, n int) Error { - _, fn, line, ok := runtime.Caller(n) - if !ok { - if fn == "" { - fn = "" - } - if line <= 0 { - line = -1 - } - } + pc, _, _, _ := runtime.Caller(n + 2) // +1 for doTrace(). +1 for the caller. // Include file & line number & msg. // Do not include the whole stack trace. - err.traces = append(err.traces, traceItem{ - filename: fn, - lineno: line, - msg: msg, + err.msgtraces = append(err.msgtraces, msgtraceItem{ + pc: pc, + msg: msg, }) return err } -// Return last known cause. -// NOTE: The meaning of "cause" is left for the caller to define. -// There exists to canonical definition of "cause". -// Instead of blaming, try to handle-or-organize it. -func (err *cmnError) Cause() error { - return err.cause +func (err *cmnError) Format(s fmt.State, verb rune) { + switch verb { + case 'p': + s.Write([]byte(fmt.Sprintf("%p", &err))) + default: + if s.Flag('#') { + s.Write([]byte("--= Error =--\n")) + // Write msg. + s.Write([]byte(fmt.Sprintf("Message: %#s\n", err.msg))) + // Write cause. + s.Write([]byte(fmt.Sprintf("Cause: %#v\n", err.cause))) + // Write type. + s.Write([]byte(fmt.Sprintf("T: %#v\n", err.t))) + // Write msg trace items. + s.Write([]byte(fmt.Sprintf("Msg Traces:\n"))) + for i, msgtrace := range err.msgtraces { + s.Write([]byte(fmt.Sprintf(" %4d %s\n", i, msgtrace.String()))) + } + // Write stack trace. + if err.stacktrace != nil { + s.Write([]byte(fmt.Sprintf("Stack Trace:\n"))) + for i, pc := range err.stacktrace { + fnc := runtime.FuncForPC(pc) + file, line := fnc.FileLine(pc) + s.Write([]byte(fmt.Sprintf(" %4d %s:%d\n", i, file, line))) + } + } + s.Write([]byte("--= /Error =--\n")) + } else { + // Write msg. + s.Write([]byte(fmt.Sprintf("Error{`%v`}", err.msg))) // TODO tick-esc? + } + } } //---------------------------------------- -// StackError +// stacktrace & msgtraceItem -// NOTE: Used by Tendermint p2p upon recovery. -// Err could be "Reason", since it isn't an error type. -type StackError struct { - Err interface{} - Stack []byte +func captureStacktrace(offset int, depth int) []uintptr { + var pcs = make([]uintptr, depth) + n := runtime.Callers(offset, pcs) + return pcs[0:n] } -func (se StackError) String() string { - return fmt.Sprintf("Error: %v\nStack: %s", se.Err, se.Stack) +type msgtraceItem struct { + pc uintptr + msg string } -func (se StackError) Error() string { - return se.String() +func (mti msgtraceItem) String() string { + fnc := runtime.FuncForPC(mti.pc) + file, line := fnc.FileLine(mti.pc) + return fmt.Sprintf("%s:%d - %s", + file, line, + mti.msg, + ) } //---------------------------------------- diff --git a/common/errors_test.go b/common/errors_test.go new file mode 100644 index 000000000..56f366179 --- /dev/null +++ b/common/errors_test.go @@ -0,0 +1,107 @@ +package common + +import ( + fmt "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestErrorPanic(t *testing.T) { + type pnk struct { + msg string + } + + capturePanic := func() (err Error) { + defer func() { + if r := recover(); r != nil { + err = ErrorWrap(r, "This is the message in ErrorWrap(r, message).") + } + return + }() + panic(pnk{"something"}) + return nil + } + + var err = capturePanic() + + assert.Equal(t, pnk{"something"}, err.Cause()) + assert.Equal(t, pnk{"something"}, err.T()) + assert.Equal(t, "This is the message in ErrorWrap(r, message).", err.Message()) + assert.Equal(t, "Error{`This is the message in ErrorWrap(r, message).`}", fmt.Sprintf("%v", err)) + assert.Contains(t, fmt.Sprintf("%#v", err), "Message: This is the message in ErrorWrap(r, message).") + assert.Contains(t, fmt.Sprintf("%#v", err), "Stack Trace:\n 0") +} + +func TestErrorWrapSomething(t *testing.T) { + + var err = ErrorWrap("something", "formatter%v%v", 0, 1) + + assert.Equal(t, "something", err.Cause()) + assert.Equal(t, "something", err.T()) + assert.Equal(t, "formatter01", err.Message()) + assert.Equal(t, "Error{`formatter01`}", fmt.Sprintf("%v", err)) + assert.Regexp(t, `Message: formatter01\n`, fmt.Sprintf("%#v", err)) + assert.Contains(t, fmt.Sprintf("%#v", err), "Stack Trace:\n 0") +} + +func TestErrorWrapNothing(t *testing.T) { + + var err = ErrorWrap(nil, "formatter%v%v", 0, 1) + + assert.Equal(t, nil, err.Cause()) + assert.Equal(t, nil, err.T()) + assert.Equal(t, "formatter01", err.Message()) + assert.Equal(t, "Error{`formatter01`}", fmt.Sprintf("%v", err)) + assert.Regexp(t, `Message: formatter01\n`, fmt.Sprintf("%#v", err)) + assert.Contains(t, fmt.Sprintf("%#v", err), "Stack Trace:\n 0") +} + +func TestErrorNewError(t *testing.T) { + + var err = NewError("formatter%v%v", 0, 1) + + assert.Equal(t, nil, err.Cause()) + assert.Equal(t, "formatter%v%v", err.T()) + assert.Equal(t, "formatter01", err.Message()) + assert.Equal(t, "Error{`formatter01`}", fmt.Sprintf("%v", err)) + assert.Regexp(t, `Message: formatter01\n`, fmt.Sprintf("%#v", err)) + assert.NotContains(t, fmt.Sprintf("%#v", err), "Stack Trace") +} + +func TestErrorNewErrorWithStacktrace(t *testing.T) { + + var err = NewError("formatter%v%v", 0, 1).Stacktrace() + + assert.Equal(t, nil, err.Cause()) + assert.Equal(t, "formatter%v%v", err.T()) + assert.Equal(t, "formatter01", err.Message()) + assert.Equal(t, "Error{`formatter01`}", fmt.Sprintf("%v", err)) + assert.Regexp(t, `Message: formatter01\n`, fmt.Sprintf("%#v", err)) + assert.Contains(t, fmt.Sprintf("%#v", err), "Stack Trace:\n 0") +} + +func TestErrorNewErrorWithTrace(t *testing.T) { + + var err = NewError("formatter%v%v", 0, 1) + err.Trace("trace %v", 1) + err.Trace("trace %v", 2) + err.Trace("trace %v", 3) + + assert.Equal(t, nil, err.Cause()) + assert.Equal(t, "formatter%v%v", err.T()) + assert.Equal(t, "formatter01", err.Message()) + assert.Equal(t, "Error{`formatter01`}", fmt.Sprintf("%v", err)) + assert.Regexp(t, `Message: formatter01\n`, fmt.Sprintf("%#v", err)) + dump := fmt.Sprintf("%#v", err) + assert.NotContains(t, dump, "Stack Trace") + assert.Regexp(t, `common/errors_test\.go:[0-9]+ - trace 1`, dump) + assert.Regexp(t, `common/errors_test\.go:[0-9]+ - trace 2`, dump) + assert.Regexp(t, `common/errors_test\.go:[0-9]+ - trace 3`, dump) +} + +func TestErrorWrapError(t *testing.T) { + var err1 error = NewError("my message") + var err2 error = ErrorWrap(err1, "another message") + assert.Equal(t, err1, err2) +} From e9cf47606cfcbdc28a7c16671b4a70b459e9d4cc Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Sun, 25 Mar 2018 00:04:47 +0100 Subject: [PATCH 23/40] Merge panics into errors in Parallel --- common/async.go | 16 ++-------------- common/async_test.go | 13 +++++++------ 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/common/async.go b/common/async.go index 31dc2e968..49714d95e 100644 --- a/common/async.go +++ b/common/async.go @@ -15,7 +15,6 @@ type Task func(i int) (val interface{}, err error, abort bool) type TaskResult struct { Value interface{} Error error - Panic interface{} } type TaskResultCh <-chan TaskResult @@ -92,17 +91,6 @@ func (trs *TaskResultSet) FirstError() error { return nil } -// Returns the firstmost (by task index) panic as -// discovered by all previous Reap() calls. -func (trs *TaskResultSet) FirstPanic() interface{} { - for _, result := range trs.results { - if result.Panic != nil { - return result.Panic - } - } - return nil -} - //---------------------------------------- // Parallel @@ -128,7 +116,7 @@ func Parallel(tasks ...Task) (trs *TaskResultSet, ok bool) { defer func() { if pnk := recover(); pnk != nil { atomic.AddInt32(numPanics, 1) - taskResultCh <- TaskResult{nil, nil, pnk} + taskResultCh <- TaskResult{nil, ErrorWrap(pnk, "Panic in task")} taskDoneCh <- false } }() @@ -136,7 +124,7 @@ func Parallel(tasks ...Task) (trs *TaskResultSet, ok bool) { var val, err, abort = task(i) // Send val/err to taskResultCh. // NOTE: Below this line, nothing must panic/ - taskResultCh <- TaskResult{val, err, nil} + taskResultCh <- TaskResult{val, err} // Decrement waitgroup. taskDoneCh <- abort }(i, task, taskResultCh) diff --git a/common/async_test.go b/common/async_test.go index 2e8db26eb..8d41ec35a 100644 --- a/common/async_test.go +++ b/common/async_test.go @@ -37,9 +37,6 @@ func TestParallel(t *testing.T) { } else if taskResult.Error != nil { assert.Fail(t, "Task should not have errored but got %v", taskResult.Error) failedTasks += 1 - } else if taskResult.Panic != nil { - assert.Fail(t, "Task should not have panic'd but got %v", taskResult.Panic) - failedTasks += 1 } else if !assert.Equal(t, -1*i, taskResult.Value.(int)) { assert.Fail(t, "Task should have returned %v but got %v", -1*i, taskResult.Value.(int)) failedTasks += 1 @@ -49,7 +46,6 @@ func TestParallel(t *testing.T) { } assert.Equal(t, failedTasks, 0, "No task should have failed") assert.Nil(t, trs.FirstError(), "There should be no errors") - assert.Nil(t, trs.FirstPanic(), "There should be no panics") assert.Equal(t, 0, trs.FirstValue(), "First value should be 0") } @@ -132,8 +128,13 @@ func checkResult(t *testing.T, taskResultSet *TaskResultSet, index int, val inte taskName := fmt.Sprintf("Task #%v", index) assert.True(t, ok, "TaskResultCh unexpectedly closed for %v", taskName) assert.Equal(t, val, taskResult.Value, taskName) - assert.Equal(t, err, taskResult.Error, taskName) - assert.Equal(t, pnk, taskResult.Panic, taskName) + if err != nil { + assert.Equal(t, err, taskResult.Error, taskName) + } else if pnk != nil { + assert.Equal(t, pnk, taskResult.Error.(Error).Cause(), taskName) + } else { + assert.Nil(t, taskResult.Error, taskName) + } } // Wait for timeout (no result) From 6e26392209cba2758a36fe37f125ecdfc22db5fa Mon Sep 17 00:00:00 2001 From: Christopher Goes Date: Wed, 28 Mar 2018 15:35:52 +0200 Subject: [PATCH 24/40] Return config parse errors (#182) --- cli/setup.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cli/setup.go b/cli/setup.go index dc34abdf9..06cf1cd1f 100644 --- a/cli/setup.go +++ b/cli/setup.go @@ -139,9 +139,8 @@ func bindFlagsLoadViper(cmd *cobra.Command, args []string) error { // stderr, so if we redirect output to json file, this doesn't appear // fmt.Fprintln(os.Stderr, "Using config file:", viper.ConfigFileUsed()) } else if _, ok := err.(viper.ConfigFileNotFoundError); !ok { - // we ignore not found error, only parse error - // stderr, so if we redirect output to json file, this doesn't appear - fmt.Fprintf(os.Stderr, "%#v", err) + // ignore not found error, return other errors + return err } return nil } From 0f2811441f4cf44b414df16ceae3c0931c74662e Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 28 Mar 2018 16:04:46 +0200 Subject: [PATCH 25/40] [pubsub] fix unsubscribing (#181) * [pubsub] fix unsubscribing by giving it the same exact query, which was used to subscribe Refs https://github.com/tendermint/tendermint/issues/1368 * use original query to unsubscribe Refs #1368 * modify the unit test the issue is fixed --- CHANGELOG.md | 6 ++++++ pubsub/pubsub.go | 30 +++++++++++++++++++++--------- pubsub/pubsub_test.go | 4 ++-- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d3e789a5..00ddd4b4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 0.8.1 (TBD) + +BUG FIXES: + + - [pubsub] fix unsubscribing + ## 0.8.0 (March 22, 2018) BREAKING: diff --git a/pubsub/pubsub.go b/pubsub/pubsub.go index 28e008ca6..90f6e4ae6 100644 --- a/pubsub/pubsub.go +++ b/pubsub/pubsub.go @@ -28,6 +28,16 @@ const ( shutdown ) +var ( + // ErrSubscriptionNotFound is returned when a client tries to unsubscribe + // from not existing subscription. + ErrSubscriptionNotFound = errors.New("subscription not found") + + // ErrAlreadySubscribed is returned when a client tries to subscribe twice or + // more using the same query. + ErrAlreadySubscribed = errors.New("already subscribed") +) + type cmd struct { op operation query Query @@ -52,7 +62,7 @@ type Server struct { cmdsCap int mtx sync.RWMutex - subscriptions map[string]map[string]struct{} // subscriber -> query -> struct{} + subscriptions map[string]map[string]Query // subscriber -> query (string) -> Query } // Option sets a parameter for the server. @@ -63,7 +73,7 @@ type Option func(*Server) // provided, the resulting server's queue is unbuffered. func NewServer(options ...Option) *Server { s := &Server{ - subscriptions: make(map[string]map[string]struct{}), + subscriptions: make(map[string]map[string]Query), } s.BaseService = *cmn.NewBaseService(nil, "PubSub", s) @@ -106,16 +116,16 @@ func (s *Server) Subscribe(ctx context.Context, clientID string, query Query, ou } s.mtx.RUnlock() if ok { - return errors.New("already subscribed") + return ErrAlreadySubscribed } select { case s.cmds <- cmd{op: sub, clientID: clientID, query: query, ch: out}: s.mtx.Lock() if _, ok = s.subscriptions[clientID]; !ok { - s.subscriptions[clientID] = make(map[string]struct{}) + s.subscriptions[clientID] = make(map[string]Query) } - s.subscriptions[clientID][query.String()] = struct{}{} + s.subscriptions[clientID][query.String()] = query s.mtx.Unlock() return nil case <-ctx.Done(): @@ -127,18 +137,20 @@ func (s *Server) Subscribe(ctx context.Context, clientID string, query Query, ou // returned to the caller if the context is canceled or if subscription does // not exist. func (s *Server) Unsubscribe(ctx context.Context, clientID string, query Query) error { + var origQuery Query s.mtx.RLock() clientSubscriptions, ok := s.subscriptions[clientID] if ok { - _, ok = clientSubscriptions[query.String()] + origQuery, ok = clientSubscriptions[query.String()] } s.mtx.RUnlock() if !ok { - return errors.New("subscription not found") + return ErrSubscriptionNotFound } + // original query is used here because we're using pointers as map keys select { - case s.cmds <- cmd{op: unsub, clientID: clientID, query: query}: + case s.cmds <- cmd{op: unsub, clientID: clientID, query: origQuery}: s.mtx.Lock() delete(clientSubscriptions, query.String()) s.mtx.Unlock() @@ -155,7 +167,7 @@ func (s *Server) UnsubscribeAll(ctx context.Context, clientID string) error { _, ok := s.subscriptions[clientID] s.mtx.RUnlock() if !ok { - return errors.New("subscription not found") + return ErrSubscriptionNotFound } select { diff --git a/pubsub/pubsub_test.go b/pubsub/pubsub_test.go index 84b6aa218..2af7cea46 100644 --- a/pubsub/pubsub_test.go +++ b/pubsub/pubsub_test.go @@ -101,9 +101,9 @@ func TestUnsubscribe(t *testing.T) { ctx := context.Background() ch := make(chan interface{}) - err := s.Subscribe(ctx, clientID, query.Empty{}, ch) + err := s.Subscribe(ctx, clientID, query.MustParse("tm.events.type='NewBlock'"), ch) require.NoError(t, err) - err = s.Unsubscribe(ctx, clientID, query.Empty{}) + err = s.Unsubscribe(ctx, clientID, query.MustParse("tm.events.type='NewBlock'")) require.NoError(t, err) err = s.Publish(ctx, "Nick Fury") From 898216d419ad046ee3c3dee91df851e4e30461d7 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 29 Mar 2018 12:04:01 +0200 Subject: [PATCH 26/40] add SplitAndTrim func (#183) Refs https://github.com/tendermint/tendermint/issues/1380 --- common/string.go | 17 +++++++++++++++++ common/string_test.go | 19 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/common/string.go b/common/string.go index ae140c9f4..dfa262d3f 100644 --- a/common/string.go +++ b/common/string.go @@ -41,3 +41,20 @@ func StringInSlice(a string, list []string) bool { } return false } + +// SplitAndTrim slices s into all subslices separated by sep and returns a +// slice of the string s with all leading and trailing Unicode code points +// contained in cutset removed. If sep is empty, SplitAndTrim splits after each +// UTF-8 sequence. First part is equivalent to strings.SplitN with a count of +// -1. +func SplitAndTrim(s, sep, cutset string) []string { + if s == "" { + return []string{} + } + + spl := strings.Split(s, sep) + for i := 0; i < len(spl); i++ { + spl[i] = strings.Trim(spl[i], cutset) + } + return spl +} diff --git a/common/string_test.go b/common/string_test.go index b8a917c16..82ba67844 100644 --- a/common/string_test.go +++ b/common/string_test.go @@ -30,3 +30,22 @@ func TestIsHex(t *testing.T) { assert.True(t, IsHex(v), "%q is hex", v) } } + +func TestSplitAndTrim(t *testing.T) { + testCases := []struct { + s string + sep string + cutset string + expected []string + }{ + {"a,b,c", ",", " ", []string{"a", "b", "c"}}, + {" a , b , c ", ",", " ", []string{"a", "b", "c"}}, + {" a, b, c ", ",", " ", []string{"a", "b", "c"}}, + {" , ", ",", " ", []string{"", ""}}, + {" ", ",", " ", []string{""}}, + } + + for _, tc := range testCases { + assert.Equal(t, tc.expected, SplitAndTrim(tc.s, tc.sep, tc.cutset), "%s", tc.s) + } +} From 382e99d06e8bdc2818704e4a22d23a11110803fa Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Mon, 2 Apr 2018 01:46:24 -0700 Subject: [PATCH 27/40] Add IsTypedNil --- common/nil.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 common/nil.go diff --git a/common/nil.go b/common/nil.go new file mode 100644 index 000000000..c7617f083 --- /dev/null +++ b/common/nil.go @@ -0,0 +1,18 @@ +package common + +import "reflect" + +// Go lacks a simple and safe way to see if something is a typed nil. +// See: +// - https://dave.cheney.net/2017/08/09/typed-nils-in-go-2 +// - https://groups.google.com/forum/#!topic/golang-nuts/wnH302gBa4I/discussion +// - https://github.com/golang/go/issues/21538 +func IsTypedNil(o interface{}) bool { + rv := reflect.ValueOf(o) + switch rv.Kind() { + case reflect.Chan, reflect.Func, reflect.Map, reflect.Ptr, reflect.Slice: + return rv.IsNil() + default: + return false + } +} From 2fbd9f15fae4a0d73f2c3d10f839a1e7f5451e36 Mon Sep 17 00:00:00 2001 From: Mohanson Date: Tue, 3 Apr 2018 15:26:47 +0800 Subject: [PATCH 28/40] bug fix: WriteFileAtomic Must close file before rename it. --- common/os.go | 1 + 1 file changed, 1 insertion(+) diff --git a/common/os.go b/common/os.go index f1e07115c..47ae4a1cc 100644 --- a/common/os.go +++ b/common/os.go @@ -148,6 +148,7 @@ func WriteFileAtomic(filename string, data []byte, perm os.FileMode) error { } else if n < len(data) { return io.ErrShortWrite } + f.Close() return os.Rename(f.Name(), filename) } From 29a8cb8d87e0ed2567938aed57bb78cc8cd6fee9 Mon Sep 17 00:00:00 2001 From: Mohanson Date: Tue, 3 Apr 2018 16:51:30 +0800 Subject: [PATCH 29/40] add comments. --- common/os.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/common/os.go b/common/os.go index 47ae4a1cc..8a0c14f46 100644 --- a/common/os.go +++ b/common/os.go @@ -148,6 +148,9 @@ func WriteFileAtomic(filename string, data []byte, perm os.FileMode) error { } else if n < len(data) { return io.ErrShortWrite } + // Close the file before renaming it, otherwise it will cause "The process + // cannot access the file because it is being used by another process." on windows or + // cause "cross-link error" on linux when you try to save it to another partition. f.Close() return os.Rename(f.Name(), filename) From ee67e34519a87a513731892d60c6ab97855f0c53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Corbi=C3=A8re?= Date: Tue, 3 Apr 2018 12:23:28 +0200 Subject: [PATCH 30/40] Fix lint errors (#190) * use increment and decrement operators. * remove unnecessary else branches. * fix receiver names. * remove omittable code. * fix dot imports. --- autofile/autofile.go | 7 +++--- autofile/group.go | 48 ++++++++++++++++++------------------------ autofile/group_test.go | 2 +- clist/clist.go | 4 ++-- clist/clist_test.go | 6 +++--- common/async_test.go | 6 +++--- common/bit_array.go | 3 +-- common/colors.go | 3 +-- common/errors.go | 7 +++--- common/io.go | 3 +-- common/os.go | 11 +++++----- common/service.go | 13 +++++------- common/string.go | 3 +-- common/word.go | 3 +-- db/backend_test.go | 2 +- db/common_test.go | 44 +++++++++++++++++++------------------- db/db_test.go | 12 +++++------ db/go_level_db.go | 13 ++++++------ db/go_level_db_test.go | 2 +- db/mem_db.go | 2 +- db/types.go | 3 +-- db/util.go | 28 ++++++++++++------------ db/util_test.go | 10 ++++----- events/events.go | 8 +++---- events/events_test.go | 4 ++-- merkle/simple_proof.go | 11 +++++----- test/mutate.go | 8 +++---- 27 files changed, 121 insertions(+), 145 deletions(-) diff --git a/autofile/autofile.go b/autofile/autofile.go index 05fb0d677..790be5224 100644 --- a/autofile/autofile.go +++ b/autofile/autofile.go @@ -5,7 +5,7 @@ import ( "sync" "time" - . "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tmlibs/common" ) /* AutoFile usage @@ -44,7 +44,7 @@ type AutoFile struct { func OpenAutoFile(path string) (af *AutoFile, err error) { af = &AutoFile{ - ID: RandStr(12) + ":" + path, + ID: cmn.RandStr(12) + ":" + path, Path: path, ticker: time.NewTicker(autoFileOpenDuration), } @@ -129,9 +129,8 @@ func (af *AutoFile) Size() (int64, error) { if err != nil { if err == os.ErrNotExist { return 0, nil - } else { - return -1, err } + return -1, err } } stat, err := af.file.Stat() diff --git a/autofile/group.go b/autofile/group.go index f2d0f2bae..652c33310 100644 --- a/autofile/group.go +++ b/autofile/group.go @@ -15,7 +15,7 @@ import ( "sync" "time" - . "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tmlibs/common" ) const ( @@ -54,7 +54,7 @@ The Group can also be used to binary-search for some line, assuming that marker lines are written occasionally. */ type Group struct { - BaseService + cmn.BaseService ID string Head *AutoFile // The head AutoFile to write to @@ -90,7 +90,7 @@ func OpenGroup(headPath string) (g *Group, err error) { minIndex: 0, maxIndex: 0, } - g.BaseService = *NewBaseService(nil, "Group", g) + g.BaseService = *cmn.NewBaseService(nil, "Group", g) gInfo := g.readGroupInfo() g.minIndex = gInfo.MinIndex @@ -267,7 +267,7 @@ func (g *Group) RotateFile() { panic(err) } - g.maxIndex += 1 + g.maxIndex++ } // NewReader returns a new group reader. @@ -277,9 +277,8 @@ func (g *Group) NewReader(index int) (*GroupReader, error) { err := r.SetIndex(index) if err != nil { return nil, err - } else { - return r, nil } + return r, nil } // Returns -1 if line comes after, 0 if found, 1 if line comes before. @@ -311,9 +310,8 @@ func (g *Group) Search(prefix string, cmp SearchFunc) (*GroupReader, bool, error if err != nil { r.Close() return nil, false, err - } else { - return r, match, err } + return r, match, err } // Read starting roughly at the middle file, @@ -349,9 +347,8 @@ func (g *Group) Search(prefix string, cmp SearchFunc) (*GroupReader, bool, error if err != nil { r.Close() return nil, false, err - } else { - return r, true, err } + return r, true, err } else { // We passed it maxIndex = curIndex - 1 @@ -429,9 +426,8 @@ GROUP_LOOP: if err == io.EOF { if found { return match, found, nil - } else { - continue GROUP_LOOP } + continue GROUP_LOOP } else if err != nil { return "", false, err } @@ -442,9 +438,8 @@ GROUP_LOOP: if r.CurIndex() > i { if found { return match, found, nil - } else { - continue GROUP_LOOP } + continue GROUP_LOOP } } } @@ -520,7 +515,7 @@ func (g *Group) readGroupInfo() GroupInfo { minIndex, maxIndex = 0, 0 } else { // Otherwise, the head file is 1 greater - maxIndex += 1 + maxIndex++ } return GroupInfo{minIndex, maxIndex, totalSize, headSize} } @@ -528,9 +523,8 @@ func (g *Group) readGroupInfo() GroupInfo { func filePathForIndex(headPath string, index int, maxIndex int) string { if index == maxIndex { return headPath - } else { - return fmt.Sprintf("%v.%03d", headPath, index) } + return fmt.Sprintf("%v.%03d", headPath, index) } //-------------------------------------------------------------------------------- @@ -567,9 +561,8 @@ func (gr *GroupReader) Close() error { gr.curFile = nil gr.curLine = nil return err - } else { - return nil } + return nil } // Read implements io.Reader, reading bytes from the current Reader @@ -598,10 +591,10 @@ func (gr *GroupReader) Read(p []byte) (n int, err error) { if err == io.EOF { if n >= lenP { return n, nil - } else { // Open the next file - if err1 := gr.openFile(gr.curIndex + 1); err1 != nil { - return n, err1 - } + } + // Open the next file + if err1 := gr.openFile(gr.curIndex + 1); err1 != nil { + return n, err1 } } else if err != nil { return n, err @@ -643,10 +636,9 @@ func (gr *GroupReader) ReadLine() (string, error) { } if len(bytesRead) > 0 && bytesRead[len(bytesRead)-1] == byte('\n') { return linePrefix + string(bytesRead[:len(bytesRead)-1]), nil - } else { - linePrefix += string(bytesRead) - continue } + linePrefix += string(bytesRead) + continue } else if err != nil { return "", err } @@ -726,11 +718,11 @@ func (gr *GroupReader) SetIndex(index int) error { func MakeSimpleSearchFunc(prefix string, target int) SearchFunc { return func(line string) (int, error) { if !strings.HasPrefix(line, prefix) { - return -1, errors.New(Fmt("Marker line did not have prefix: %v", prefix)) + return -1, errors.New(cmn.Fmt("Marker line did not have prefix: %v", prefix)) } i, err := strconv.Atoi(line[len(prefix):]) if err != nil { - return -1, errors.New(Fmt("Failed to parse marker line: %v", err.Error())) + return -1, errors.New(cmn.Fmt("Failed to parse marker line: %v", err.Error())) } if target < i { return 1, nil diff --git a/autofile/group_test.go b/autofile/group_test.go index c4f68f057..1a1111961 100644 --- a/autofile/group_test.go +++ b/autofile/group_test.go @@ -175,7 +175,7 @@ func TestSearch(t *testing.T) { if !strings.HasPrefix(line, fmt.Sprintf("INFO %v ", cur)) { t.Fatalf("Unexpected INFO #. Expected %v got:\n%v", cur, line) } - cur += 1 + cur++ } gr.Close() } diff --git a/clist/clist.go b/clist/clist.go index 28d771a28..ccb1f5777 100644 --- a/clist/clist.go +++ b/clist/clist.go @@ -316,7 +316,7 @@ func (l *CList) PushBack(v interface{}) *CElement { l.wg.Done() close(l.waitCh) } - l.len += 1 + l.len++ // Modify the tail if l.tail == nil { @@ -357,7 +357,7 @@ func (l *CList) Remove(e *CElement) interface{} { } // Update l.len - l.len -= 1 + l.len-- // Connect next/prev and set head/tail if prev == nil { diff --git a/clist/clist_test.go b/clist/clist_test.go index 31f821653..6171f1a39 100644 --- a/clist/clist_test.go +++ b/clist/clist_test.go @@ -122,7 +122,7 @@ func _TestGCRandom(t *testing.T) { v.Int = i l.PushBack(v) runtime.SetFinalizer(v, func(v *value) { - gcCount += 1 + gcCount++ }) } @@ -177,10 +177,10 @@ func TestScanRightDeleteRandom(t *testing.T) { } if el == nil { el = l.FrontWait() - restartCounter += 1 + restartCounter++ } el = el.Next() - counter += 1 + counter++ } fmt.Printf("Scanner %v restartCounter: %v counter: %v\n", scannerID, restartCounter, counter) }(i) diff --git a/common/async_test.go b/common/async_test.go index 8d41ec35a..9f060ca2d 100644 --- a/common/async_test.go +++ b/common/async_test.go @@ -33,13 +33,13 @@ func TestParallel(t *testing.T) { taskResult, ok := trs.LatestResult(i) if !ok { assert.Fail(t, "Task #%v did not complete.", i) - failedTasks += 1 + failedTasks++ } else if taskResult.Error != nil { assert.Fail(t, "Task should not have errored but got %v", taskResult.Error) - failedTasks += 1 + failedTasks++ } else if !assert.Equal(t, -1*i, taskResult.Value.(int)) { assert.Fail(t, "Task should have returned %v but got %v", -1*i, taskResult.Value.(int)) - failedTasks += 1 + failedTasks++ } else { // Good! } diff --git a/common/bit_array.go b/common/bit_array.go index a3a87ccab..ea6a6ee1f 100644 --- a/common/bit_array.go +++ b/common/bit_array.go @@ -168,9 +168,8 @@ func (bA *BitArray) Sub(o *BitArray) *BitArray { } } return c - } else { - return bA.and(o.Not()) // Note degenerate case where o == nil } + return bA.and(o.Not()) // Note degenerate case where o == nil } func (bA *BitArray) IsEmpty() bool { diff --git a/common/colors.go b/common/colors.go index 776b22e2e..85e592248 100644 --- a/common/colors.go +++ b/common/colors.go @@ -38,9 +38,8 @@ const ( func treat(s string, color string) string { if len(s) > 2 && s[:2] == "\x1b[" { return s - } else { - return color + s + ANSIReset } + return color + s + ANSIReset } func treatAll(color string, args ...interface{}) string { diff --git a/common/errors.go b/common/errors.go index 52c45c799..c5efae9cf 100644 --- a/common/errors.go +++ b/common/errors.go @@ -13,11 +13,10 @@ func ErrorWrap(cause interface{}, format string, args ...interface{}) Error { msg := Fmt(format, args...) if causeCmnError, ok := cause.(*cmnError); ok { return causeCmnError.TraceFrom(1, msg) - } else { - // NOTE: cause may be nil. - // NOTE: do not use causeCmnError here, not the same as nil. - return newError(msg, cause, cause).Stacktrace() } + // NOTE: cause may be nil. + // NOTE: do not use causeCmnError here, not the same as nil. + return newError(msg, cause, cause).Stacktrace() } //---------------------------------------- diff --git a/common/io.go b/common/io.go index 378c19fc6..fa0443e09 100644 --- a/common/io.go +++ b/common/io.go @@ -20,9 +20,8 @@ func (pr *PrefixedReader) Read(p []byte) (n int, err error) { read := copy(p, pr.Prefix) pr.Prefix = pr.Prefix[read:] return read, nil - } else { - return pr.reader.Read(p) } + return pr.reader.Read(p) } // NOTE: Not goroutine safe diff --git a/common/os.go b/common/os.go index f1e07115c..60a4217aa 100644 --- a/common/os.go +++ b/common/os.go @@ -183,11 +183,10 @@ func Prompt(prompt string, defaultValue string) (string, error) { line, err := reader.ReadString('\n') if err != nil { return defaultValue, err - } else { - line = strings.TrimSpace(line) - if line == "" { - return defaultValue, nil - } - return line, nil } + line = strings.TrimSpace(line) + if line == "" { + return defaultValue, nil + } + return line, nil } diff --git a/common/service.go b/common/service.go index 2502d671c..2f90fa4f9 100644 --- a/common/service.go +++ b/common/service.go @@ -125,9 +125,8 @@ func (bs *BaseService) Start() error { if atomic.LoadUint32(&bs.stopped) == 1 { bs.Logger.Error(Fmt("Not starting %v -- already stopped", bs.name), "impl", bs.impl) return ErrAlreadyStopped - } else { - bs.Logger.Info(Fmt("Starting %v", bs.name), "impl", bs.impl) } + bs.Logger.Info(Fmt("Starting %v", bs.name), "impl", bs.impl) err := bs.impl.OnStart() if err != nil { // revert flag @@ -135,10 +134,9 @@ func (bs *BaseService) Start() error { return err } return nil - } else { - bs.Logger.Debug(Fmt("Not starting %v -- already started", bs.name), "impl", bs.impl) - return ErrAlreadyStarted } + bs.Logger.Debug(Fmt("Not starting %v -- already started", bs.name), "impl", bs.impl) + return ErrAlreadyStarted } // OnStart implements Service by doing nothing. @@ -154,10 +152,9 @@ func (bs *BaseService) Stop() error { bs.impl.OnStop() close(bs.quit) return nil - } else { - bs.Logger.Debug(Fmt("Stopping %v (ignoring: already stopped)", bs.name), "impl", bs.impl) - return ErrAlreadyStopped } + bs.Logger.Debug(Fmt("Stopping %v (ignoring: already stopped)", bs.name), "impl", bs.impl) + return ErrAlreadyStopped } // OnStop implements Service by doing nothing. diff --git a/common/string.go b/common/string.go index dfa262d3f..0e2231e91 100644 --- a/common/string.go +++ b/common/string.go @@ -10,9 +10,8 @@ import ( var Fmt = func(format string, a ...interface{}) string { if len(a) == 0 { return format - } else { - return fmt.Sprintf(format, a...) } + return fmt.Sprintf(format, a...) } // IsHex returns true for non-empty hex-string prefixed with "0x" diff --git a/common/word.go b/common/word.go index 4072482b8..a5b841f55 100644 --- a/common/word.go +++ b/common/word.go @@ -72,9 +72,8 @@ func (tuple Tuple256) Compare(other Tuple256) int { firstCompare := tuple.First.Compare(other.First) if firstCompare == 0 { return tuple.Second.Compare(other.Second) - } else { - return firstCompare } + return firstCompare } func Tuple256Split(t Tuple256) (Word256, Word256) { diff --git a/db/backend_test.go b/db/backend_test.go index 80fbbb140..c407b214f 100644 --- a/db/backend_test.go +++ b/db/backend_test.go @@ -47,7 +47,7 @@ func testBackendGetSetDelete(t *testing.T, backend DBBackendType) { } func TestBackendsGetSetDelete(t *testing.T) { - for dbType, _ := range backends { + for dbType := range backends { testBackendGetSetDelete(t, dbType) } } diff --git a/db/common_test.go b/db/common_test.go index 7f9d10e9b..1d8d52c5f 100644 --- a/db/common_test.go +++ b/db/common_test.go @@ -82,73 +82,73 @@ func (mdb *mockDB) Mutex() *sync.Mutex { } func (mdb *mockDB) Get([]byte) []byte { - mdb.calls["Get"] += 1 + mdb.calls["Get"]++ return nil } func (mdb *mockDB) Has([]byte) bool { - mdb.calls["Has"] += 1 + mdb.calls["Has"]++ return false } func (mdb *mockDB) Set([]byte, []byte) { - mdb.calls["Set"] += 1 + mdb.calls["Set"]++ } func (mdb *mockDB) SetSync([]byte, []byte) { - mdb.calls["SetSync"] += 1 + mdb.calls["SetSync"]++ } func (mdb *mockDB) SetNoLock([]byte, []byte) { - mdb.calls["SetNoLock"] += 1 + mdb.calls["SetNoLock"]++ } func (mdb *mockDB) SetNoLockSync([]byte, []byte) { - mdb.calls["SetNoLockSync"] += 1 + mdb.calls["SetNoLockSync"]++ } func (mdb *mockDB) Delete([]byte) { - mdb.calls["Delete"] += 1 + mdb.calls["Delete"]++ } func (mdb *mockDB) DeleteSync([]byte) { - mdb.calls["DeleteSync"] += 1 + mdb.calls["DeleteSync"]++ } func (mdb *mockDB) DeleteNoLock([]byte) { - mdb.calls["DeleteNoLock"] += 1 + mdb.calls["DeleteNoLock"]++ } func (mdb *mockDB) DeleteNoLockSync([]byte) { - mdb.calls["DeleteNoLockSync"] += 1 + mdb.calls["DeleteNoLockSync"]++ } func (mdb *mockDB) Iterator(start, end []byte) Iterator { - mdb.calls["Iterator"] += 1 + mdb.calls["Iterator"]++ return &mockIterator{} } func (mdb *mockDB) ReverseIterator(start, end []byte) Iterator { - mdb.calls["ReverseIterator"] += 1 + mdb.calls["ReverseIterator"]++ return &mockIterator{} } func (mdb *mockDB) Close() { - mdb.calls["Close"] += 1 + mdb.calls["Close"]++ } func (mdb *mockDB) NewBatch() Batch { - mdb.calls["NewBatch"] += 1 + mdb.calls["NewBatch"]++ return &memBatch{db: mdb} } func (mdb *mockDB) Print() { - mdb.calls["Print"] += 1 + mdb.calls["Print"]++ fmt.Printf("mockDB{%v}", mdb.Stats()) } func (mdb *mockDB) Stats() map[string]string { - mdb.calls["Stats"] += 1 + mdb.calls["Stats"]++ res := make(map[string]string) for key, count := range mdb.calls { @@ -162,24 +162,24 @@ func (mdb *mockDB) Stats() map[string]string { type mockIterator struct{} -func (_ mockIterator) Domain() (start []byte, end []byte) { +func (mockIterator) Domain() (start []byte, end []byte) { return nil, nil } -func (_ mockIterator) Valid() bool { +func (mockIterator) Valid() bool { return false } -func (_ mockIterator) Next() { +func (mockIterator) Next() { } -func (_ mockIterator) Key() []byte { +func (mockIterator) Key() []byte { return nil } -func (_ mockIterator) Value() []byte { +func (mockIterator) Value() []byte { return nil } -func (_ mockIterator) Close() { +func (mockIterator) Close() { } diff --git a/db/db_test.go b/db/db_test.go index 3d6ac38c4..a56901016 100644 --- a/db/db_test.go +++ b/db/db_test.go @@ -8,7 +8,7 @@ import ( ) func TestDBIteratorSingleKey(t *testing.T) { - for backend, _ := range backends { + for backend := range backends { t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { db := newTempDB(t, backend) db.SetSync(bz("1"), bz("value_1")) @@ -26,7 +26,7 @@ func TestDBIteratorSingleKey(t *testing.T) { } func TestDBIteratorTwoKeys(t *testing.T) { - for backend, _ := range backends { + for backend := range backends { t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { db := newTempDB(t, backend) db.SetSync(bz("1"), bz("value_1")) @@ -52,7 +52,7 @@ func TestDBIteratorTwoKeys(t *testing.T) { } func TestDBIteratorMany(t *testing.T) { - for backend, _ := range backends { + for backend := range backends { t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { db := newTempDB(t, backend) @@ -76,7 +76,7 @@ func TestDBIteratorMany(t *testing.T) { } func TestDBIteratorEmpty(t *testing.T) { - for backend, _ := range backends { + for backend := range backends { t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { db := newTempDB(t, backend) itr := db.Iterator(nil, nil) @@ -87,7 +87,7 @@ func TestDBIteratorEmpty(t *testing.T) { } func TestDBIteratorEmptyBeginAfter(t *testing.T) { - for backend, _ := range backends { + for backend := range backends { t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { db := newTempDB(t, backend) itr := db.Iterator(bz("1"), nil) @@ -98,7 +98,7 @@ func TestDBIteratorEmptyBeginAfter(t *testing.T) { } func TestDBIteratorNonemptyBeginAfter(t *testing.T) { - for backend, _ := range backends { + for backend := range backends { t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { db := newTempDB(t, backend) db.SetSync(bz("1"), bz("value_1")) diff --git a/db/go_level_db.go b/db/go_level_db.go index 55ca36c39..9ff162e38 100644 --- a/db/go_level_db.go +++ b/db/go_level_db.go @@ -10,7 +10,7 @@ import ( "github.com/syndtr/goleveldb/leveldb/iterator" "github.com/syndtr/goleveldb/leveldb/opt" - . "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tmlibs/common" ) func init() { @@ -46,9 +46,8 @@ func (db *GoLevelDB) Get(key []byte) []byte { if err != nil { if err == errors.ErrNotFound { return nil - } else { - panic(err) } + panic(err) } return res } @@ -64,7 +63,7 @@ func (db *GoLevelDB) Set(key []byte, value []byte) { value = nonNilBytes(value) err := db.db.Put(key, value, nil) if err != nil { - PanicCrisis(err) + cmn.PanicCrisis(err) } } @@ -74,7 +73,7 @@ func (db *GoLevelDB) SetSync(key []byte, value []byte) { value = nonNilBytes(value) err := db.db.Put(key, value, &opt.WriteOptions{Sync: true}) if err != nil { - PanicCrisis(err) + cmn.PanicCrisis(err) } } @@ -83,7 +82,7 @@ func (db *GoLevelDB) Delete(key []byte) { key = nonNilBytes(key) err := db.db.Delete(key, nil) if err != nil { - PanicCrisis(err) + cmn.PanicCrisis(err) } } @@ -92,7 +91,7 @@ func (db *GoLevelDB) DeleteSync(key []byte) { key = nonNilBytes(key) err := db.db.Delete(key, &opt.WriteOptions{Sync: true}) if err != nil { - PanicCrisis(err) + cmn.PanicCrisis(err) } } diff --git a/db/go_level_db_test.go b/db/go_level_db_test.go index 88b6730f3..266add8b5 100644 --- a/db/go_level_db_test.go +++ b/db/go_level_db_test.go @@ -30,7 +30,7 @@ func BenchmarkRandomReadsWrites(b *testing.B) { // Write something { idx := (int64(cmn.RandInt()) % numItems) - internal[idx] += 1 + internal[idx]++ val := internal[idx] idxBytes := int642Bytes(int64(idx)) valBytes := int642Bytes(int64(val)) diff --git a/db/mem_db.go b/db/mem_db.go index 5439d6789..2d802947c 100644 --- a/db/mem_db.go +++ b/db/mem_db.go @@ -235,7 +235,7 @@ func (itr *memDBIterator) assertIsValid() { func (db *MemDB) getSortedKeys(start, end []byte, reverse bool) []string { keys := []string{} - for key, _ := range db.db { + for key := range db.db { if IsKeyInDomain([]byte(key), start, end, false) { keys = append(keys, key) } diff --git a/db/types.go b/db/types.go index 45146942e..ad78859a7 100644 --- a/db/types.go +++ b/db/types.go @@ -129,7 +129,6 @@ func bz(s string) []byte { func nonNilBytes(bz []byte) []byte { if bz == nil { return []byte{} - } else { - return bz } + return bz } diff --git a/db/util.go b/db/util.go index ecb392dd6..1ad5002d6 100644 --- a/db/util.go +++ b/db/util.go @@ -21,14 +21,13 @@ func cpIncr(bz []byte) (ret []byte) { ret = cp(bz) for i := len(bz) - 1; i >= 0; i-- { if ret[i] < byte(0xFF) { - ret[i] += 1 + ret[i]++ return - } else { - ret[i] = byte(0x00) - if i == 0 { - // Overflow - return nil - } + } + ret[i] = byte(0x00) + if i == 0 { + // Overflow + return nil } } return nil @@ -44,13 +43,12 @@ func IsKeyInDomain(key, start, end []byte, isReverse bool) bool { return false } return true - } else { - if start != nil && bytes.Compare(start, key) < 0 { - return false - } - if end != nil && bytes.Compare(key, end) <= 0 { - return false - } - return true } + if start != nil && bytes.Compare(start, key) < 0 { + return false + } + if end != nil && bytes.Compare(key, end) <= 0 { + return false + } + return true } diff --git a/db/util_test.go b/db/util_test.go index 854448af3..44f1f9f73 100644 --- a/db/util_test.go +++ b/db/util_test.go @@ -7,7 +7,7 @@ import ( // Empty iterator for empty db. func TestPrefixIteratorNoMatchNil(t *testing.T) { - for backend, _ := range backends { + for backend := range backends { t.Run(fmt.Sprintf("Prefix w/ backend %s", backend), func(t *testing.T) { db := newTempDB(t, backend) itr := IteratePrefix(db, []byte("2")) @@ -19,7 +19,7 @@ func TestPrefixIteratorNoMatchNil(t *testing.T) { // Empty iterator for db populated after iterator created. func TestPrefixIteratorNoMatch1(t *testing.T) { - for backend, _ := range backends { + for backend := range backends { t.Run(fmt.Sprintf("Prefix w/ backend %s", backend), func(t *testing.T) { db := newTempDB(t, backend) itr := IteratePrefix(db, []byte("2")) @@ -32,7 +32,7 @@ func TestPrefixIteratorNoMatch1(t *testing.T) { // Empty iterator for prefix starting after db entry. func TestPrefixIteratorNoMatch2(t *testing.T) { - for backend, _ := range backends { + for backend := range backends { t.Run(fmt.Sprintf("Prefix w/ backend %s", backend), func(t *testing.T) { db := newTempDB(t, backend) db.SetSync(bz("3"), bz("value_3")) @@ -45,7 +45,7 @@ func TestPrefixIteratorNoMatch2(t *testing.T) { // Iterator with single val for db with single val, starting from that val. func TestPrefixIteratorMatch1(t *testing.T) { - for backend, _ := range backends { + for backend := range backends { t.Run(fmt.Sprintf("Prefix w/ backend %s", backend), func(t *testing.T) { db := newTempDB(t, backend) db.SetSync(bz("2"), bz("value_2")) @@ -63,7 +63,7 @@ func TestPrefixIteratorMatch1(t *testing.T) { // Iterator with prefix iterates over everything with same prefix. func TestPrefixIteratorMatches1N(t *testing.T) { - for backend, _ := range backends { + for backend := range backends { t.Run(fmt.Sprintf("Prefix w/ backend %s", backend), func(t *testing.T) { db := newTempDB(t, backend) diff --git a/events/events.go b/events/events.go index 3bc349306..f1b2a754e 100644 --- a/events/events.go +++ b/events/events.go @@ -6,7 +6,7 @@ package events import ( "sync" - . "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tmlibs/common" ) // Generic event data can be typed and registered with tendermint/go-amino @@ -27,7 +27,7 @@ type Fireable interface { } type EventSwitch interface { - Service + cmn.Service Fireable AddListenerForEvent(listenerID, event string, cb EventCallback) @@ -36,7 +36,7 @@ type EventSwitch interface { } type eventSwitch struct { - BaseService + cmn.BaseService mtx sync.RWMutex eventCells map[string]*eventCell @@ -45,7 +45,7 @@ type eventSwitch struct { func NewEventSwitch() EventSwitch { evsw := &eventSwitch{} - evsw.BaseService = *NewBaseService(nil, "EventSwitch", evsw) + evsw.BaseService = *cmn.NewBaseService(nil, "EventSwitch", evsw) return evsw } diff --git a/events/events_test.go b/events/events_test.go index 87db2a304..4995ae730 100644 --- a/events/events_test.go +++ b/events/events_test.go @@ -221,11 +221,11 @@ func TestRemoveListener(t *testing.T) { // add some listeners and make sure they work evsw.AddListenerForEvent("listener", "event1", func(data EventData) { - sum1 += 1 + sum1++ }) evsw.AddListenerForEvent("listener", "event2", func(data EventData) { - sum2 += 1 + sum2++ }) for i := 0; i < count; i++ { evsw.FireEvent("event1", true) diff --git a/merkle/simple_proof.go b/merkle/simple_proof.go index 83f89e598..7b8f82b46 100644 --- a/merkle/simple_proof.go +++ b/merkle/simple_proof.go @@ -67,13 +67,12 @@ func computeHashFromAunts(index int, total int, leafHash []byte, innerHashes [][ return nil } return SimpleHashFromTwoHashes(leftHash, innerHashes[len(innerHashes)-1]) - } else { - rightHash := computeHashFromAunts(index-numLeft, total-numLeft, leafHash, innerHashes[:len(innerHashes)-1]) - if rightHash == nil { - return nil - } - return SimpleHashFromTwoHashes(innerHashes[len(innerHashes)-1], rightHash) } + rightHash := computeHashFromAunts(index-numLeft, total-numLeft, leafHash, innerHashes[:len(innerHashes)-1]) + if rightHash == nil { + return nil + } + return SimpleHashFromTwoHashes(innerHashes[len(innerHashes)-1], rightHash) } } diff --git a/test/mutate.go b/test/mutate.go index 1dbe7a6bf..76534e8b1 100644 --- a/test/mutate.go +++ b/test/mutate.go @@ -1,7 +1,7 @@ package test import ( - . "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tmlibs/common" ) // Contract: !bytes.Equal(input, output) && len(input) >= len(output) @@ -17,11 +17,11 @@ func MutateByteSlice(bytez []byte) []byte { bytez = mBytez // Try a random mutation - switch RandInt() % 2 { + switch cmn.RandInt() % 2 { case 0: // Mutate a single byte - bytez[RandInt()%len(bytez)] += byte(RandInt()%255 + 1) + bytez[cmn.RandInt()%len(bytez)] += byte(cmn.RandInt()%255 + 1) case 1: // Remove an arbitrary byte - pos := RandInt() % len(bytez) + pos := cmn.RandInt() % len(bytez) bytez = append(bytez[:pos], bytez[pos+1:]...) } return bytez From b221ca0efa588db6875ee8f35f815319f25a3c96 Mon Sep 17 00:00:00 2001 From: Mohanson Date: Tue, 3 Apr 2018 19:04:09 +0800 Subject: [PATCH 31/40] refine comments --- common/os.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/common/os.go b/common/os.go index 8a0c14f46..450388564 100644 --- a/common/os.go +++ b/common/os.go @@ -149,8 +149,7 @@ func WriteFileAtomic(filename string, data []byte, perm os.FileMode) error { return io.ErrShortWrite } // Close the file before renaming it, otherwise it will cause "The process - // cannot access the file because it is being used by another process." on windows or - // cause "cross-link error" on linux when you try to save it to another partition. + // cannot access the file because it is being used by another process." on windows. f.Close() return os.Rename(f.Name(), filename) From 3375dac049d2e7c597b29485d294b6f30798b4d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Corbi=C3=A8re?= Date: Tue, 3 Apr 2018 14:09:50 +0200 Subject: [PATCH 32/40] add Float64() and RandFloat64(). --- common/random.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/common/random.go b/common/random.go index af531992e..b767e3589 100644 --- a/common/random.go +++ b/common/random.go @@ -113,6 +113,10 @@ func RandFloat32() float32 { return grand.Float32() } +func RandFloat64() float64 { + return grand.Float64() +} + func RandTime() time.Time { return grand.Time() } @@ -272,6 +276,14 @@ func (r *Rand) Float32() float32 { return f32 } +// It is not safe for cryptographic usage. +func (r *Rand) Float64() float64 { + r.Lock() + f64 := r.rand.Float64() + r.Unlock() + return f64 +} + // It is not safe for cryptographic usage. func (r *Rand) Time() time.Time { return time.Unix(int64(r.Uint64Exp()), 0) From 3cd4dcf13ba5ba6b4b3a8d224bda5ed3d3043c04 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 3 Apr 2018 16:23:41 +0300 Subject: [PATCH 33/40] changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d6d8914a..70061a409 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,11 +16,13 @@ FEATURES: - [db] NewPrefixDB for a DB with all keys prefixed - [db] NewDebugDB prints everything during operation - [common] SplitAndTrim func + - [common] RandFloat64() and rand.Float64() BUG FIXES: - [pubsub] Fix unsubscribing - [cli] Return config errors + - [common] Fix WriteFileAtomic Windows bug ## 0.7.1 (March 22, 2018) From f457435199f4e38337b3e7c90dc4384ca928e09a Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Tue, 3 Apr 2018 07:01:08 -0700 Subject: [PATCH 34/40] HexBytes formatting; Make computeHashFromAunts more defensive --- common/bytes.go | 9 +++++++++ merkle/simple_proof.go | 6 +++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/common/bytes.go b/common/bytes.go index ba81bbe97..711720aa7 100644 --- a/common/bytes.go +++ b/common/bytes.go @@ -51,3 +51,12 @@ func (bz HexBytes) Bytes() []byte { func (bz HexBytes) String() string { return strings.ToUpper(hex.EncodeToString(bz)) } + +func (bz HexBytes) Format(s fmt.State, verb rune) { + switch verb { + case 'p': + s.Write([]byte(fmt.Sprintf("%p", bz))) + default: + s.Write([]byte(fmt.Sprintf("%X", []byte(bz)))) + } +} diff --git a/merkle/simple_proof.go b/merkle/simple_proof.go index 7b8f82b46..c81ed674a 100644 --- a/merkle/simple_proof.go +++ b/merkle/simple_proof.go @@ -43,9 +43,9 @@ func (sp *SimpleProof) StringIndented(indent string) string { // Use the leafHash and innerHashes to get the root merkle hash. // If the length of the innerHashes slice isn't exactly correct, the result is nil. +// Recursive impl. func computeHashFromAunts(index int, total int, leafHash []byte, innerHashes [][]byte) []byte { - // Recursive impl. - if index >= total { + if index >= total || index < 0 || total <= 0 { return nil } switch total { @@ -80,7 +80,7 @@ func computeHashFromAunts(index int, total int, leafHash []byte, innerHashes [][ // The node and the tree is thrown away afterwards. // Exactly one of node.Left and node.Right is nil, unless node is the root, in which case both are nil. // node.Parent.Hash = hash(node.Hash, node.Right.Hash) or -// hash(node.Left.Hash, node.Hash), depending on whether node is a left/right child. +// hash(node.Left.Hash, node.Hash), depending on whether node is a left/right child. type SimpleProofNode struct { Hash []byte Parent *SimpleProofNode From d66d43d2eaa5d2611ed1e2814b1df7bf01df705a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Corbi=C3=A8re?= Date: Tue, 3 Apr 2018 16:23:36 +0200 Subject: [PATCH 35/40] Add Int31n() and RandInt31n(). --- common/random.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/common/random.go b/common/random.go index b767e3589..5653142eb 100644 --- a/common/random.go +++ b/common/random.go @@ -93,6 +93,10 @@ func RandInt31() int32 { return grand.Int31() } +func RandInt31n(n int32) int32 { + return grand.Int31n(n) +} + func RandInt63() int64 { return grand.Int63() } @@ -224,6 +228,14 @@ func (r *Rand) Int31() int32 { return i31 } +// It is not safe for cryptographic usage. +func (r *Rand) Int31n(n int32) int32 { + r.Lock() + i31n := r.rand.Int31n(n) + r.Unlock() + return i31n +} + // It is not safe for cryptographic usage. func (r *Rand) Int63() int64 { r.Lock() From 74486f7f932a7ec2c54f55e2b0319c621b2d5bdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Corbi=C3=A8re?= Date: Tue, 3 Apr 2018 16:24:38 +0200 Subject: [PATCH 36/40] Add Int63n() and RandInt63n(). --- common/random.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/common/random.go b/common/random.go index 5653142eb..389a32fc2 100644 --- a/common/random.go +++ b/common/random.go @@ -101,6 +101,10 @@ func RandInt63() int64 { return grand.Int63() } +func RandInt63n(n int64) int64 { + return grand.Int63n(n) +} + func RandUint16Exp() uint16 { return grand.Uint16Exp() } @@ -244,6 +248,14 @@ func (r *Rand) Int63() int64 { return i63 } +// It is not safe for cryptographic usage. +func (r *Rand) Int63n(n int64) int64 { + r.Lock() + i63n := r.rand.Int63n(n) + r.Unlock() + return i63n +} + // Distributed pseudo-exponentially to test for various cases // It is not safe for cryptographic usage. func (r *Rand) Uint16Exp() uint16 { From 390de81bbc41cd27cabf70d1d172e1c2ef890106 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Wed, 4 Apr 2018 12:30:13 -0700 Subject: [PATCH 37/40] Release version 0.8.0 --- CHANGELOG.md | 5 +++-- version/version.go | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70061a409..7f372d028 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 0.8.0 (TBD) +## 0.8.0 (develop branch) BREAKING: @@ -16,7 +16,8 @@ FEATURES: - [db] NewPrefixDB for a DB with all keys prefixed - [db] NewDebugDB prints everything during operation - [common] SplitAndTrim func - - [common] RandFloat64() and rand.Float64() + - [common] rand.Float64(), rand.Int63n(n), rand.Int31n(n) and global equivalents + - [common] HexBytes Format() BUG FIXES: diff --git a/version/version.go b/version/version.go index c683dd245..f17ed2ed3 100644 --- a/version/version.go +++ b/version/version.go @@ -1,3 +1,3 @@ package version -const Version = "0.8.0-dev" +const Version = "0.8.0" From fb7bde9c24b3c9d7c5e9926606760cb36ada3e8b Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Wed, 4 Apr 2018 13:43:19 -0700 Subject: [PATCH 38/40] Add cause in Error.Error() --- CHANGELOG.md | 6 ++++++ common/errors.go | 6 +++++- common/errors_test.go | 4 ++-- version/version.go | 2 +- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f372d028..0da5ee204 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 0.8.1 (develop branch) + +FEATURES: + + - [common] Error.Error() includes cause + ## 0.8.0 (develop branch) BREAKING: diff --git a/common/errors.go b/common/errors.go index c5efae9cf..1ee1fb349 100644 --- a/common/errors.go +++ b/common/errors.go @@ -200,7 +200,11 @@ func (err *cmnError) Format(s fmt.State, verb rune) { s.Write([]byte("--= /Error =--\n")) } else { // Write msg. - s.Write([]byte(fmt.Sprintf("Error{`%v`}", err.msg))) // TODO tick-esc? + if err.cause != nil { + s.Write([]byte(fmt.Sprintf("Error{`%s` (cause: %v)}", err.msg, err.cause))) // TODO tick-esc? + } else { + s.Write([]byte(fmt.Sprintf("Error{`%s`}", err.msg))) // TODO tick-esc? + } } } } diff --git a/common/errors_test.go b/common/errors_test.go index 56f366179..2c5234f9f 100644 --- a/common/errors_test.go +++ b/common/errors_test.go @@ -28,7 +28,7 @@ func TestErrorPanic(t *testing.T) { assert.Equal(t, pnk{"something"}, err.Cause()) assert.Equal(t, pnk{"something"}, err.T()) assert.Equal(t, "This is the message in ErrorWrap(r, message).", err.Message()) - assert.Equal(t, "Error{`This is the message in ErrorWrap(r, message).`}", fmt.Sprintf("%v", err)) + assert.Equal(t, "Error{`This is the message in ErrorWrap(r, message).` (cause: {something})}", fmt.Sprintf("%v", err)) assert.Contains(t, fmt.Sprintf("%#v", err), "Message: This is the message in ErrorWrap(r, message).") assert.Contains(t, fmt.Sprintf("%#v", err), "Stack Trace:\n 0") } @@ -40,7 +40,7 @@ func TestErrorWrapSomething(t *testing.T) { assert.Equal(t, "something", err.Cause()) assert.Equal(t, "something", err.T()) assert.Equal(t, "formatter01", err.Message()) - assert.Equal(t, "Error{`formatter01`}", fmt.Sprintf("%v", err)) + assert.Equal(t, "Error{`formatter01` (cause: something)}", fmt.Sprintf("%v", err)) assert.Regexp(t, `Message: formatter01\n`, fmt.Sprintf("%#v", err)) assert.Contains(t, fmt.Sprintf("%#v", err), "Stack Trace:\n 0") } diff --git a/version/version.go b/version/version.go index f17ed2ed3..b389a63a0 100644 --- a/version/version.go +++ b/version/version.go @@ -1,3 +1,3 @@ package version -const Version = "0.8.0" +const Version = "0.8.1" From 2e24b64fc121dcdf1cabceab8dc2f7257675483c Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Thu, 5 Apr 2018 03:12:21 -0700 Subject: [PATCH 39/40] Add IsEmpty; Publish 0.8.1 --- CHANGELOG.md | 1 + common/nil.go | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0da5ee204..b93066f96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ FEATURES: - [common] Error.Error() includes cause + - [common] IsEmpty() for 0 length ## 0.8.0 (develop branch) diff --git a/common/nil.go b/common/nil.go index c7617f083..31f75f008 100644 --- a/common/nil.go +++ b/common/nil.go @@ -16,3 +16,14 @@ func IsTypedNil(o interface{}) bool { return false } } + +// Returns true if it has zero length. +func IsEmpty(o interface{}) bool { + rv := reflect.ValueOf(o) + switch rv.Kind() { + case reflect.Array, reflect.Chan, reflect.Map, reflect.Slice, reflect.String: + return rv.Len() == 0 + default: + return false + } +} From 0f92a017377bb5fb243d77f2a3f53930895ee7e1 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 9 Apr 2018 15:51:54 +0300 Subject: [PATCH 40/40] changelog dates --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b93066f96..7a55fb554 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,13 @@ # Changelog -## 0.8.1 (develop branch) +## 0.8.1 (April 5th, 2018) FEATURES: - [common] Error.Error() includes cause - [common] IsEmpty() for 0 length -## 0.8.0 (develop branch) +## 0.8.0 (April 4th, 2018) BREAKING: