From df25a22469f1a14586c3227b4b0c6c1608fec372 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 5 Mar 2016 20:59:51 -0500 Subject: [PATCH 01/19] rpc: num_unconfirmed_txs (avoid sending txs back) --- rpc/core/mempool.go | 5 +++++ rpc/core/routes.go | 9 +++++++++ 2 files changed, 14 insertions(+) diff --git a/rpc/core/mempool.go b/rpc/core/mempool.go index e5a34026c..e10c72660 100644 --- a/rpc/core/mempool.go +++ b/rpc/core/mempool.go @@ -39,3 +39,8 @@ func UnconfirmedTxs() (*ctypes.ResultUnconfirmedTxs, error) { txs := mempoolReactor.Mempool.Reap(0) return &ctypes.ResultUnconfirmedTxs{len(txs), txs}, nil } + +func NumUnconfirmedTxs() (*ctypes.ResultUnconfirmedTxs, error) { + txs := mempoolReactor.Mempool.Reap() + return &ctypes.ResultUnconfirmedTxs{len(txs), txs[:0]}, nil +} diff --git a/rpc/core/routes.go b/rpc/core/routes.go index 53604c0fc..5a4987da5 100644 --- a/rpc/core/routes.go +++ b/rpc/core/routes.go @@ -22,6 +22,7 @@ var Routes = map[string]*rpc.RPCFunc{ "broadcast_tx_sync": rpc.NewRPCFunc(BroadcastTxSyncResult, "tx"), "broadcast_tx_async": rpc.NewRPCFunc(BroadcastTxAsyncResult, "tx"), "unconfirmed_txs": rpc.NewRPCFunc(UnconfirmedTxsResult, ""), + "num_unconfirmed_txs": rpc.NewRPCFunc(NumUnconfirmedTxsResult, ""), "unsafe_set_config": rpc.NewRPCFunc(UnsafeSetConfigResult, "type,key,value"), } @@ -114,6 +115,14 @@ func UnconfirmedTxsResult() (ctypes.TMResult, error) { } } +func NumUnconfirmedTxsResult() (ctypes.TMResult, error) { + if r, err := NumUnconfirmedTxs(); err != nil { + return nil, err + } else { + return r, nil + } +} + func BroadcastTxSyncResult(tx []byte) (ctypes.TMResult, error) { if r, err := BroadcastTxSync(tx); err != nil { return nil, err From 2215ad74ff4b9c881e9e474e692604f0fe2a0c05 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 5 Mar 2016 21:17:21 -0500 Subject: [PATCH 02/19] consensus: cswal doesnt write any consensus msgs in light mode --- consensus/wal.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/consensus/wal.go b/consensus/wal.go index 56bb294e4..b5c79993d 100644 --- a/consensus/wal.go +++ b/consensus/wal.go @@ -61,21 +61,20 @@ func NewWAL(file string, light bool) (*WAL, error) { } // called in newStep and for each pass in receiveRoutine -func (wal *WAL) Save(msg ConsensusLogMessageInterface) { +func (wal *WAL) Save(clm ConsensusLogMessageInterface) { if wal != nil { if wal.light { - if m, ok := msg.(msgInfo); ok { - if _, ok := m.Msg.(*BlockPartMessage); ok { - return - } + // in light mode we only write new steps and timeouts (no votes, proposals, block parts) + if _, ok := clm.(msgInfo); ok { + return } } var n int var err error - wire.WriteJSON(ConsensusLogMessage{time.Now(), msg}, wal.fp, &n, &err) + wire.WriteJSON(ConsensusLogMessage{time.Now(), clm}, wal.fp, &n, &err) wire.WriteTo([]byte("\n"), wal.fp, &n, &err) // one message per line if err != nil { - PanicQ(Fmt("Error writing msg to consensus wal. Error: %v \n\nMessage: %v", err, msg)) + PanicQ(Fmt("Error writing msg to consensus wal. Error: %v \n\nMessage: %v", err, clm)) } } } From f5be3cd8f933d5dd7a3e62f875ebf5758dee3877 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 7 Mar 2016 18:38:05 -0500 Subject: [PATCH 03/19] rpc: start/stop cpu profiler --- mempool/mempool.go | 4 +++ rpc/core/dev.go | 56 +++++++++++++++++++++++++++++++++++++ rpc/core/mempool.go | 3 +- rpc/core/routes.go | 20 ++++++++++++- rpc/core/status.go | 28 ------------------- rpc/core/types/responses.go | 8 +++++- 6 files changed, 87 insertions(+), 32 deletions(-) create mode 100644 rpc/core/dev.go diff --git a/mempool/mempool.go b/mempool/mempool.go index 0744880a1..46e71f4e8 100644 --- a/mempool/mempool.go +++ b/mempool/mempool.go @@ -77,6 +77,10 @@ func NewMempool(proxyAppConn proxy.AppConn) *Mempool { return mempool } +func (mem *Mempool) Size() int { + return mem.txs.Len() +} + // Return the first element of mem.txs for peer goroutines to call .NextWait() on. // Blocks until txs has elements. func (mem *Mempool) TxsFrontWait() *clist.CElement { diff --git a/rpc/core/dev.go b/rpc/core/dev.go new file mode 100644 index 000000000..a7a5c54df --- /dev/null +++ b/rpc/core/dev.go @@ -0,0 +1,56 @@ +package core + +import ( + "fmt" + "os" + "runtime/pprof" + "strconv" + + ctypes "github.com/tendermint/tendermint/rpc/core/types" +) + +func UnsafeSetConfig(typ, key, value string) (*ctypes.ResultUnsafeSetConfig, error) { + switch typ { + case "string": + config.Set(key, value) + case "int": + val, err := strconv.Atoi(value) + if err != nil { + return nil, fmt.Errorf("non-integer value found. key:%s; value:%s; err:%v", key, value, err) + } + config.Set(key, val) + case "bool": + switch value { + case "true": + config.Set(key, true) + case "false": + config.Set(key, false) + default: + return nil, fmt.Errorf("bool value must be true or false. got %s", value) + } + default: + return nil, fmt.Errorf("Unknown type %s", typ) + } + return &ctypes.ResultUnsafeSetConfig{}, nil +} + +var profFile *os.File + +func UnsafeStartCPUProfiler(filename string) (*ctypes.ResultUnsafeCPUProfiler, error) { + var err error + profFile, err = os.Create(filename) + if err != nil { + return nil, err + } + err = pprof.StartCPUProfile(profFile) + if err != nil { + return nil, err + } + return &ctypes.ResultUnsafeCPUProfiler{}, nil +} + +func UnsafeStopCPUProfiler() (*ctypes.ResultUnsafeCPUProfiler, error) { + pprof.StopCPUProfile() + profFile.Close() + return &ctypes.ResultUnsafeCPUProfiler{}, nil +} diff --git a/rpc/core/mempool.go b/rpc/core/mempool.go index e10c72660..d06fca8b2 100644 --- a/rpc/core/mempool.go +++ b/rpc/core/mempool.go @@ -41,6 +41,5 @@ func UnconfirmedTxs() (*ctypes.ResultUnconfirmedTxs, error) { } func NumUnconfirmedTxs() (*ctypes.ResultUnconfirmedTxs, error) { - txs := mempoolReactor.Mempool.Reap() - return &ctypes.ResultUnconfirmedTxs{len(txs), txs[:0]}, nil + return &ctypes.ResultUnconfirmedTxs{N: mempoolReactor.Mempool.Size()}, nil } diff --git a/rpc/core/routes.go b/rpc/core/routes.go index 5a4987da5..b642ad7fe 100644 --- a/rpc/core/routes.go +++ b/rpc/core/routes.go @@ -24,7 +24,9 @@ var Routes = map[string]*rpc.RPCFunc{ "unconfirmed_txs": rpc.NewRPCFunc(UnconfirmedTxsResult, ""), "num_unconfirmed_txs": rpc.NewRPCFunc(NumUnconfirmedTxsResult, ""), - "unsafe_set_config": rpc.NewRPCFunc(UnsafeSetConfigResult, "type,key,value"), + "unsafe_set_config": rpc.NewRPCFunc(UnsafeSetConfigResult, "type,key,value"), + "unsafe_start_cpu_profiler": rpc.NewRPCFunc(UnsafeStartCPUProfilerResult, "filename"), + "unsafe_stop_cpu_profiler": rpc.NewRPCFunc(UnsafeStopCPUProfilerResult, ""), } func SubscribeResult(wsCtx rpctypes.WSRPCContext, event string) (ctypes.TMResult, error) { @@ -146,3 +148,19 @@ func UnsafeSetConfigResult(typ, key, value string) (ctypes.TMResult, error) { return r, nil } } + +func UnsafeStartCPUProfilerResult(filename string) (ctypes.TMResult, error) { + if r, err := UnsafeStartCPUProfiler(filename); err != nil { + return nil, err + } else { + return r, nil + } +} + +func UnsafeStopCPUProfilerResult() (ctypes.TMResult, error) { + if r, err := UnsafeStopCPUProfiler(); err != nil { + return nil, err + } else { + return r, nil + } +} diff --git a/rpc/core/status.go b/rpc/core/status.go index 8f056fd4e..bf3d69ffe 100644 --- a/rpc/core/status.go +++ b/rpc/core/status.go @@ -1,9 +1,6 @@ package core import ( - "fmt" - "strconv" - ctypes "github.com/tendermint/tendermint/rpc/core/types" "github.com/tendermint/tendermint/types" ) @@ -31,28 +28,3 @@ func Status() (*ctypes.ResultStatus, error) { LatestBlockHeight: latestHeight, LatestBlockTime: latestBlockTime}, nil } - -func UnsafeSetConfig(typ, key, value string) (*ctypes.ResultUnsafeSetConfig, error) { - switch typ { - case "string": - config.Set(key, value) - case "int": - val, err := strconv.Atoi(value) - if err != nil { - return nil, fmt.Errorf("non-integer value found. key:%s; value:%s; err:%v", key, value, err) - } - config.Set(key, val) - case "bool": - switch value { - case "true": - config.Set(key, true) - case "false": - config.Set(key, false) - default: - return nil, fmt.Errorf("bool value must be true or false. got %s", value) - } - default: - return nil, fmt.Errorf("Unknown type %s", typ) - } - return &ctypes.ResultUnsafeSetConfig{}, nil -} diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index 660c6e0c1..ee7307a26 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -70,6 +70,8 @@ type ResultUnconfirmedTxs struct { type ResultUnsafeSetConfig struct{} +type ResultUnsafeCPUProfiler struct{} + type ResultSubscribe struct { } @@ -109,7 +111,9 @@ const ( ResultTypeEvent = byte(0x82) // 0xa bytes for testing - ResultTypeUnsafeSetConfig = byte(0xa0) + ResultTypeUnsafeSetConfig = byte(0xa0) + ResultTypeUnsafeStartCPUProfiler = byte(0xa1) + ResultTypeUnsafeStopCPUProfiler = byte(0xa2) ) type TMResult interface { @@ -133,4 +137,6 @@ var _ = wire.RegisterInterface( wire.ConcreteType{&ResultUnsubscribe{}, ResultTypeUnsubscribe}, wire.ConcreteType{&ResultEvent{}, ResultTypeEvent}, wire.ConcreteType{&ResultUnsafeSetConfig{}, ResultTypeUnsafeSetConfig}, + wire.ConcreteType{&ResultUnsafeCPUProfiler{}, ResultTypeUnsafeStartCPUProfiler}, + wire.ConcreteType{&ResultUnsafeCPUProfiler{}, ResultTypeUnsafeStopCPUProfiler}, ) From 196af01707a5f9193367a32091bd5cb602fe4b6c Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 8 Mar 2016 00:43:39 -0500 Subject: [PATCH 04/19] maxTxs=-1 for Reap to return empty --- mempool/mempool.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mempool/mempool.go b/mempool/mempool.go index 46e71f4e8..0fab674bf 100644 --- a/mempool/mempool.go +++ b/mempool/mempool.go @@ -201,10 +201,12 @@ func (mem *Mempool) Reap(maxTxs int) []types.Tx { return txs } -// maxTxs: 0 means uncapped +// maxTxs: 0 means uncapped, -1 means none func (mem *Mempool) collectTxs(maxTxs int) []types.Tx { if maxTxs == 0 { maxTxs = mem.txs.Len() + } else if maxTxs < 0 { + return []types.Tx{} } txs := make([]types.Tx, 0, MinInt(mem.txs.Len(), maxTxs)) for e := mem.txs.Front(); e != nil && len(txs) < maxTxs; e = e.Next() { From 898f44ddda29ab1c3d19462a87ea79e1caac3cc9 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 10 Mar 2016 15:50:35 -0500 Subject: [PATCH 05/19] config: disable_data_hash (for testing) --- config/tendermint/config.go | 1 + config/tendermint_test/config.go | 1 + types/block.go | 4 ++++ 3 files changed, 6 insertions(+) diff --git a/config/tendermint/config.go b/config/tendermint/config.go index 6378607fe..74d7aefeb 100644 --- a/config/tendermint/config.go +++ b/config/tendermint/config.go @@ -70,6 +70,7 @@ func GetConfig(rootDir string) cfg.Config { mapConfig.SetDefault("cswal_light", false) mapConfig.SetDefault("block_size", 10000) + mapConfig.SetDefault("disable_data_hash", false) mapConfig.SetDefault("timeout_propose", 3000) mapConfig.SetDefault("timeout_propose_delta", 500) mapConfig.SetDefault("timeout_prevote", 1000) diff --git a/config/tendermint_test/config.go b/config/tendermint_test/config.go index b90fe2549..c2690ae90 100644 --- a/config/tendermint_test/config.go +++ b/config/tendermint_test/config.go @@ -88,6 +88,7 @@ func GetConfig(rootDir string) cfg.Config { mapConfig.SetDefault("cswal_light", false) mapConfig.SetDefault("block_size", 10000) + mapConfig.SetDefault("disable_data_hash", false) mapConfig.SetDefault("timeout_propose", 100) mapConfig.SetDefault("timeout_propose_delta", 1) mapConfig.SetDefault("timeout_prevote", 1) diff --git a/types/block.go b/types/block.go index 1e197c621..f2d1ef578 100644 --- a/types/block.go +++ b/types/block.go @@ -336,6 +336,10 @@ type Data struct { } func (data *Data) Hash() []byte { + if config.GetBool("disable_data_hash") { + data.hash = []byte{} + return data.hash + } if data.hash == nil { txs := make([]interface{}, len(data.Txs)) for i, tx := range data.Txs { From 5e42c962673609097b528439ff26c11c2999c1e8 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 10 Mar 2016 20:20:07 -0500 Subject: [PATCH 06/19] crank the part size; prioritize data channel --- consensus/reactor.go | 6 +++--- consensus/state.go | 4 ++-- types/block.go | 1 + types/part_set.go | 3 ++- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/consensus/reactor.go b/consensus/reactor.go index c2874df25..b97a45a2d 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -91,9 +91,9 @@ func (conR *ConsensusReactor) GetChannels() []*p2p.ChannelDescriptor { SendQueueCapacity: 100, }, &p2p.ChannelDescriptor{ - ID: DataChannel, - Priority: 2, - SendQueueCapacity: 50, + ID: DataChannel, // maybe split between gossiping current block and catchup stuff + Priority: 200, // once we gossip the whole block there's nothing left to send until next height or round + SendQueueCapacity: 100, RecvBufferCapacity: 50 * 4096, }, &p2p.ChannelDescriptor{ diff --git a/consensus/state.go b/consensus/state.go index aafef7ec0..7dcb6eb0b 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -835,8 +835,8 @@ func (cs *ConsensusState) decideProposal(height, round int) { part := blockParts.GetPart(i) cs.sendInternalMessage(msgInfo{&BlockPartMessage{cs.Height, cs.Round, part}, ""}) } - log.Info("Signed and sent proposal", "height", height, "round", round, "proposal", proposal) - log.Debug(Fmt("Signed and sent proposal block: %v", block)) + log.Info("Signed proposal", "height", height, "round", round, "proposal", proposal) + log.Debug(Fmt("Signed proposal block: %v", block)) } else { log.Warn("enterPropose: Error signing proposal", "height", height, "round", round, "error", err) } diff --git a/types/block.go b/types/block.go index f2d1ef578..739a61b1f 100644 --- a/types/block.go +++ b/types/block.go @@ -337,6 +337,7 @@ type Data struct { func (data *Data) Hash() []byte { if config.GetBool("disable_data_hash") { + // we could use the part_set hash instead data.hash = []byte{} return data.hash } diff --git a/types/part_set.go b/types/part_set.go index 5940a85d8..bb7d11b16 100644 --- a/types/part_set.go +++ b/types/part_set.go @@ -15,7 +15,7 @@ import ( ) const ( - partSize = 4096 // 4KB + partSize = 65536 // 64KB ... 4096 // 4KB ) var ( @@ -203,6 +203,7 @@ func (ps *PartSet) AddPart(part *Part) (bool, error) { } // Check hash proof + // TODO: minor gains for not checking part sets we made if !part.Proof.Verify(part.Index, ps.total, part.Hash(), ps.Hash()) { return false, ErrPartSetInvalidProof } From 79c9a9f03a4e5186cf1a54bc5fa03f22c71ea0e1 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 11 Mar 2016 21:38:15 -0500 Subject: [PATCH 07/19] don't verify own block parts --- consensus/state.go | 6 +++--- types/part_set.go | 9 +++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/consensus/state.go b/consensus/state.go index 7dcb6eb0b..6a0c54bf7 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -655,7 +655,7 @@ func (cs *ConsensusState) handleMsg(mi msgInfo, rs RoundState) { err = cs.setProposal(msg.Proposal) case *BlockPartMessage: // if the proposal is complete, we'll enterPrevote or tryFinalizeCommit - _, err = cs.addProposalBlockPart(msg.Height, msg.Part) + _, err = cs.addProposalBlockPart(msg.Height, msg.Part, peerKey != "") if err != nil && msg.Round != cs.Round { err = nil } @@ -1291,7 +1291,7 @@ func (cs *ConsensusState) setProposal(proposal *types.Proposal) error { // NOTE: block is not necessarily valid. // Asynchronously triggers either enterPrevote (before we timeout of propose) or tryFinalizeCommit, once we have the full block. -func (cs *ConsensusState) addProposalBlockPart(height int, part *types.Part) (added bool, err error) { +func (cs *ConsensusState) addProposalBlockPart(height int, part *types.Part, verify bool) (added bool, err error) { // Blocks might be reused, so round mismatch is OK if cs.Height != height { return false, nil @@ -1302,7 +1302,7 @@ func (cs *ConsensusState) addProposalBlockPart(height int, part *types.Part) (ad return false, nil // TODO: bad peer? Return error? } - added, err = cs.ProposalBlockParts.AddPart(part) + added, err = cs.ProposalBlockParts.AddPart(part, verify) if err != nil { return added, err } diff --git a/types/part_set.go b/types/part_set.go index bb7d11b16..06314fe1d 100644 --- a/types/part_set.go +++ b/types/part_set.go @@ -188,7 +188,7 @@ func (ps *PartSet) Total() int { return ps.total } -func (ps *PartSet) AddPart(part *Part) (bool, error) { +func (ps *PartSet) AddPart(part *Part, verify bool) (bool, error) { ps.mtx.Lock() defer ps.mtx.Unlock() @@ -203,9 +203,10 @@ func (ps *PartSet) AddPart(part *Part) (bool, error) { } // Check hash proof - // TODO: minor gains for not checking part sets we made - if !part.Proof.Verify(part.Index, ps.total, part.Hash(), ps.Hash()) { - return false, ErrPartSetInvalidProof + if verify { + if !part.Proof.Verify(part.Index, ps.total, part.Hash(), ps.Hash()) { + return false, ErrPartSetInvalidProof + } } // Add part From cb80a7aa42fa880ddb5c1d2d769ec0ea53742fe1 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 11 Mar 2016 21:45:19 -0500 Subject: [PATCH 08/19] rpc: unsafe_write_heap_profile --- rpc/core/dev.go | 19 +++++++++++++++---- rpc/core/routes.go | 9 +++++++++ rpc/core/types/responses.go | 8 +++++--- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/rpc/core/dev.go b/rpc/core/dev.go index a7a5c54df..2b8dfb8f3 100644 --- a/rpc/core/dev.go +++ b/rpc/core/dev.go @@ -36,7 +36,7 @@ func UnsafeSetConfig(typ, key, value string) (*ctypes.ResultUnsafeSetConfig, err var profFile *os.File -func UnsafeStartCPUProfiler(filename string) (*ctypes.ResultUnsafeCPUProfiler, error) { +func UnsafeStartCPUProfiler(filename string) (*ctypes.ResultUnsafeProfile, error) { var err error profFile, err = os.Create(filename) if err != nil { @@ -46,11 +46,22 @@ func UnsafeStartCPUProfiler(filename string) (*ctypes.ResultUnsafeCPUProfiler, e if err != nil { return nil, err } - return &ctypes.ResultUnsafeCPUProfiler{}, nil + return &ctypes.ResultUnsafeProfile{}, nil } -func UnsafeStopCPUProfiler() (*ctypes.ResultUnsafeCPUProfiler, error) { +func UnsafeStopCPUProfiler() (*ctypes.ResultUnsafeProfile, error) { pprof.StopCPUProfile() profFile.Close() - return &ctypes.ResultUnsafeCPUProfiler{}, nil + return &ctypes.ResultUnsafeProfile{}, nil +} + +func UnsafeWriteHeapProfile(filename string) (*ctypes.ResultUnsafeProfile, error) { + memProfFile, err := os.Create(filename) + if err != nil { + return nil, err + } + pprof.WriteHeapProfile(memProfFile) + memProfFile.Close() + + return &ctypes.ResultUnsafeProfile{}, nil } diff --git a/rpc/core/routes.go b/rpc/core/routes.go index b642ad7fe..585a18cd2 100644 --- a/rpc/core/routes.go +++ b/rpc/core/routes.go @@ -27,6 +27,7 @@ var Routes = map[string]*rpc.RPCFunc{ "unsafe_set_config": rpc.NewRPCFunc(UnsafeSetConfigResult, "type,key,value"), "unsafe_start_cpu_profiler": rpc.NewRPCFunc(UnsafeStartCPUProfilerResult, "filename"), "unsafe_stop_cpu_profiler": rpc.NewRPCFunc(UnsafeStopCPUProfilerResult, ""), + "unsafe_write_heap_profile": rpc.NewRPCFunc(UnsafeWriteHeapProfileResult, "filename"), } func SubscribeResult(wsCtx rpctypes.WSRPCContext, event string) (ctypes.TMResult, error) { @@ -164,3 +165,11 @@ func UnsafeStopCPUProfilerResult() (ctypes.TMResult, error) { return r, nil } } + +func UnsafeWriteHeapProfileResult(filename string) (ctypes.TMResult, error) { + if r, err := UnsafeWriteHeapProfile(filename); err != nil { + return nil, err + } else { + return r, nil + } +} diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index ee7307a26..d08892e8c 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -70,7 +70,7 @@ type ResultUnconfirmedTxs struct { type ResultUnsafeSetConfig struct{} -type ResultUnsafeCPUProfiler struct{} +type ResultUnsafeProfile struct{} type ResultSubscribe struct { } @@ -114,6 +114,7 @@ const ( ResultTypeUnsafeSetConfig = byte(0xa0) ResultTypeUnsafeStartCPUProfiler = byte(0xa1) ResultTypeUnsafeStopCPUProfiler = byte(0xa2) + ResultTypeUnsafeWriteHeapProfile = byte(0xa3) ) type TMResult interface { @@ -137,6 +138,7 @@ var _ = wire.RegisterInterface( wire.ConcreteType{&ResultUnsubscribe{}, ResultTypeUnsubscribe}, wire.ConcreteType{&ResultEvent{}, ResultTypeEvent}, wire.ConcreteType{&ResultUnsafeSetConfig{}, ResultTypeUnsafeSetConfig}, - wire.ConcreteType{&ResultUnsafeCPUProfiler{}, ResultTypeUnsafeStartCPUProfiler}, - wire.ConcreteType{&ResultUnsafeCPUProfiler{}, ResultTypeUnsafeStopCPUProfiler}, + wire.ConcreteType{&ResultUnsafeProfile{}, ResultTypeUnsafeStartCPUProfiler}, + wire.ConcreteType{&ResultUnsafeProfile{}, ResultTypeUnsafeStopCPUProfiler}, + wire.ConcreteType{&ResultUnsafeProfile{}, ResultTypeUnsafeWriteHeapProfile}, ) From a34f9e082ef6430ade6f6ef7c61997169749759b Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 11 Mar 2016 22:49:43 -0500 Subject: [PATCH 09/19] lazy part_set reader --- types/part_set.go | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/types/part_set.go b/types/part_set.go index 06314fe1d..146e23c43 100644 --- a/types/part_set.go +++ b/types/part_set.go @@ -230,6 +230,8 @@ func (ps *PartSet) GetReader() io.Reader { if !ps.IsComplete() { PanicSanity("Cannot GetReader() on incomplete PartSet") } + return NewPartSetReader(ps.parts) + buf := []byte{} for _, part := range ps.parts { buf = append(buf, part.Bytes...) @@ -237,6 +239,41 @@ func (ps *PartSet) GetReader() io.Reader { return bytes.NewReader(buf) } +type PartSetReader struct { + i int + parts []*Part + reader *bytes.Reader +} + +func NewPartSetReader(parts []*Part) *PartSetReader { + return &PartSetReader{ + i: 0, + parts: parts, + reader: bytes.NewReader(parts[0].Bytes), + } +} + +func (psr *PartSetReader) Read(p []byte) (n int, err error) { + readerLen := psr.reader.Len() + if readerLen >= len(p) { + return psr.reader.Read(p) + } else if readerLen > 0 { + n1, err := psr.Read(p[:readerLen]) + if err != nil { + return n1, err + } + n2, err := psr.Read(p[readerLen:]) + return n1 + n2, err + } + + psr.i += 1 + if psr.i >= len(psr.parts) { + return 0, fmt.Errorf("Attempt to read from PartSet but no parts left") + } + psr.reader = bytes.NewReader(psr.parts[psr.i].Bytes) + return psr.Read(p) +} + func (ps *PartSet) StringShort() string { if ps == nil { return "nil-PartSet" From d23f38b4f3eea39f02b967ad77106ac8bf580517 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 12 Mar 2016 13:01:08 -0500 Subject: [PATCH 10/19] txs.Hash() to avoid extra allocs --- types/block.go | 8 ++------ types/tx.go | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/types/block.go b/types/block.go index 739a61b1f..9ce45bce0 100644 --- a/types/block.go +++ b/types/block.go @@ -329,7 +329,7 @@ type Data struct { // Txs that will be applied by state @ block.Height+1. // NOTE: not all txs here are valid. We're just agreeing on the order first. // This means that block.AppHash does not include these txs. - Txs []Tx `json:"txs"` + Txs Txs `json:"txs"` // Volatile hash []byte @@ -342,11 +342,7 @@ func (data *Data) Hash() []byte { return data.hash } if data.hash == nil { - txs := make([]interface{}, len(data.Txs)) - for i, tx := range data.Txs { - txs[i] = tx - } - data.hash = merkle.SimpleHashFromBinaries(txs) // NOTE: leaves are TxIDs. + data.hash = data.Txs.Hash() // NOTE: leaves of merkle tree are TxIDs } return data.hash } diff --git a/types/tx.go b/types/tx.go index a3cb9fc04..63a7afad4 100644 --- a/types/tx.go +++ b/types/tx.go @@ -1,3 +1,24 @@ package types +import ( + "github.com/tendermint/go-merkle" +) + type Tx []byte + +type Txs []Tx + +func (txs Txs) Hash() []byte { + // Recursive impl. + // Copied from go-merkle to avoid allocations + switch len(txs) { + case 0: + return nil + case 1: + return txs[0] + default: + left := Txs(txs[:(len(txs)+1)/2]).Hash() + right := Txs(txs[(len(txs)+1)/2:]).Hash() + return merkle.SimpleHashFromTwoHashes(left, right) + } +} From 27eede190a765e613c1d3d4f703c308f57183861 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 12 Mar 2016 01:06:04 -0500 Subject: [PATCH 11/19] glide go-p2p:develop --- glide.lock | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/glide.lock b/glide.lock index dd884dd42..f3539476e 100644 --- a/glide.lock +++ b/glide.lock @@ -1,18 +1,18 @@ hash: f3eab3f91c9d2c07574e8ec6f2f5d56bd946af1b061533a0baf9db8765f97a51 -updated: 2016-03-05T17:20:40.721925401-05:00 +updated: 2016-03-12T13:07:06.8783023-05:00 imports: - name: github.com/gogo/protobuf - version: f4cc07910fc38f5b6b8d6e75d7457cf504157b6c + version: 5dbe2a9482b48e988cadb27b8e06448688fc2f25 subpackages: - proto - name: github.com/golang/protobuf - version: c75fbf01dc6cb73649c4cd4326182c3e44aa9dbb + version: 0fd8c908d872c921af513ef5091964bbd2e0d904 subpackages: - proto - name: github.com/golang/snappy version: 5f1c01d9f64b941dd9582c638279d046eda6ca31 - name: github.com/gorilla/websocket - version: c45a635370221f34fea2d5163fd156fcb4e38e8a + version: a622679ebd7a3b813862379232f645f8e690e43f - name: github.com/inconshreveable/log15 version: 210d6fdc4d979ef6579778f1b6ed84571454abb4 subpackages: @@ -33,7 +33,7 @@ imports: - name: github.com/spf13/pflag version: 7f60f83a2c81bc3c3c0d5297f61ddfa68da9d3b7 - name: github.com/syndtr/goleveldb - version: ad0d8b2ab58a55ed5c58073aa46451d5e1ca1280 + version: 917f41c560270110ceb73c5b38be2a9127387071 subpackages: - leveldb - leveldb/errors @@ -61,7 +61,7 @@ imports: - name: github.com/tendermint/go-common version: 1559ae1ac90c88b1373ff114c409399c5a1cedac - name: github.com/tendermint/go-config - version: c077af2c1ecf584fb797fd1956758545b25d952b + version: c47b67203b070d8bea835a928d50cb739972c48a - name: github.com/tendermint/go-crypto version: 76ba23e4c0c627b8c66d1f97b6a18dc77f4f0297 - name: github.com/tendermint/go-db @@ -69,13 +69,13 @@ imports: - name: github.com/tendermint/go-events version: 7b75ca7bb55aa25e9ef765eb8c0b69486b227357 - name: github.com/tendermint/go-logger - version: 4901b71ade2b834ca0f4c2ca69edb96792dca05b + version: 84391b36d3f5960e691c688d06b768708f0fa2f3 - name: github.com/tendermint/go-logio version: 04f3aa0a3b38d06dcadefbafd988c8b85e499225 - name: github.com/tendermint/go-merkle - version: 67b535ce9633be7df575dc3a7833fa2301020c25 + version: 05042c6ab9cad51d12e4cecf717ae68e3b1409a8 - name: github.com/tendermint/go-p2p - version: 7f6aad20fbad6ef1a132d5a8bebd18f3521fff1a + version: dda17bf1f666ea692fa5ead79a5d35ce06daea9c subpackages: - upnp - name: github.com/tendermint/go-rpc @@ -85,18 +85,18 @@ imports: - types - server - name: github.com/tendermint/go-wire - version: 9acb294893c790427e2b9abf2877e69690cd5b6c + version: d9da1ad5fe838fbf89e64aabc68e5cf97007c02d - name: github.com/tendermint/log15 version: 6e460758f10ef42a4724b8e4a82fee59aaa0e41d - name: github.com/tendermint/tmsp - version: 72540f9cac4840989cb05b147cc89be8cd91f043 + version: 61c34ade0d480b8f23d6cdcd8fe7fea0dfec3279 subpackages: - types - example/dummy - example/nil - client - name: golang.org/x/crypto - version: 5dc8cb4b8a8eb076cbb5a06bc3b8682c15bdbbd3 + version: de93d05161db39bcbd84d3da2e54c4a18f37f0b1 subpackages: - ripemd160 - nacl/box From ae07c0171da905df0268d10c92e47dc07344c042 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 16 Mar 2016 19:45:46 -0400 Subject: [PATCH 12/19] light weight block event --- consensus/state.go | 2 +- types/events.go | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/consensus/state.go b/consensus/state.go index 6a0c54bf7..a768a7a0f 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -1205,7 +1205,7 @@ func (cs *ConsensusState) finalizeCommit(height int) { // Fire off event for new block. // TODO: Handle app failure. See #177 - cs.evsw.FireEvent(types.EventStringNewBlock(), types.EventDataNewBlock{block}) + cs.evsw.FireEvent(types.EventStringNewBlock(), types.EventDataNewBlock{&types.BlockHeader{block.Header}}) // Create a copy of the state for staging stateCopy := cs.state.Copy() diff --git a/types/events.go b/types/events.go index 247b1e7b9..02e6b072a 100644 --- a/types/events.go +++ b/types/events.go @@ -57,7 +57,11 @@ var _ = wire.RegisterInterface( // but some (an input to a call tx or a receive) are more exotic type EventDataNewBlock struct { - Block *Block `json:"block"` + // we drop block data but keep the form the same + Block *BlockHeader `json:"block"` +} +type BlockHeader struct { + Header *Header `json:"header"` } // All txs fire EventDataTx From ae295bd9b6c69bd0feba23c8bfa734a129057ae2 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 16 Mar 2016 21:15:33 -0400 Subject: [PATCH 13/19] bump rpc version; add consensus version --- consensus/version.go | 13 +++++++++++++ node/node.go | 1 + rpc/core/version.go | 2 +- 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 consensus/version.go diff --git a/consensus/version.go b/consensus/version.go new file mode 100644 index 000000000..2acc3b77b --- /dev/null +++ b/consensus/version.go @@ -0,0 +1,13 @@ +package consensus + +import ( + . "github.com/tendermint/go-common" +) + +// kind of arbitrary +var Spec = "1" // async +var Major = "0" // +var Minor = "2" // replay refactor +var Revision = "1" // round state fix + +var Version = Fmt("v%s/%s.%s.%s", Spec, Major, Minor, Revision) diff --git a/node/node.go b/node/node.go index 5ad301800..96010a1a4 100644 --- a/node/node.go +++ b/node/node.go @@ -224,6 +224,7 @@ func makeNodeInfo(sw *p2p.Switch, privKey crypto.PrivKeyEd25519) *p2p.NodeInfo { Other: []string{ Fmt("wire_version=%v", wire.Version), Fmt("p2p_version=%v", p2p.Version), + Fmt("consensus_version=%v", consensus.Version), Fmt("rpc_version=%v/%v", rpc.Version, rpccore.Version), }, } diff --git a/rpc/core/version.go b/rpc/core/version.go index 0890da87a..e283de479 100644 --- a/rpc/core/version.go +++ b/rpc/core/version.go @@ -2,4 +2,4 @@ package core // a single integer is sufficient here -const Version = "2" // add DialSeeds; re-organize type bytes +const Version = "3" // rpc routes for profiling, setting config From eaf222a75684103b3202eb86d9019f53743083ab Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 17 Mar 2016 04:27:58 -0400 Subject: [PATCH 14/19] check vote exists before verifiying --- types/vote_set.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/types/vote_set.go b/types/vote_set.go index 8c03f79bf..22fe246b1 100644 --- a/types/vote_set.go +++ b/types/vote_set.go @@ -128,17 +128,16 @@ func (voteSet *VoteSet) addVote(val *Validator, valIndex int, vote *Vote) (bool, return false, 0, ErrVoteUnexpectedStep } - // Check signature. - if !val.PubKey.VerifyBytes(SignBytes(config.GetString("chain_id"), vote), vote.Signature) { - // Bad signature. - return false, 0, ErrVoteInvalidSignature - } - // If vote already exists, return false. if existingVote := voteSet.votes[valIndex]; existingVote != nil { if bytes.Equal(existingVote.BlockHash, vote.BlockHash) { return false, valIndex, nil } else { + // Check signature. + if !val.PubKey.VerifyBytes(SignBytes(config.GetString("chain_id"), vote), vote.Signature) { + // Bad signature. + return false, 0, ErrVoteInvalidSignature + } return false, valIndex, &ErrVoteConflictingSignature{ VoteA: existingVote, VoteB: vote, @@ -146,6 +145,12 @@ func (voteSet *VoteSet) addVote(val *Validator, valIndex int, vote *Vote) (bool, } } + // Check signature. + if !val.PubKey.VerifyBytes(SignBytes(config.GetString("chain_id"), vote), vote.Signature) { + // Bad signature. + return false, 0, ErrVoteInvalidSignature + } + // Add vote. voteSet.votes[valIndex] = vote voteSet.votesBitArray.SetIndex(valIndex, true) From 16208dc172717202b5bcf45f28fe5888fca4e067 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 17 Mar 2016 05:05:27 -0400 Subject: [PATCH 15/19] glide update --- glide.lock | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/glide.lock b/glide.lock index f3539476e..c91e9664d 100644 --- a/glide.lock +++ b/glide.lock @@ -1,18 +1,18 @@ hash: f3eab3f91c9d2c07574e8ec6f2f5d56bd946af1b061533a0baf9db8765f97a51 -updated: 2016-03-12T13:07:06.8783023-05:00 +updated: 2016-03-18T02:38:49.170573737-04:00 imports: - name: github.com/gogo/protobuf - version: 5dbe2a9482b48e988cadb27b8e06448688fc2f25 + version: d5d4941acc4490628827091cafee2168f8c4bc67 subpackages: - proto - name: github.com/golang/protobuf - version: 0fd8c908d872c921af513ef5091964bbd2e0d904 + version: 62e4364d64b32762febb61f2c88c0a29bc49a225 subpackages: - proto - name: github.com/golang/snappy version: 5f1c01d9f64b941dd9582c638279d046eda6ca31 - name: github.com/gorilla/websocket - version: a622679ebd7a3b813862379232f645f8e690e43f + version: e2e3d8414d0fbae04004f151979f4e27c6747fe7 - name: github.com/inconshreveable/log15 version: 210d6fdc4d979ef6579778f1b6ed84571454abb4 subpackages: @@ -59,11 +59,11 @@ imports: - name: github.com/tendermint/go-clist version: 634527f5b60fd7c71ca811262493df2ad65ee0ca - name: github.com/tendermint/go-common - version: 1559ae1ac90c88b1373ff114c409399c5a1cedac + version: dcfa46af1341d03b80d32e4901019d1668b978b9 - name: github.com/tendermint/go-config version: c47b67203b070d8bea835a928d50cb739972c48a - name: github.com/tendermint/go-crypto - version: 76ba23e4c0c627b8c66d1f97b6a18dc77f4f0297 + version: 8152c18c355bb17e38292d9473e71d7998a72a43 - name: github.com/tendermint/go-db version: a7878f1d0d8eaebf15f87bc2df15f7a1088cce7f - name: github.com/tendermint/go-events @@ -75,7 +75,7 @@ imports: - name: github.com/tendermint/go-merkle version: 05042c6ab9cad51d12e4cecf717ae68e3b1409a8 - name: github.com/tendermint/go-p2p - version: dda17bf1f666ea692fa5ead79a5d35ce06daea9c + version: 69c7ae5e3fd5fb9a3bf26d4b93f9ea8d6b921b44 subpackages: - upnp - name: github.com/tendermint/go-rpc @@ -85,27 +85,29 @@ imports: - types - server - name: github.com/tendermint/go-wire - version: d9da1ad5fe838fbf89e64aabc68e5cf97007c02d + version: 39ed90899108c7bdd66482447f40a1e72f88a9da - name: github.com/tendermint/log15 version: 6e460758f10ef42a4724b8e4a82fee59aaa0e41d - name: github.com/tendermint/tmsp - version: 61c34ade0d480b8f23d6cdcd8fe7fea0dfec3279 + version: 29a6d511b48e5c2dc3feff0c57237062baa83f33 subpackages: - types - example/dummy - example/nil - client - name: golang.org/x/crypto - version: de93d05161db39bcbd84d3da2e54c4a18f37f0b1 + version: 6025851c7c2bf210daf74d22300c699b16541847 subpackages: - ripemd160 - nacl/box - nacl/secretbox + - openpgp/armor - curve25519 - salsa20/salsa - poly1305 + - openpgp/errors - name: golang.org/x/sys - version: 7a56174f0086b32866ebd746a794417edbc678a1 + version: 9d4e42a20653790449273b3c85e67d6d8bae6e2e subpackages: - unix devImports: [] From 922f720cf615061a46238ce42c6a54fe06280134 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 21 Mar 2016 01:24:26 -0400 Subject: [PATCH 16/19] cswal_light logs own votes; fix tests --- consensus/replay_test.go | 5 +++-- consensus/state_test.go | 2 +- consensus/wal.go | 9 ++++++--- types/part_set.go | 2 +- types/part_set_test.go | 6 +++--- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 40ec34ca8..044bb96cc 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -80,19 +80,20 @@ func TestReplayCatchup(t *testing.T) { // start timeout and receive routines cs.startRoutines(0) + // cs.scheduleRound0(cs.Height) + // open wal and run catchup messages openWAL(t, cs, name) if err := cs.catchupReplay(cs.Height); err != nil { t.Fatalf("Error on catchup replay %v", err) } - after := time.After(time.Second * 2) + after := time.After(time.Second * 15) select { case <-newBlockCh: case <-after: t.Fatal("Timed out waiting for new block") } - } func openWAL(t *testing.T, cs *ConsensusState, file string) { diff --git a/consensus/state_test.go b/consensus/state_test.go index 277471267..eaac14dfe 100644 --- a/consensus/state_test.go +++ b/consensus/state_test.go @@ -584,7 +584,7 @@ func TestLockPOLRelock(t *testing.T) { t.Fatal("Expected height to increment") } - if !bytes.Equal(b.Block.Hash(), propBlockHash) { + if !bytes.Equal(b.Block.Header.Hash(), propBlockHash) { t.Fatal("Expected new block to be proposal block") } } diff --git a/consensus/wal.go b/consensus/wal.go index b5c79993d..5b4747a6f 100644 --- a/consensus/wal.go +++ b/consensus/wal.go @@ -64,9 +64,12 @@ func NewWAL(file string, light bool) (*WAL, error) { func (wal *WAL) Save(clm ConsensusLogMessageInterface) { if wal != nil { if wal.light { - // in light mode we only write new steps and timeouts (no votes, proposals, block parts) - if _, ok := clm.(msgInfo); ok { - return + // in light mode we only write new steps, timeouts, and our own votes (no proposals, block parts) + if mi, ok := clm.(msgInfo); ok { + _ = mi + if mi.PeerKey != "" { + return + } } } var n int diff --git a/types/part_set.go b/types/part_set.go index 146e23c43..c3e34cba3 100644 --- a/types/part_set.go +++ b/types/part_set.go @@ -268,7 +268,7 @@ func (psr *PartSetReader) Read(p []byte) (n int, err error) { psr.i += 1 if psr.i >= len(psr.parts) { - return 0, fmt.Errorf("Attempt to read from PartSet but no parts left") + return 0, io.EOF } psr.reader = bytes.NewReader(psr.parts[psr.i].Bytes) return psr.Read(p) diff --git a/types/part_set_test.go b/types/part_set_test.go index 4e74d7772..bbc3da9ed 100644 --- a/types/part_set_test.go +++ b/types/part_set_test.go @@ -30,7 +30,7 @@ func TestBasicPartSet(t *testing.T) { for i := 0; i < partSet.Total(); i++ { part := partSet.GetPart(i) //t.Logf("\n%v", part) - added, err := partSet2.AddPart(part) + added, err := partSet2.AddPart(part, true) if !added || err != nil { t.Errorf("Failed to add part %v, error: %v", i, err) } @@ -70,7 +70,7 @@ func TestWrongProof(t *testing.T) { // Test adding a part with wrong trail. part := partSet.GetPart(0) part.Proof.Aunts[0][0] += byte(0x01) - added, err := partSet2.AddPart(part) + added, err := partSet2.AddPart(part, true) if added || err == nil { t.Errorf("Expected to fail adding a part with bad trail.") } @@ -78,7 +78,7 @@ func TestWrongProof(t *testing.T) { // Test adding a part with wrong bytes. part = partSet.GetPart(1) part.Bytes[0] += byte(0x01) - added, err = partSet2.AddPart(part) + added, err = partSet2.AddPart(part, true) if added || err == nil { t.Errorf("Expected to fail adding a part with bad bytes.") } From 08d12a5c8495c0b2b8475d4180a1ca12d2ec3948 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 24 Mar 2016 16:39:53 -0400 Subject: [PATCH 17/19] glide update --- glide.lock | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/glide.lock b/glide.lock index c91e9664d..92cf1a87c 100644 --- a/glide.lock +++ b/glide.lock @@ -1,12 +1,12 @@ hash: f3eab3f91c9d2c07574e8ec6f2f5d56bd946af1b061533a0baf9db8765f97a51 -updated: 2016-03-18T02:38:49.170573737-04:00 +updated: 2016-03-24T16:39:27.330201414-04:00 imports: - name: github.com/gogo/protobuf - version: d5d4941acc4490628827091cafee2168f8c4bc67 + version: 4168943e65a2802828518e95310aeeed6d84c4e5 subpackages: - proto - name: github.com/golang/protobuf - version: 62e4364d64b32762febb61f2c88c0a29bc49a225 + version: 8d92cf5fc15a4382f8964b08e1f42a75c0591aa3 subpackages: - proto - name: github.com/golang/snappy @@ -63,7 +63,7 @@ imports: - name: github.com/tendermint/go-config version: c47b67203b070d8bea835a928d50cb739972c48a - name: github.com/tendermint/go-crypto - version: 8152c18c355bb17e38292d9473e71d7998a72a43 + version: 3f0d9b3f29f30e5d0cbc2cef04fa45e5a606c622 - name: github.com/tendermint/go-db version: a7878f1d0d8eaebf15f87bc2df15f7a1088cce7f - name: github.com/tendermint/go-events @@ -75,7 +75,7 @@ imports: - name: github.com/tendermint/go-merkle version: 05042c6ab9cad51d12e4cecf717ae68e3b1409a8 - name: github.com/tendermint/go-p2p - version: 69c7ae5e3fd5fb9a3bf26d4b93f9ea8d6b921b44 + version: 10619248c665dee6b8f81455f7f27ab93d5ec366 subpackages: - upnp - name: github.com/tendermint/go-rpc @@ -85,18 +85,18 @@ imports: - types - server - name: github.com/tendermint/go-wire - version: 39ed90899108c7bdd66482447f40a1e72f88a9da + version: 7a15dd53dfdecc0f967676edcd6b335c59344c83 - name: github.com/tendermint/log15 version: 6e460758f10ef42a4724b8e4a82fee59aaa0e41d - name: github.com/tendermint/tmsp - version: 29a6d511b48e5c2dc3feff0c57237062baa83f33 + version: 1dfc6950dddf47ff397e670a67d405d25da138ea subpackages: - types + - client - example/dummy - example/nil - - client - name: golang.org/x/crypto - version: 6025851c7c2bf210daf74d22300c699b16541847 + version: c197bcf24cde29d3f73c7b4ac6fd41f4384e8af6 subpackages: - ripemd160 - nacl/box @@ -107,7 +107,7 @@ imports: - poly1305 - openpgp/errors - name: golang.org/x/sys - version: 9d4e42a20653790449273b3c85e67d6d8bae6e2e + version: afce3de5756ca82699128ebae46ac95ad59d6297 subpackages: - unix devImports: [] From 523a170c3e1353c3ab403bb79804289b8948c433 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 19 Apr 2016 20:59:52 -0400 Subject: [PATCH 18/19] EventDataTypeNewBlockHeader --- consensus/state.go | 3 ++- consensus/state_test.go | 6 +++--- types/events.go | 25 +++++++++++++++---------- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/consensus/state.go b/consensus/state.go index a768a7a0f..fdbfdb6e6 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -1205,7 +1205,8 @@ func (cs *ConsensusState) finalizeCommit(height int) { // Fire off event for new block. // TODO: Handle app failure. See #177 - cs.evsw.FireEvent(types.EventStringNewBlock(), types.EventDataNewBlock{&types.BlockHeader{block.Header}}) + cs.evsw.FireEvent(types.EventStringNewBlock(), types.EventDataNewBlock{block}) + cs.evsw.FireEvent(types.EventStringNewBlockHeader(), types.EventDataNewBlockHeader{block.Header}) // Create a copy of the state for staging stateCopy := cs.state.Copy() diff --git a/consensus/state_test.go b/consensus/state_test.go index eaac14dfe..dd2dbe162 100644 --- a/consensus/state_test.go +++ b/consensus/state_test.go @@ -491,7 +491,7 @@ func TestLockPOLRelock(t *testing.T) { proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1) voteCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringVote(), 1) newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1) - newBlockCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewBlock(), 1) + newBlockCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewBlockHeader(), 1) log.Debug("cs2 last round", "lr", cs2.PrivValidator.LastRound) @@ -577,14 +577,14 @@ func TestLockPOLRelock(t *testing.T) { _, _ = <-voteCh, <-voteCh be := <-newBlockCh - b := be.(types.EventDataNewBlock) + b := be.(types.EventDataNewBlockHeader) re = <-newRoundCh rs = re.(types.EventDataRoundState).RoundState.(*RoundState) if rs.Height != 2 { t.Fatal("Expected height to increment") } - if !bytes.Equal(b.Block.Header.Hash(), propBlockHash) { + if !bytes.Equal(b.Header.Hash(), propBlockHash) { t.Fatal("Expected new block to be proposal block") } } diff --git a/types/events.go b/types/events.go index 02e6b072a..3328911c1 100644 --- a/types/events.go +++ b/types/events.go @@ -16,6 +16,7 @@ func EventStringDupeout() string { return "Dupeout" } func EventStringFork() string { return "Fork" } func EventStringNewBlock() string { return "NewBlock" } +func EventStringNewBlockHeader() string { return "NewBlockHeader" } func EventStringNewRound() string { return "NewRound" } func EventStringNewRoundStep() string { return "NewRoundStep" } func EventStringTimeoutPropose() string { return "TimeoutPropose" } @@ -36,9 +37,10 @@ type TMEventData interface { } const ( - EventDataTypeNewBlock = byte(0x01) - EventDataTypeFork = byte(0x02) - EventDataTypeTx = byte(0x03) + EventDataTypeNewBlock = byte(0x01) + EventDataTypeFork = byte(0x02) + EventDataTypeTx = byte(0x03) + EventDataTypeNewBlockHeader = byte(0x04) EventDataTypeRoundState = byte(0x11) EventDataTypeVote = byte(0x12) @@ -47,6 +49,7 @@ const ( var _ = wire.RegisterInterface( struct{ TMEventData }{}, wire.ConcreteType{EventDataNewBlock{}, EventDataTypeNewBlock}, + wire.ConcreteType{EventDataNewBlockHeader{}, EventDataTypeNewBlockHeader}, // wire.ConcreteType{EventDataFork{}, EventDataTypeFork }, wire.ConcreteType{EventDataTx{}, EventDataTypeTx}, wire.ConcreteType{EventDataRoundState{}, EventDataTypeRoundState}, @@ -57,10 +60,11 @@ var _ = wire.RegisterInterface( // but some (an input to a call tx or a receive) are more exotic type EventDataNewBlock struct { - // we drop block data but keep the form the same - Block *BlockHeader `json:"block"` + Block *Block `json:"block"` } -type BlockHeader struct { + +// light weight event for benchmarking +type EventDataNewBlockHeader struct { Header *Header `json:"header"` } @@ -88,7 +92,8 @@ type EventDataVote struct { Vote *Vote } -func (_ EventDataNewBlock) AssertIsTMEventData() {} -func (_ EventDataTx) AssertIsTMEventData() {} -func (_ EventDataRoundState) AssertIsTMEventData() {} -func (_ EventDataVote) AssertIsTMEventData() {} +func (_ EventDataNewBlock) AssertIsTMEventData() {} +func (_ EventDataNewBlockHeader) AssertIsTMEventData() {} +func (_ EventDataTx) AssertIsTMEventData() {} +func (_ EventDataRoundState) AssertIsTMEventData() {} +func (_ EventDataVote) AssertIsTMEventData() {} From 501c4e4bac7ac13ab1ab4ed71b717cf7dc791635 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 26 Apr 2016 22:17:13 -0400 Subject: [PATCH 19/19] fixes from review --- config/config.go | 117 --------------------------------------- consensus/reactor.go | 2 +- consensus/replay_test.go | 2 - mempool/mempool.go | 6 +- mempool/mempool_test.go | 2 +- types/part_set.go | 6 -- types/tx.go | 2 +- 7 files changed, 6 insertions(+), 131 deletions(-) delete mode 100644 config/config.go diff --git a/config/config.go b/config/config.go deleted file mode 100644 index 21c6b81f8..000000000 --- a/config/config.go +++ /dev/null @@ -1,117 +0,0 @@ - -package config - -import ( - "github.com/naoina/toml" - "sync" - "time" - - . "github.com/tendermint/go-common" -) - -type Config interface { - Get(key string) interface{} - GetBool(key string) bool - GetFloat64(key string) float64 - GetInt(key string) int - GetString(key string) string - GetStringMap(key string) map[string]interface{} - GetStringMapString(key string) map[string]string - GetStringSlice(key string) []string - GetTime(key string) time.Time - IsSet(key string) bool - Set(key string, value interface{}) -} - -type MapConfig struct { - required map[string]struct{} // blows up if trying to use before setting. - data map[string]interface{} -} - -func ReadMapConfigFromFile(filePath string) (MapConfig, error) { - var configData = make(map[string]interface{}) - fileBytes := MustReadFile(filePath) - err := toml.Unmarshal(fileBytes, configData) - if err != nil { - return MapConfig{}, err - } - return NewMapConfig(configData), nil -} - -func NewMapConfig(data map[string]interface{}) MapConfig { - if data == nil { - data = make(map[string]interface{}) - } - return MapConfig{ - required: make(map[string]struct{}), - data: data, - } -} - -func (cfg MapConfig) Get(key string) interface{} { - if _, ok := cfg.required[key]; ok { - PanicSanity(Fmt("config key %v is required but was not set.", key)) - } - return cfg.data[key] -} -func (cfg MapConfig) GetBool(key string) bool { return cfg.Get(key).(bool) } -func (cfg MapConfig) GetFloat64(key string) float64 { return cfg.Get(key).(float64) } -func (cfg MapConfig) GetInt(key string) int { return cfg.Get(key).(int) } -func (cfg MapConfig) GetString(key string) string { return cfg.Get(key).(string) } -func (cfg MapConfig) GetStringMap(key string) map[string]interface{} { - return cfg.Get(key).(map[string]interface{}) -} -func (cfg MapConfig) GetStringMapString(key string) map[string]string { - return cfg.Get(key).(map[string]string) -} -func (cfg MapConfig) GetStringSlice(key string) []string { return cfg.Get(key).([]string) } -func (cfg MapConfig) GetTime(key string) time.Time { return cfg.Get(key).(time.Time) } -func (cfg MapConfig) IsSet(key string) bool { _, ok := cfg.data[key]; return ok } -func (cfg MapConfig) Set(key string, value interface{}) { - delete(cfg.required, key) - cfg.data[key] = value -} -func (cfg MapConfig) SetDefault(key string, value interface{}) { - delete(cfg.required, key) - if cfg.IsSet(key) { - return - } - cfg.data[key] = value -} -func (cfg MapConfig) SetRequired(key string) { - if cfg.IsSet(key) { - return - } - cfg.required[key] = struct{}{} -} - -//-------------------------------------------------------------------------------- -// A little convenient hack to notify listeners upon config changes. - -type Configurable func(Config) - -var mtx sync.Mutex -var globalConfig Config -var confs []Configurable - -func OnConfig(conf func(Config)) { - mtx.Lock() - defer mtx.Unlock() - - confs = append(confs, conf) - if globalConfig != nil { - conf(globalConfig) - } -} - -func ApplyConfig(config Config) { - mtx.Lock() - globalConfig = config - confsCopy := make([]Configurable, len(confs)) - copy(confsCopy, confs) - mtx.Unlock() - - for _, conf := range confsCopy { - conf(config) - } -} diff --git a/consensus/reactor.go b/consensus/reactor.go index b97a45a2d..98f6510dd 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -92,7 +92,7 @@ func (conR *ConsensusReactor) GetChannels() []*p2p.ChannelDescriptor { }, &p2p.ChannelDescriptor{ ID: DataChannel, // maybe split between gossiping current block and catchup stuff - Priority: 200, // once we gossip the whole block there's nothing left to send until next height or round + Priority: 10, // once we gossip the whole block there's nothing left to send until next height or round SendQueueCapacity: 100, RecvBufferCapacity: 50 * 4096, }, diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 044bb96cc..d46f68cd1 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -80,8 +80,6 @@ func TestReplayCatchup(t *testing.T) { // start timeout and receive routines cs.startRoutines(0) - // cs.scheduleRound0(cs.Height) - // open wal and run catchup messages openWAL(t, cs, name) if err := cs.catchupReplay(cs.Height); err != nil { diff --git a/mempool/mempool.go b/mempool/mempool.go index 0fab674bf..6df62feee 100644 --- a/mempool/mempool.go +++ b/mempool/mempool.go @@ -201,12 +201,12 @@ func (mem *Mempool) Reap(maxTxs int) []types.Tx { return txs } -// maxTxs: 0 means uncapped, -1 means none +// maxTxs: -1 means uncapped, 0 means none func (mem *Mempool) collectTxs(maxTxs int) []types.Tx { if maxTxs == 0 { - maxTxs = mem.txs.Len() - } else if maxTxs < 0 { return []types.Tx{} + } else if maxTxs < 0 { + maxTxs = mem.txs.Len() } txs := make([]types.Tx, 0, MinInt(mem.txs.Len(), maxTxs)) for e := mem.txs.Front(); e != nil && len(txs) < maxTxs; e = e.Next() { diff --git a/mempool/mempool_test.go b/mempool/mempool_test.go index 750f987da..a15496962 100644 --- a/mempool/mempool_test.go +++ b/mempool/mempool_test.go @@ -48,7 +48,7 @@ func TestSerialReap(t *testing.T) { } reapCheck := func(exp int) { - txs := mempool.Reap(0) + txs := mempool.Reap(-1) if len(txs) != exp { t.Fatalf("Expected to reap %v txs but got %v", exp, len(txs)) } diff --git a/types/part_set.go b/types/part_set.go index c3e34cba3..bdf198d94 100644 --- a/types/part_set.go +++ b/types/part_set.go @@ -231,12 +231,6 @@ func (ps *PartSet) GetReader() io.Reader { PanicSanity("Cannot GetReader() on incomplete PartSet") } return NewPartSetReader(ps.parts) - - buf := []byte{} - for _, part := range ps.parts { - buf = append(buf, part.Bytes...) - } - return bytes.NewReader(buf) } type PartSetReader struct { diff --git a/types/tx.go b/types/tx.go index 63a7afad4..60699d534 100644 --- a/types/tx.go +++ b/types/tx.go @@ -15,7 +15,7 @@ func (txs Txs) Hash() []byte { case 0: return nil case 1: - return txs[0] + return merkle.SimpleHashFromBinary(txs[0]) default: left := Txs(txs[:(len(txs)+1)/2]).Hash() right := Txs(txs[(len(txs)+1)/2:]).Hash()