From 78446fd99c923586a04fd6a1500be10175e87035 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 21 Aug 2017 16:31:54 -0400 Subject: [PATCH 01/92] state: persist validators --- consensus/replay.go | 3 ++ state/errors.go | 8 +++ state/execution.go | 3 ++ state/state.go | 115 ++++++++++++++++++++++++++++++++++++++------ state/state_test.go | 60 +++++++++++++++++------ 5 files changed, 161 insertions(+), 28 deletions(-) diff --git a/consensus/replay.go b/consensus/replay.go index 0c7c27e90..71dc81b5a 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -324,8 +324,11 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp p func (h *Handshaker) replayBlocks(proxyApp proxy.AppConns, appBlockHeight, storeBlockHeight int, mutateState bool) ([]byte, error) { // App is further behind than it should be, so we need to replay blocks. // We replay all blocks from appBlockHeight+1. + // // Note that we don't have an old version of the state, // so we by-pass state validation/mutation using sm.ExecCommitBlock. + // This also means we won't be saving validator sets if they change during this period. + // // If mutateState == true, the final block is replayed with h.replayBlock() var appHash []byte diff --git a/state/errors.go b/state/errors.go index 50c5a2c04..4a87384a0 100644 --- a/state/errors.go +++ b/state/errors.go @@ -33,6 +33,10 @@ type ( Got *State Expected *State } + + ErrNoValSetForHeight struct { + Height int + } ) func (e ErrUnknownBlock) Error() string { @@ -53,3 +57,7 @@ func (e ErrLastStateMismatch) Error() string { func (e ErrStateMismatch) Error() string { return cmn.Fmt("State after replay does not match saved state. Got ----\n%v\nExpected ----\n%v\n", e.Got, e.Expected) } + +func (e ErrNoValSetForHeight) Error() string { + return cmn.Fmt("Could not find validator set for height #%d", e.Height) +} diff --git a/state/execution.go b/state/execution.go index 9e4a5fb29..869d28d48 100644 --- a/state/execution.go +++ b/state/execution.go @@ -231,6 +231,9 @@ func (s *State) ApplyBlock(eventCache types.Fireable, proxyAppConn proxy.AppConn // now update the block and validators s.SetBlockAndValidators(block.Header, partsHeader, abciResponses) + // save the validators for the next block now that we know them + s.SaveValidators() + // lock mempool, commit state, update mempoool err = s.CommitStateUpdateMempool(proxyAppConn, block, mempool) if err != nil { diff --git a/state/state.go b/state/state.go index 21ad92776..6106891b5 100644 --- a/state/state.go +++ b/state/state.go @@ -22,6 +22,10 @@ var ( abciResponsesKey = []byte("abciResponsesKey") ) +func calcValidatorsKey(height int) []byte { + return []byte(cmn.Fmt("validatorsKey:%v", height)) +} + //----------------------------------------------------------------------------- // State represents the latest committed state of the Tendermint consensus, @@ -47,8 +51,11 @@ type State struct { // AppHash is updated after Commit AppHash []byte + // XXX: do we need this json tag ? TxIndexer txindex.TxIndexer `json:"-"` // Transaction indexer. + lastHeightValidatorsChanged int + logger log.Logger } @@ -71,6 +78,14 @@ func loadState(db dbm.DB, key []byte) *State { } // TODO: ensure that buf is completely read. } + + v := s.loadValidators(s.LastBlockHeight) + if v != nil { + s.lastHeightValidatorsChanged = v.LastHeightChanged + } else { + s.lastHeightValidatorsChanged = 1 + } + return s } @@ -93,6 +108,8 @@ func (s *State) Copy() *State { AppHash: s.AppHash, TxIndexer: s.TxIndexer, // pointer here, not value logger: s.logger, + + lastHeightValidatorsChanged: s.lastHeightValidatorsChanged, } } @@ -126,6 +143,56 @@ func (s *State) LoadABCIResponses() *ABCIResponses { return abciResponses } +// SaveValidators persists the validator set for the next block to disk. +// It should be called after the validator set is updated with the results of EndBlock. +// If the validator set did not change after processing the latest block, +// only the last height for which the validators changed is persisted. +func (s *State) SaveValidators() { + lastHeight := s.lastHeightValidatorsChanged + nextHeight := s.LastBlockHeight + 1 + v := &Validators{ + LastHeightChanged: lastHeight, + } + if lastHeight == nextHeight { + v.ValidatorSet = s.Validators + } + s.db.SetSync(calcValidatorsKey(nextHeight), v.Bytes()) +} + +// LoadValidators loads the ValidatorSet for a given height. +func (s *State) LoadValidators(height int) (*types.ValidatorSet, error) { + v := s.loadValidators(height) + if v == nil { + return nil, ErrNoValSetForHeight{height} + } + + if v.ValidatorSet == nil { + v = s.loadValidators(v.LastHeightChanged) + if v == nil { + return nil, ErrNoValSetForHeight{height} + } + } + + return v.ValidatorSet, nil +} + +func (s *State) loadValidators(height int) *Validators { + buf := s.db.Get(calcValidatorsKey(height)) + if len(buf) == 0 { + return nil + } + + v := new(Validators) + r, n, err := bytes.NewReader(buf), new(int), new(error) + wire.ReadBinaryPtr(v, r, 0, n, err) + if *err != nil { + // DATA HAS BEEN CORRUPTED OR THE SPEC HAS CHANGED + cmn.Exit(cmn.Fmt("LoadValidators: Data has been corrupted or its spec has changed: %v\n", *err)) + } + // TODO: ensure that buf is completely read. + return v +} + // Equals returns true if the States are identical. func (s *State) Equals(s2 *State) bool { return bytes.Equal(s.Bytes(), s2.Bytes()) @@ -144,10 +211,15 @@ func (s *State) SetBlockAndValidators(header *types.Header, blockPartsHeader typ prevValSet := s.Validators.Copy() nextValSet := prevValSet.Copy() - err := updateValidators(nextValSet, abciResponses.EndBlock.Diffs) - if err != nil { - s.logger.Error("Error changing validator set", "err", err) - // TODO: err or carry on? + // update the validator set with the latest abciResponses + if len(abciResponses.EndBlock.Diffs) > 0 { + err := updateValidators(nextValSet, abciResponses.EndBlock.Diffs) + if err != nil { + s.logger.Error("Error changing validator set", "err", err) + // TODO: err or carry on? + } + // change results from this height but only applies to the next height + s.lastHeightValidatorsChanged = header.Height + 1 } // Update validator accums and set state variables @@ -182,6 +254,7 @@ func GetState(stateDB dbm.DB, genesisFile string) *State { state := LoadState(stateDB) if state == nil { state = MakeGenesisStateFromFile(stateDB, genesisFile) + state.SaveValidators() // save the validators right away for height 1 state.Save() } @@ -215,6 +288,19 @@ func (a *ABCIResponses) Bytes() []byte { return wire.BinaryBytes(*a) } +//----------------------------------------------------------------------------- + +// Validators represents the latest validator set, or the last time it changed +type Validators struct { + ValidatorSet *types.ValidatorSet + LastHeightChanged int +} + +// Bytes serializes the Validators using go-wire +func (v *Validators) Bytes() []byte { + return wire.BinaryBytes(*v) +} + //----------------------------------------------------------------------------- // Genesis @@ -260,15 +346,16 @@ func MakeGenesisState(db dbm.DB, genDoc *types.GenesisDoc) *State { } return &State{ - db: db, - GenesisDoc: genDoc, - ChainID: genDoc.ChainID, - LastBlockHeight: 0, - LastBlockID: types.BlockID{}, - LastBlockTime: genDoc.GenesisTime, - Validators: types.NewValidatorSet(validators), - LastValidators: types.NewValidatorSet(nil), - AppHash: genDoc.AppHash, - TxIndexer: &null.TxIndex{}, // we do not need indexer during replay and in tests + db: db, + GenesisDoc: genDoc, + ChainID: genDoc.ChainID, + LastBlockHeight: 0, + LastBlockID: types.BlockID{}, + LastBlockTime: genDoc.GenesisTime, + Validators: types.NewValidatorSet(validators), + LastValidators: types.NewValidatorSet(nil), + AppHash: genDoc.AppHash, + TxIndexer: &null.TxIndex{}, // we do not need indexer during replay and in tests + lastHeightValidatorsChanged: 1, } } diff --git a/state/state_test.go b/state/state_test.go index e97c3289a..22543cc17 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -1,18 +1,21 @@ package state import ( - "fmt" "testing" "github.com/stretchr/testify/assert" + abci "github.com/tendermint/abci/types" crypto "github.com/tendermint/go-crypto" - cfg "github.com/tendermint/tendermint/config" + cmn "github.com/tendermint/tmlibs/common" dbm "github.com/tendermint/tmlibs/db" "github.com/tendermint/tmlibs/log" + + cfg "github.com/tendermint/tendermint/config" ) func TestStateCopyEquals(t *testing.T) { + assert := assert.New(t) config := cfg.ResetTestRoot("state_") // Get State db @@ -22,18 +25,13 @@ func TestStateCopyEquals(t *testing.T) { stateCopy := state.Copy() - if !state.Equals(stateCopy) { - t.Fatal("expected state and its copy to be identical. got %v\n expected %v\n", stateCopy, state) - } - + assert.True(state.Equals(stateCopy), cmn.Fmt("expected state and its copy to be identical. got %v\n expected %v\n", stateCopy, state)) stateCopy.LastBlockHeight += 1 - - if state.Equals(stateCopy) { - t.Fatal("expected states to be different. got same %v", state) - } + assert.False(state.Equals(stateCopy), cmn.Fmt("expected states to be different. got same %v", state)) } func TestStateSaveLoad(t *testing.T) { + assert := assert.New(t) config := cfg.ResetTestRoot("state_") // Get State db stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir()) @@ -44,9 +42,7 @@ func TestStateSaveLoad(t *testing.T) { state.Save() loadedState := LoadState(stateDB) - if !state.Equals(loadedState) { - t.Fatal("expected state and its copy to be identical. got %v\n expected %v\n", loadedState, state) - } + assert.True(state.Equals(loadedState), cmn.Fmt("expected state and its copy to be identical. got %v\n expected %v\n", loadedState, state)) } func TestABCIResponsesSaveLoad(t *testing.T) { @@ -74,5 +70,41 @@ func TestABCIResponsesSaveLoad(t *testing.T) { state.SaveABCIResponses(abciResponses) abciResponses2 := state.LoadABCIResponses() - assert.Equal(abciResponses, abciResponses2, fmt.Sprintf("ABCIResponses don't match: Got %v, Expected %v", abciResponses2, abciResponses)) + assert.Equal(abciResponses, abciResponses2, cmn.Fmt("ABCIResponses don't match: Got %v, Expected %v", abciResponses2, abciResponses)) +} + +func TestValidatorsSaveLoad(t *testing.T) { + assert := assert.New(t) + config := cfg.ResetTestRoot("state_") + // Get State db + stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir()) + state := GetState(stateDB, config.GenesisFile()) + state.SetLogger(log.TestingLogger()) + + // cant load anything for height 0 + v, err := state.LoadValidators(0) + assert.NotNil(err, "expected err at height 0") + + // should be able to load for height 1 + v, err = state.LoadValidators(1) + assert.Nil(err, "expected no err at height 1") + assert.Equal(v.Hash(), state.Validators.Hash(), "expected validator hashes to match") + + // increment height, save; should be able to load for next height + state.LastBlockHeight += 1 + state.SaveValidators() + v, err = state.LoadValidators(state.LastBlockHeight + 1) + assert.Nil(err, "expected no err") + assert.Equal(v.Hash(), state.Validators.Hash(), "expected validator hashes to match") + + // increment height, save; should be able to load for next height + state.LastBlockHeight += 10 + state.SaveValidators() + v, err = state.LoadValidators(state.LastBlockHeight + 1) + assert.Nil(err, "expected no err") + assert.Equal(v.Hash(), state.Validators.Hash(), "expected validator hashes to match") + + // should be able to load for next next height + _, err = state.LoadValidators(state.LastBlockHeight + 2) + assert.NotNil(err, "expected err") } From e2e87460444ba0c8097c416da9b91734f74af498 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 21 Aug 2017 18:11:16 -0400 Subject: [PATCH 02/92] rpc: historical validators --- rpc/client/httpclient.go | 6 ++++-- rpc/client/interface.go | 2 +- rpc/client/localclient.go | 4 ++-- rpc/client/mock/client.go | 4 ++-- rpc/client/rpc_test.go | 2 +- rpc/core/blocks.go | 4 +++- rpc/core/consensus.go | 17 +++++++++++++---- rpc/core/pipe.go | 5 ++++- rpc/core/routes.go | 2 +- 9 files changed, 31 insertions(+), 15 deletions(-) diff --git a/rpc/client/httpclient.go b/rpc/client/httpclient.go index 9623229e6..4e54cbf7d 100644 --- a/rpc/client/httpclient.go +++ b/rpc/client/httpclient.go @@ -174,9 +174,11 @@ func (c *HTTP) Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { return result, nil } -func (c *HTTP) Validators() (*ctypes.ResultValidators, error) { +func (c *HTTP) Validators(height *int) (*ctypes.ResultValidators, error) { result := new(ctypes.ResultValidators) - _, err := c.rpc.Call("validators", map[string]interface{}{}, result) + _, err := c.rpc.Call("validators", map[string]interface{}{ + "height": height, + }, result) if err != nil { return nil, errors.Wrap(err, "Validators") } diff --git a/rpc/client/interface.go b/rpc/client/interface.go index 0cd0f29bb..8845e5e5a 100644 --- a/rpc/client/interface.go +++ b/rpc/client/interface.go @@ -44,7 +44,7 @@ type ABCIClient interface { type SignClient interface { Block(height int) (*ctypes.ResultBlock, error) Commit(height int) (*ctypes.ResultCommit, error) - Validators() (*ctypes.ResultValidators, error) + Validators(height *int) (*ctypes.ResultValidators, error) Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) } diff --git a/rpc/client/localclient.go b/rpc/client/localclient.go index f4eb00d78..0d8500e1e 100644 --- a/rpc/client/localclient.go +++ b/rpc/client/localclient.go @@ -101,8 +101,8 @@ func (c Local) Commit(height int) (*ctypes.ResultCommit, error) { return core.Commit(height) } -func (c Local) Validators() (*ctypes.ResultValidators, error) { - return core.Validators() +func (c Local) Validators(height *int) (*ctypes.ResultValidators, error) { + return core.Validators(height) } func (c Local) Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { diff --git a/rpc/client/mock/client.go b/rpc/client/mock/client.go index bf8d78dce..be61b7af3 100644 --- a/rpc/client/mock/client.go +++ b/rpc/client/mock/client.go @@ -124,6 +124,6 @@ func (c Client) Commit(height int) (*ctypes.ResultCommit, error) { return core.Commit(height) } -func (c Client) Validators() (*ctypes.ResultValidators, error) { - return core.Validators() +func (c Client) Validators(height *int) (*ctypes.ResultValidators, error) { + return core.Validators(height) } diff --git a/rpc/client/rpc_test.go b/rpc/client/rpc_test.go index e82f8df42..304d2b906 100644 --- a/rpc/client/rpc_test.go +++ b/rpc/client/rpc_test.go @@ -87,7 +87,7 @@ func TestGenesisAndValidators(t *testing.T) { gval := gen.Genesis.Validators[0] // get the current validators - vals, err := c.Validators() + vals, err := c.Validators(nil) require.Nil(t, err, "%d: %+v", i, err) require.Equal(t, 1, len(vals.Validators)) val := vals.Validators[0] diff --git a/rpc/core/blocks.go b/rpc/core/blocks.go index 1f9e85422..5603cdd22 100644 --- a/rpc/core/blocks.go +++ b/rpc/core/blocks.go @@ -85,6 +85,7 @@ func BlockchainInfo(minHeight, maxHeight int) (*ctypes.ResultBlockchainInfo, err } // Get block at a given height. +// If no height is provided, it will fetch the latest block. // // ```shell // curl 'localhost:46657/block?height=10' @@ -196,7 +197,8 @@ func Block(height int) (*ctypes.ResultBlock, error) { return &ctypes.ResultBlock{blockMeta, block}, nil } -// Get block commit at a given height. +// Get block commit at a given height. If the height is left out, it +// If no height is provided, it will fetch the commit for the latest block. // // ```shell // curl 'localhost:46657/commit?height=11' diff --git a/rpc/core/consensus.go b/rpc/core/consensus.go index fdce29e48..55596ec26 100644 --- a/rpc/core/consensus.go +++ b/rpc/core/consensus.go @@ -7,7 +7,8 @@ import ( "github.com/tendermint/tendermint/types" ) -// Get current validators set along with a block height. +// Get the validator set at a give block height. +// If no height is provided, it will fetch the current validator set. // // ```shell // curl 'localhost:46657/validators' @@ -41,9 +42,17 @@ import ( // "jsonrpc": "2.0" // } // ``` -func Validators() (*ctypes.ResultValidators, error) { - blockHeight, validators := consensusState.GetValidators() - return &ctypes.ResultValidators{blockHeight, validators}, nil +func Validators(height *int) (*ctypes.ResultValidators, error) { + if height == nil { + blockHeight, validators := consensusState.GetValidators() + return &ctypes.ResultValidators{blockHeight, validators}, nil + } + state := consensusState.GetState() + validators, err := state.LoadValidators(*height) + if err != nil { + return nil, err + } + return &ctypes.ResultValidators{*height, validators.Validators}, nil } // Dump consensus state. diff --git a/rpc/core/pipe.go b/rpc/core/pipe.go index 92e5746a6..86c6c65e1 100644 --- a/rpc/core/pipe.go +++ b/rpc/core/pipe.go @@ -2,18 +2,21 @@ package core import ( crypto "github.com/tendermint/go-crypto" + "github.com/tendermint/tmlibs/log" + "github.com/tendermint/tendermint/consensus" p2p "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/proxy" + sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/state/txindex" "github.com/tendermint/tendermint/types" - "github.com/tendermint/tmlibs/log" ) //---------------------------------------------- // These interfaces are used by RPC and must be thread safe type Consensus interface { + GetState() *sm.State GetValidators() (int, []*types.Validator) GetRoundState() *consensus.RoundState } diff --git a/rpc/core/routes.go b/rpc/core/routes.go index 795acde9b..485f7a00f 100644 --- a/rpc/core/routes.go +++ b/rpc/core/routes.go @@ -18,7 +18,7 @@ var Routes = map[string]*rpc.RPCFunc{ "block": rpc.NewRPCFunc(Block, "height"), "commit": rpc.NewRPCFunc(Commit, "height"), "tx": rpc.NewRPCFunc(Tx, "hash,prove"), - "validators": rpc.NewRPCFunc(Validators, ""), + "validators": rpc.NewRPCFunc(Validators, "height"), "dump_consensus_state": rpc.NewRPCFunc(DumpConsensusState, ""), "unconfirmed_txs": rpc.NewRPCFunc(UnconfirmedTxs, ""), "num_unconfirmed_txs": rpc.NewRPCFunc(NumUnconfirmedTxs, ""), From f0f1ebe013ac4bae2b8769b9ff66704145f923ec Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 22 Aug 2017 17:42:23 -0400 Subject: [PATCH 03/92] rpc: Block and Commit take pointers; return latest on nil --- rpc/client/httpclient.go | 8 +++----- rpc/client/interface.go | 4 ++-- rpc/client/localclient.go | 4 ++-- rpc/client/mock/client.go | 4 ++-- rpc/client/rpc_test.go | 10 ++++++---- rpc/core/blocks.go | 24 ++++++++++++++++++++---- rpc/core/consensus.go | 10 ++++++---- 7 files changed, 41 insertions(+), 23 deletions(-) diff --git a/rpc/client/httpclient.go b/rpc/client/httpclient.go index 4e54cbf7d..7f29183d5 100644 --- a/rpc/client/httpclient.go +++ b/rpc/client/httpclient.go @@ -143,7 +143,7 @@ func (c *HTTP) Genesis() (*ctypes.ResultGenesis, error) { return result, nil } -func (c *HTTP) Block(height int) (*ctypes.ResultBlock, error) { +func (c *HTTP) Block(height *int) (*ctypes.ResultBlock, error) { result := new(ctypes.ResultBlock) _, err := c.rpc.Call("block", map[string]interface{}{"height": height}, result) if err != nil { @@ -152,7 +152,7 @@ func (c *HTTP) Block(height int) (*ctypes.ResultBlock, error) { return result, nil } -func (c *HTTP) Commit(height int) (*ctypes.ResultCommit, error) { +func (c *HTTP) Commit(height *int) (*ctypes.ResultCommit, error) { result := new(ctypes.ResultCommit) _, err := c.rpc.Call("commit", map[string]interface{}{"height": height}, result) if err != nil { @@ -176,9 +176,7 @@ func (c *HTTP) Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { func (c *HTTP) Validators(height *int) (*ctypes.ResultValidators, error) { result := new(ctypes.ResultValidators) - _, err := c.rpc.Call("validators", map[string]interface{}{ - "height": height, - }, result) + _, err := c.rpc.Call("validators", map[string]interface{}{"height": height}, result) if err != nil { return nil, errors.Wrap(err, "Validators") } diff --git a/rpc/client/interface.go b/rpc/client/interface.go index 8845e5e5a..ed7ccabaf 100644 --- a/rpc/client/interface.go +++ b/rpc/client/interface.go @@ -42,8 +42,8 @@ type ABCIClient interface { // SignClient groups together the interfaces need to get valid // signatures and prove anything about the chain type SignClient interface { - Block(height int) (*ctypes.ResultBlock, error) - Commit(height int) (*ctypes.ResultCommit, error) + Block(height *int) (*ctypes.ResultBlock, error) + Commit(height *int) (*ctypes.ResultCommit, error) Validators(height *int) (*ctypes.ResultValidators, error) Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) } diff --git a/rpc/client/localclient.go b/rpc/client/localclient.go index 0d8500e1e..134f935ca 100644 --- a/rpc/client/localclient.go +++ b/rpc/client/localclient.go @@ -93,11 +93,11 @@ func (c Local) Genesis() (*ctypes.ResultGenesis, error) { return core.Genesis() } -func (c Local) Block(height int) (*ctypes.ResultBlock, error) { +func (c Local) Block(height *int) (*ctypes.ResultBlock, error) { return core.Block(height) } -func (c Local) Commit(height int) (*ctypes.ResultCommit, error) { +func (c Local) Commit(height *int) (*ctypes.ResultCommit, error) { return core.Commit(height) } diff --git a/rpc/client/mock/client.go b/rpc/client/mock/client.go index be61b7af3..f32694edd 100644 --- a/rpc/client/mock/client.go +++ b/rpc/client/mock/client.go @@ -116,11 +116,11 @@ func (c Client) Genesis() (*ctypes.ResultGenesis, error) { return core.Genesis() } -func (c Client) Block(height int) (*ctypes.ResultBlock, error) { +func (c Client) Block(height *int) (*ctypes.ResultBlock, error) { return core.Block(height) } -func (c Client) Commit(height int) (*ctypes.ResultCommit, error) { +func (c Client) Commit(height *int) (*ctypes.ResultCommit, error) { return core.Commit(height) } diff --git a/rpc/client/rpc_test.go b/rpc/client/rpc_test.go index 304d2b906..fee92b4a9 100644 --- a/rpc/client/rpc_test.go +++ b/rpc/client/rpc_test.go @@ -110,7 +110,8 @@ func TestAppCalls(t *testing.T) { sh := s.LatestBlockHeight // look for the future - _, err = c.Block(sh + 2) + h := sh + 2 + _, err = c.Block(&h) assert.NotNil(err) // no block yet // write something @@ -137,7 +138,7 @@ func TestAppCalls(t *testing.T) { assert.EqualValues(tx, ptx.Tx) // and we can even check the block is added - block, err := c.Block(apph) + block, err := c.Block(&apph) require.Nil(err, "%d: %+v", i, err) appHash := block.BlockMeta.Header.AppHash assert.True(len(appHash) > 0) @@ -158,14 +159,15 @@ func TestAppCalls(t *testing.T) { } // and get the corresponding commit with the same apphash - commit, err := c.Commit(apph) + commit, err := c.Commit(&apph) require.Nil(err, "%d: %+v", i, err) cappHash := commit.Header.AppHash assert.Equal(appHash, cappHash) assert.NotNil(commit.Commit) // compare the commits (note Commit(2) has commit from Block(3)) - commit2, err := c.Commit(apph - 1) + h = apph - 1 + commit2, err := c.Commit(&h) require.Nil(err, "%d: %+v", i, err) assert.Equal(block.Block.LastCommit, commit2.Commit) diff --git a/rpc/core/blocks.go b/rpc/core/blocks.go index 5603cdd22..aa22da0f4 100644 --- a/rpc/core/blocks.go +++ b/rpc/core/blocks.go @@ -184,8 +184,16 @@ func BlockchainInfo(minHeight, maxHeight int) (*ctypes.ResultBlockchainInfo, err // "jsonrpc": "2.0" // } // ``` -func Block(height int) (*ctypes.ResultBlock, error) { - if height == 0 { +func Block(heightPtr *int) (*ctypes.ResultBlock, error) { + if heightPtr == nil { + height := blockStore.Height() + blockMeta := blockStore.LoadBlockMeta(height) + block := blockStore.LoadBlock(height) + return &ctypes.ResultBlock{blockMeta, block}, nil + } + + height := *heightPtr + if height <= 0 { return nil, fmt.Errorf("Height must be greater than 0") } if height > blockStore.Height() { @@ -267,8 +275,16 @@ func Block(height int) (*ctypes.ResultBlock, error) { // "jsonrpc": "2.0" // } // ``` -func Commit(height int) (*ctypes.ResultCommit, error) { - if height == 0 { +func Commit(heightPtr *int) (*ctypes.ResultCommit, error) { + if heightPtr == nil { + height := blockStore.Height() + header := blockStore.LoadBlockMeta(height).Header + commit := blockStore.LoadSeenCommit(height) + return &ctypes.ResultCommit{header, commit, false}, nil + } + + height := *heightPtr + if height <= 0 { return nil, fmt.Errorf("Height must be greater than 0") } storeHeight := blockStore.Height() diff --git a/rpc/core/consensus.go b/rpc/core/consensus.go index 55596ec26..ca50ff595 100644 --- a/rpc/core/consensus.go +++ b/rpc/core/consensus.go @@ -42,17 +42,19 @@ import ( // "jsonrpc": "2.0" // } // ``` -func Validators(height *int) (*ctypes.ResultValidators, error) { - if height == nil { +func Validators(heightPtr *int) (*ctypes.ResultValidators, error) { + if heightPtr == nil { blockHeight, validators := consensusState.GetValidators() return &ctypes.ResultValidators{blockHeight, validators}, nil } + + height := *heightPtr state := consensusState.GetState() - validators, err := state.LoadValidators(*height) + validators, err := state.LoadValidators(height) if err != nil { return nil, err } - return &ctypes.ResultValidators{*height, validators.Validators}, nil + return &ctypes.ResultValidators{height, validators.Validators}, nil } // Dump consensus state. From 9deb6473034e44ebf6022b95570ad9f51d27768c Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 4 Sep 2017 18:27:04 -0400 Subject: [PATCH 04/92] fixes from review --- rpc/core/blocks.go | 2 +- rpc/core/consensus.go | 2 +- state/execution.go | 5 +-- state/state.go | 87 +++++++++++++++++++------------------------ state/state_test.go | 8 ++-- 5 files changed, 46 insertions(+), 58 deletions(-) diff --git a/rpc/core/blocks.go b/rpc/core/blocks.go index aa22da0f4..17b27303f 100644 --- a/rpc/core/blocks.go +++ b/rpc/core/blocks.go @@ -205,7 +205,7 @@ func Block(heightPtr *int) (*ctypes.ResultBlock, error) { return &ctypes.ResultBlock{blockMeta, block}, nil } -// Get block commit at a given height. If the height is left out, it +// Get block commit at a given height. // If no height is provided, it will fetch the commit for the latest block. // // ```shell diff --git a/rpc/core/consensus.go b/rpc/core/consensus.go index ca50ff595..848288447 100644 --- a/rpc/core/consensus.go +++ b/rpc/core/consensus.go @@ -7,7 +7,7 @@ import ( "github.com/tendermint/tendermint/types" ) -// Get the validator set at a give block height. +// Get the validator set at the given block height. // If no height is provided, it will fetch the current validator set. // // ```shell diff --git a/state/execution.go b/state/execution.go index 869d28d48..1785f6025 100644 --- a/state/execution.go +++ b/state/execution.go @@ -231,9 +231,6 @@ func (s *State) ApplyBlock(eventCache types.Fireable, proxyAppConn proxy.AppConn // now update the block and validators s.SetBlockAndValidators(block.Header, partsHeader, abciResponses) - // save the validators for the next block now that we know them - s.SaveValidators() - // lock mempool, commit state, update mempoool err = s.CommitStateUpdateMempool(proxyAppConn, block, mempool) if err != nil { @@ -242,7 +239,7 @@ func (s *State) ApplyBlock(eventCache types.Fireable, proxyAppConn proxy.AppConn fail.Fail() // XXX - // save the state + // save the state and the validators s.Save() return nil diff --git a/state/state.go b/state/state.go index 6106891b5..f2b02309f 100644 --- a/state/state.go +++ b/state/state.go @@ -51,10 +51,9 @@ type State struct { // AppHash is updated after Commit AppHash []byte - // XXX: do we need this json tag ? TxIndexer txindex.TxIndexer `json:"-"` // Transaction indexer. - lastHeightValidatorsChanged int + LastHeightValidatorsChanged int logger log.Logger } @@ -79,13 +78,6 @@ func loadState(db dbm.DB, key []byte) *State { // TODO: ensure that buf is completely read. } - v := s.loadValidators(s.LastBlockHeight) - if v != nil { - s.lastHeightValidatorsChanged = v.LastHeightChanged - } else { - s.lastHeightValidatorsChanged = 1 - } - return s } @@ -97,19 +89,18 @@ func (s *State) SetLogger(l log.Logger) { // Copy makes a copy of the State for mutating. func (s *State) Copy() *State { return &State{ - db: s.db, - GenesisDoc: s.GenesisDoc, - ChainID: s.ChainID, - LastBlockHeight: s.LastBlockHeight, - LastBlockID: s.LastBlockID, - LastBlockTime: s.LastBlockTime, - Validators: s.Validators.Copy(), - LastValidators: s.LastValidators.Copy(), - AppHash: s.AppHash, - TxIndexer: s.TxIndexer, // pointer here, not value - logger: s.logger, - - lastHeightValidatorsChanged: s.lastHeightValidatorsChanged, + db: s.db, + GenesisDoc: s.GenesisDoc, + ChainID: s.ChainID, + LastBlockHeight: s.LastBlockHeight, + LastBlockID: s.LastBlockID, + LastBlockTime: s.LastBlockTime, + Validators: s.Validators.Copy(), + LastValidators: s.LastValidators.Copy(), + AppHash: s.AppHash, + TxIndexer: s.TxIndexer, // pointer here, not value + LastHeightValidatorsChanged: s.LastHeightValidatorsChanged, + logger: s.logger, } } @@ -117,6 +108,7 @@ func (s *State) Copy() *State { func (s *State) Save() { s.mtx.Lock() defer s.mtx.Unlock() + s.saveValidatorsInfo() s.db.SetSync(stateKey, s.Bytes()) } @@ -143,22 +135,6 @@ func (s *State) LoadABCIResponses() *ABCIResponses { return abciResponses } -// SaveValidators persists the validator set for the next block to disk. -// It should be called after the validator set is updated with the results of EndBlock. -// If the validator set did not change after processing the latest block, -// only the last height for which the validators changed is persisted. -func (s *State) SaveValidators() { - lastHeight := s.lastHeightValidatorsChanged - nextHeight := s.LastBlockHeight + 1 - v := &Validators{ - LastHeightChanged: lastHeight, - } - if lastHeight == nextHeight { - v.ValidatorSet = s.Validators - } - s.db.SetSync(calcValidatorsKey(nextHeight), v.Bytes()) -} - // LoadValidators loads the ValidatorSet for a given height. func (s *State) LoadValidators(height int) (*types.ValidatorSet, error) { v := s.loadValidators(height) @@ -176,13 +152,13 @@ func (s *State) LoadValidators(height int) (*types.ValidatorSet, error) { return v.ValidatorSet, nil } -func (s *State) loadValidators(height int) *Validators { +func (s *State) loadValidators(height int) *ValidatorsInfo { buf := s.db.Get(calcValidatorsKey(height)) if len(buf) == 0 { return nil } - v := new(Validators) + v := new(ValidatorsInfo) r, n, err := bytes.NewReader(buf), new(int), new(error) wire.ReadBinaryPtr(v, r, 0, n, err) if *err != nil { @@ -193,6 +169,22 @@ func (s *State) loadValidators(height int) *Validators { return v } +// saveValidatorsInfo persists the validator set for the next block to disk. +// It should be called after the validator set is updated with the results of EndBlock. +// If the validator set did not change after processing the latest block, +// only the last height for which the validators changed is persisted. +func (s *State) saveValidatorsInfo() { + changeHeight := s.LastHeightValidatorsChanged + nextHeight := s.LastBlockHeight + 1 + vi := &ValidatorsInfo{ + LastHeightChanged: changeHeight, + } + if changeHeight == nextHeight { + vi.ValidatorSet = s.Validators + } + s.db.SetSync(calcValidatorsKey(nextHeight), vi.Bytes()) +} + // Equals returns true if the States are identical. func (s *State) Equals(s2 *State) bool { return bytes.Equal(s.Bytes(), s2.Bytes()) @@ -219,7 +211,7 @@ func (s *State) SetBlockAndValidators(header *types.Header, blockPartsHeader typ // TODO: err or carry on? } // change results from this height but only applies to the next height - s.lastHeightValidatorsChanged = header.Height + 1 + s.LastHeightValidatorsChanged = header.Height + 1 } // Update validator accums and set state variables @@ -254,7 +246,6 @@ func GetState(stateDB dbm.DB, genesisFile string) *State { state := LoadState(stateDB) if state == nil { state = MakeGenesisStateFromFile(stateDB, genesisFile) - state.SaveValidators() // save the validators right away for height 1 state.Save() } @@ -290,15 +281,15 @@ func (a *ABCIResponses) Bytes() []byte { //----------------------------------------------------------------------------- -// Validators represents the latest validator set, or the last time it changed -type Validators struct { +// ValidatorsInfo represents the latest validator set, or the last time it changed +type ValidatorsInfo struct { ValidatorSet *types.ValidatorSet LastHeightChanged int } -// Bytes serializes the Validators using go-wire -func (v *Validators) Bytes() []byte { - return wire.BinaryBytes(*v) +// Bytes serializes the ValidatorsInfo using go-wire +func (vi *ValidatorsInfo) Bytes() []byte { + return wire.BinaryBytes(*vi) } //----------------------------------------------------------------------------- @@ -356,6 +347,6 @@ func MakeGenesisState(db dbm.DB, genDoc *types.GenesisDoc) *State { LastValidators: types.NewValidatorSet(nil), AppHash: genDoc.AppHash, TxIndexer: &null.TxIndex{}, // we do not need indexer during replay and in tests - lastHeightValidatorsChanged: 1, + LastHeightValidatorsChanged: 1, } } diff --git a/state/state_test.go b/state/state_test.go index 22543cc17..ddcb98c22 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -83,7 +83,7 @@ func TestValidatorsSaveLoad(t *testing.T) { // cant load anything for height 0 v, err := state.LoadValidators(0) - assert.NotNil(err, "expected err at height 0") + assert.IsType(ErrNoValSetForHeight{}, err, "expected err at height 0") // should be able to load for height 1 v, err = state.LoadValidators(1) @@ -92,19 +92,19 @@ func TestValidatorsSaveLoad(t *testing.T) { // increment height, save; should be able to load for next height state.LastBlockHeight += 1 - state.SaveValidators() + state.saveValidatorsInfo() v, err = state.LoadValidators(state.LastBlockHeight + 1) assert.Nil(err, "expected no err") assert.Equal(v.Hash(), state.Validators.Hash(), "expected validator hashes to match") // increment height, save; should be able to load for next height state.LastBlockHeight += 10 - state.SaveValidators() + state.saveValidatorsInfo() v, err = state.LoadValidators(state.LastBlockHeight + 1) assert.Nil(err, "expected no err") assert.Equal(v.Hash(), state.Validators.Hash(), "expected validator hashes to match") // should be able to load for next next height _, err = state.LoadValidators(state.LastBlockHeight + 2) - assert.NotNil(err, "expected err") + assert.IsType(ErrNoValSetForHeight{}, err, "expected err at unknown height") } From fae0603413e39873e04eae0abda16f812e34a8aa Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 5 Sep 2017 21:57:36 -0400 Subject: [PATCH 05/92] more fixes from review --- state/state.go | 10 ++++- state/state_test.go | 91 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 98 insertions(+), 3 deletions(-) diff --git a/state/state.go b/state/state.go index f2b02309f..72c3e64da 100644 --- a/state/state.go +++ b/state/state.go @@ -2,6 +2,7 @@ package state import ( "bytes" + "fmt" "io/ioutil" "sync" "time" @@ -53,6 +54,10 @@ type State struct { TxIndexer txindex.TxIndexer `json:"-"` // Transaction indexer. + // When a block returns a validator set change via EndBlock, + // the change only applies to the next block. + // So, if s.LastBlockHeight causes a valset change, + // we set s.LastHeightValidatorsChanged = s.LastBlockHeight + 1 LastHeightValidatorsChanged int logger log.Logger @@ -145,7 +150,8 @@ func (s *State) LoadValidators(height int) (*types.ValidatorSet, error) { if v.ValidatorSet == nil { v = s.loadValidators(v.LastHeightChanged) if v == nil { - return nil, ErrNoValSetForHeight{height} + cmn.PanicSanity(fmt.Sprintf(`Couldn't find validators at + height %d as last changed from height %d`, v.LastHeightChanged, height)) } } @@ -170,7 +176,7 @@ func (s *State) loadValidators(height int) *ValidatorsInfo { } // saveValidatorsInfo persists the validator set for the next block to disk. -// It should be called after the validator set is updated with the results of EndBlock. +// It should be called from s.Save(), right before the state itself is persisted. // If the validator set did not change after processing the latest block, // only the last height for which the validators changed is persisted. func (s *State) saveValidatorsInfo() { diff --git a/state/state_test.go b/state/state_test.go index ddcb98c22..713d76f80 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -1,6 +1,8 @@ package state import ( + "bytes" + "fmt" "testing" "github.com/stretchr/testify/assert" @@ -12,6 +14,7 @@ import ( "github.com/tendermint/tmlibs/log" cfg "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/types" ) func TestStateCopyEquals(t *testing.T) { @@ -73,7 +76,7 @@ func TestABCIResponsesSaveLoad(t *testing.T) { assert.Equal(abciResponses, abciResponses2, cmn.Fmt("ABCIResponses don't match: Got %v, Expected %v", abciResponses2, abciResponses)) } -func TestValidatorsSaveLoad(t *testing.T) { +func TestValidatorSimpleSaveLoad(t *testing.T) { assert := assert.New(t) config := cfg.ResetTestRoot("state_") // Get State db @@ -108,3 +111,89 @@ func TestValidatorsSaveLoad(t *testing.T) { _, err = state.LoadValidators(state.LastBlockHeight + 2) assert.IsType(ErrNoValSetForHeight{}, err, "expected err at unknown height") } + +func TestValidatorChangesSaveLoad(t *testing.T) { + assert := assert.New(t) + config := cfg.ResetTestRoot("state_") + // Get State db + stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir()) + state := GetState(stateDB, config.GenesisFile()) + state.SetLogger(log.TestingLogger()) + + // change vals at these heights + changeHeights := []int{1, 2, 4, 5, 10, 15, 16, 17, 20} + N := len(changeHeights) + + // each valset is just one validator. + // create list of them + pubkeys := make([]crypto.PubKey, N+1) + pubkeys[0] = state.GenesisDoc.Validators[0].PubKey + for i := 1; i < N+1; i++ { + pubkeys[i] = crypto.GenPrivKeyEd25519().PubKey() + } + + // build the validator history by running SetBlockAndValidators + // with the right validator set for each height + highestHeight := changeHeights[N-1] + 5 + changeIndex := 0 + pubkey := pubkeys[changeIndex] + for i := 1; i < highestHeight; i++ { + // when we get to a change height, + // use the next pubkey + if changeIndex < len(changeHeights) && i == changeHeights[changeIndex] { + changeIndex += 1 + pubkey = pubkeys[changeIndex] + } + header, parts, responses := makeHeaderPartsResponses(state, i, pubkey) + state.SetBlockAndValidators(header, parts, responses) + state.saveValidatorsInfo() + } + + // make all the test cases by using the same validator until after the change + testCases := make([]valChangeTestCase, highestHeight) + changeIndex = 0 + pubkey = pubkeys[changeIndex] + for i := 1; i < highestHeight+1; i++ { + // we we get to the height after a change height + // use the next pubkey (note our counter starts at 0 this time) + if changeIndex < len(changeHeights) && i == changeHeights[changeIndex]+1 { + changeIndex += 1 + pubkey = pubkeys[changeIndex] + } + testCases[i-1] = valChangeTestCase{i, pubkey} + } + + for _, testCase := range testCases { + v, err := state.LoadValidators(testCase.height) + assert.Nil(err, fmt.Sprintf("expected no err at height %d", testCase.height)) + assert.Equal(v.Size(), 1, "validator set size is greater than 1: %d", v.Size()) + addr, _ := v.GetByIndex(0) + + assert.Equal(addr, testCase.vals.Address(), fmt.Sprintf("unexpected pubkey at height %d", testCase.height)) + } +} + +func makeHeaderPartsResponses(state *State, height int, pubkey crypto.PubKey) (*types.Header, types.PartSetHeader, *ABCIResponses) { + block := makeBlock(height, state) + _, val := state.Validators.GetByIndex(0) + abciResponses := &ABCIResponses{ + Height: height, + } + + // if the pubkey is new, remove the old and add the new + if !bytes.Equal(pubkey.Bytes(), val.PubKey.Bytes()) { + abciResponses.EndBlock = abci.ResponseEndBlock{ + Diffs: []*abci.Validator{ + {val.PubKey.Bytes(), 0}, + {pubkey.Bytes(), 10}, + }, + } + } + + return block.Header, types.PartSetHeader{}, abciResponses +} + +type valChangeTestCase struct { + height int + vals crypto.PubKey +} From 399fb9aa706b4f71e6dcda457fa67a6431448d29 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 6 Sep 2017 01:53:41 -0400 Subject: [PATCH 06/92] consensus: remove support for replay by #HEIGHT. closes #567 --- consensus/replay.go | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/consensus/replay.go b/consensus/replay.go index 71dc81b5a..2104ffd5e 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -115,35 +115,13 @@ func (cs *ConsensusState) catchupReplay(csHeight int) error { gr, found, err = cs.wal.group.Search("#ENDHEIGHT: ", makeHeightSearchFunc(csHeight-1)) if err == io.EOF { cs.Logger.Error("Replay: wal.group.Search returned EOF", "#ENDHEIGHT", csHeight-1) - // if we upgraded from 0.9 to 0.9.1, we may have #HEIGHT instead - // TODO (0.10.0): remove this - gr, found, err = cs.wal.group.Search("#HEIGHT: ", makeHeightSearchFunc(csHeight)) - if err == io.EOF { - cs.Logger.Error("Replay: wal.group.Search returned EOF", "#HEIGHT", csHeight) - return nil - } else if err != nil { - return err - } } else if err != nil { return err } else { defer gr.Close() } if !found { - // if we upgraded from 0.9 to 0.9.1, we may have #HEIGHT instead - // TODO (0.10.0): remove this - gr, _, err = cs.wal.group.Search("#HEIGHT: ", makeHeightSearchFunc(csHeight)) - if err == io.EOF { - cs.Logger.Error("Replay: wal.group.Search returned EOF", "#HEIGHT", csHeight) - return nil - } else if err != nil { - return err - } else { - defer gr.Close() - } - - // TODO (0.10.0): uncomment - // return errors.New(cmn.Fmt("Cannot replay height %d. WAL does not contain #ENDHEIGHT for %d.", csHeight, csHeight-1)) + return errors.New(cmn.Fmt("Cannot replay height %d. WAL does not contain #ENDHEIGHT for %d.", csHeight, csHeight-1)) } cs.Logger.Info("Catchup by replaying consensus messages", "height", csHeight) From aa78fc14b5824725f8a14dd2c2ad727a9ab89cc3 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 6 Sep 2017 01:57:21 -0400 Subject: [PATCH 07/92] cmd: dont wait for genesis. closes #562 --- cmd/tendermint/commands/run_node.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index e92911da2..6740df289 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -2,13 +2,11 @@ package commands import ( "fmt" - "time" "github.com/spf13/cobra" "github.com/tendermint/tendermint/node" "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" ) var runNodeCmd = &cobra.Command{ @@ -58,16 +56,7 @@ func AddNodeFlags(cmd *cobra.Command) { // with their custom priv validator and/or custom proxy.ClientCreator func runNode(cmd *cobra.Command, args []string) error { - // Wait until the genesis doc becomes available - // This is for Mintnet compatibility. - // TODO: If Mintnet gets deprecated or genesis_file is - // always available, remove. genDocFile := config.GenesisFile() - for !cmn.FileExists(genDocFile) { - logger.Info(cmn.Fmt("Waiting for genesis file %v...", genDocFile)) - time.Sleep(time.Second) - } - genDoc, err := types.GenesisDocFromFile(genDocFile) if err != nil { return err From c48e772115ed8ae382a800f2cf869e26455b0176 Mon Sep 17 00:00:00 2001 From: Emmanuel Odeke Date: Fri, 25 Aug 2017 19:35:00 -0600 Subject: [PATCH 08/92] p2p: fully test PeerSet, more docs, parallelize PeerSet tests * Full test PeerSet and check its concurrent guarantees * Improve the doc for PeerSet.Has and remove unnecessary defer for a path that sets a variable, make it fast anyways. * Parallelize PeerSet tests with t.Parallel() * Document functions in peer_set.go more. --- p2p/peer_set.go | 20 +++++--- p2p/peer_set_test.go | 106 +++++++++++++++++++++++++++++++++++++------ 2 files changed, 106 insertions(+), 20 deletions(-) diff --git a/p2p/peer_set.go b/p2p/peer_set.go index c5206d2d5..19531731f 100644 --- a/p2p/peer_set.go +++ b/p2p/peer_set.go @@ -27,6 +27,7 @@ type peerSetItem struct { index int } +// NewPeerSet creates a new peerSet with a list of initial capacity of 256 items. func NewPeerSet() *PeerSet { return &PeerSet{ lookup: make(map[string]*peerSetItem), @@ -34,7 +35,8 @@ func NewPeerSet() *PeerSet { } } -// Returns false if peer with key (PubKeyEd25519) is already set +// Add returns false if the peer's Key (PubKeyEd25519) is already memoized. +// If the peer was already added, it returns ErrSwitchDuplicatePeer. func (ps *PeerSet) Add(peer *Peer) error { ps.mtx.Lock() defer ps.mtx.Unlock() @@ -50,13 +52,16 @@ func (ps *PeerSet) Add(peer *Peer) error { return nil } +// Has returns true iff the peerset contains +// the peer referred to by this peerKey. func (ps *PeerSet) Has(peerKey string) bool { ps.mtx.Lock() - defer ps.mtx.Unlock() _, ok := ps.lookup[peerKey] + ps.mtx.Unlock() return ok } +// Get looks up a peer by the provided peerKey. func (ps *PeerSet) Get(peerKey string) *Peer { ps.mtx.Lock() defer ps.mtx.Unlock() @@ -68,6 +73,7 @@ func (ps *PeerSet) Get(peerKey string) *Peer { } } +// Remove discards peer by its Key, if the peer was previously memoized. func (ps *PeerSet) Remove(peer *Peer) { ps.mtx.Lock() defer ps.mtx.Unlock() @@ -77,8 +83,8 @@ func (ps *PeerSet) Remove(peer *Peer) { } index := item.index - // Copy the list but without the last element. - // (we must copy because we're mutating the list) + // Create a new copy of the list but with one less item. + // (we must copy because we'll be mutating the list). newList := make([]*Peer, len(ps.list)-1) copy(newList, ps.list) // If it's the last peer, that's an easy special case. @@ -88,7 +94,7 @@ func (ps *PeerSet) Remove(peer *Peer) { return } - // Move the last item from ps.list to "index" in list. + // Replace the popped item with the last item in the old list. lastPeer := ps.list[len(ps.list)-1] lastPeerKey := lastPeer.Key lastPeerItem := ps.lookup[lastPeerKey] @@ -96,16 +102,16 @@ func (ps *PeerSet) Remove(peer *Peer) { lastPeerItem.index = index ps.list = newList delete(ps.lookup, peer.Key) - } +// Size returns the number of unique items in the peerSet. func (ps *PeerSet) Size() int { ps.mtx.Lock() defer ps.mtx.Unlock() return len(ps.list) } -// threadsafe list of peers. +// List returns the threadsafe list of peers. func (ps *PeerSet) List() []*Peer { ps.mtx.Lock() defer ps.mtx.Unlock() diff --git a/p2p/peer_set_test.go b/p2p/peer_set_test.go index 9214b2eb4..309b7e970 100644 --- a/p2p/peer_set_test.go +++ b/p2p/peer_set_test.go @@ -2,8 +2,11 @@ package p2p import ( "math/rand" + "sync" "testing" + "github.com/stretchr/testify/assert" + cmn "github.com/tendermint/tmlibs/common" ) @@ -18,28 +21,47 @@ func randPeer() *Peer { } } -func TestAddRemoveOne(t *testing.T) { +func TestPeerSetAddRemoveOne(t *testing.T) { + t.Parallel() peerSet := NewPeerSet() - peer := randPeer() - err := peerSet.Add(peer) - if err != nil { - t.Errorf("Failed to add new peer") + var peerList []*Peer + for i := 0; i < 5; i++ { + p := randPeer() + peerSet.Add(p) + peerList = append(peerList, p) } - if peerSet.Size() != 1 { - t.Errorf("Failed to add new peer and increment size") + + n := len(peerList) + // 1. Test removing from the front + for i, peerAtFront := range peerList { + peerSet.Remove(peerAtFront) + wantSize := n - i - 1 + for j := 0; j < 2; j++ { + assert.Equal(t, false, peerSet.Has(peerAtFront.Key), "#%d Run #%d: failed to remove peer", i, j) + assert.Equal(t, wantSize, peerSet.Size(), "#%d Run #%d: failed to remove peer and decrement size", i, j) + // Test the route of removing the now non-existent element + peerSet.Remove(peerAtFront) + } } - peerSet.Remove(peer) - if peerSet.Has(peer.Key) { - t.Errorf("Failed to remove peer") + // 2. Next we are testing removing the peer at the end + // a) Replenish the peerSet + for _, peer := range peerList { + peerSet.Add(peer) } - if peerSet.Size() != 0 { - t.Errorf("Failed to remove peer and decrement size") + + // b) In reverse, remove each element + for i := n - 1; i >= 0; i-- { + peerAtEnd := peerList[i] + peerSet.Remove(peerAtEnd) + assert.Equal(t, false, peerSet.Has(peerAtEnd.Key), "#%d: failed to remove item at end", i) + assert.Equal(t, i, peerSet.Size(), "#%d: differing sizes after peerSet.Remove(atEndPeer)", i) } } -func TestAddRemoveMany(t *testing.T) { +func TestPeerSetAddRemoveMany(t *testing.T) { + t.Parallel() peerSet := NewPeerSet() peers := []*Peer{} @@ -65,3 +87,61 @@ func TestAddRemoveMany(t *testing.T) { } } } + +func TestPeerSetAddDuplicate(t *testing.T) { + t.Parallel() + peerSet := NewPeerSet() + peer := randPeer() + + n := 20 + errsChan := make(chan error) + // Add the same asynchronously to test the + // concurrent guarantees of our APIs, and + // our expectation in the end is that only + // one addition succeeded, but the rest are + // instances of ErrSwitchDuplicatePeer. + for i := 0; i < n; i++ { + go func() { + errsChan <- peerSet.Add(peer) + }() + } + + // Now collect and tally the results + errsTally := make(map[error]int) + for i := 0; i < n; i++ { + err := <-errsChan + errsTally[err] += 1 + } + + // Our next procedure is to ensure that only one addition + // succeeded and that the rest are each ErrSwitchDuplicatePeer. + wantErrCount, gotErrCount := n-1, errsTally[ErrSwitchDuplicatePeer] + assert.Equal(t, wantErrCount, gotErrCount, "invalid ErrSwitchDuplicatePeer count") + + wantNilErrCount, gotNilErrCount := 1, errsTally[nil] + assert.Equal(t, wantNilErrCount, gotNilErrCount, "invalid nil errCount") +} + +func TestPeerSetGet(t *testing.T) { + t.Parallel() + peerSet := NewPeerSet() + peer := randPeer() + assert.Nil(t, peerSet.Get(peer.Key), "expecting a nil lookup, before .Add") + + if err := peerSet.Add(peer); err != nil { + t.Fatalf("Failed to add new peer: %v", err) + } + + var wg sync.WaitGroup + for i := 0; i < 10; i++ { + // Add them asynchronously to test the + // concurrent guarantees of our APIs. + wg.Add(1) + go func(i int) { + defer wg.Done() + got, want := peerSet.Get(peer.Key), peer + assert.Equal(t, got, want, "#%d: got=%v want=%v", i, got, want) + }(i) + } + wg.Wait() +} From 54c63726b03cbba067d10fa6b69b885c85678d17 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 6 Sep 2017 16:40:21 -0400 Subject: [PATCH 09/92] p2p: minor comment fixes --- p2p/peer_set.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/p2p/peer_set.go b/p2p/peer_set.go index 19531731f..a5a443d87 100644 --- a/p2p/peer_set.go +++ b/p2p/peer_set.go @@ -35,8 +35,8 @@ func NewPeerSet() *PeerSet { } } -// Add returns false if the peer's Key (PubKeyEd25519) is already memoized. -// If the peer was already added, it returns ErrSwitchDuplicatePeer. +// Add adds the peer to the PeerSet. +// It returns ErrSwitchDuplicatePeer if the peer is already present. func (ps *PeerSet) Add(peer *Peer) error { ps.mtx.Lock() defer ps.mtx.Unlock() @@ -52,7 +52,7 @@ func (ps *PeerSet) Add(peer *Peer) error { return nil } -// Has returns true iff the peerset contains +// Has returns true iff the PeerSet contains // the peer referred to by this peerKey. func (ps *PeerSet) Has(peerKey string) bool { ps.mtx.Lock() From 340d273f8387f708fde030c7f505df07055a07a5 Mon Sep 17 00:00:00 2001 From: GaryTomato Date: Mon, 11 Sep 2017 11:34:36 -0400 Subject: [PATCH 10/92] Started expanding "automated deployments" --- docs/deploy-testnets.rst | 51 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/docs/deploy-testnets.rst b/docs/deploy-testnets.rst index 66cee8cd4..e1aa7074d 100644 --- a/docs/deploy-testnets.rst +++ b/docs/deploy-testnets.rst @@ -40,8 +40,51 @@ Automated Deployments --------------------- While the manual deployment is easy enough, an automated deployment is -always better. For this, we have the `mintnet-kubernetes -tool `__, -which allows us to automate the deployment of a Tendermint network on an -already provisioned kubernetes cluster. And for simple provisioning of kubernetes +usually quicker. The below examples show different tools that can be used +for automated deployments. + +Automated Deployment using Kubernetes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The `mintnet-kubernetes tool `__ +allows automating the deployment of a Tendermint network on an already +provisioned kubernetes cluster. For simple provisioning of a kubernetes cluster, check out the `Google Cloud Platform `__. + +Automated Deployment using Terraform and Ansible +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The `terraform-digitalocean tool `__ +allows creating a set of servers on the DigitalOcean cloud. + +The `ansible playbooks `__ +allow creating and managing a ``basecoin`` or ``ethermint`` testnet on provisioned servers. + +Package Deployment on Linux for developers +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The ``tendermint`` and ``basecoin`` applications can be installed from RPM or DEB packages on +Linux machines for development purposes. The packages are configured to be validators on the +one-node network that the machine represents. The services are not started after installation, +this way giving an opportunity to reconfigure the applications before starting. + +The Ansible playbooks in the previous section use this repository to install ``basecoin``. +After installation, additional steps are executed to make sure that the multi-node testnet has +the right configuration before start. + +Install from the CentOS/RedHat repository: + +:: + + rpm --import https://tendermint-packages.interblock.io/centos/7/os/x86_64/RPM-GPG-KEY-Tendermint + wget -O /etc/yum.repos.d/tendermint.repo https://tendermint-packages.interblock.io/centos/7/os/x86_64/tendermint.repo + yum install basecoin + +Install from the Debian/Ubuntu repository: + +:: + + wget -O - https://tendermint-packages.interblock.io/centos/7/os/x86_64/RPM-GPG-KEY-Tendermint | apt-key add - + wget -O /etc/apt/sources.list.d/tendermint.list https://tendermint-packages.interblock.io/debian/tendermint.list + apt-get update && apt-get install basecoin + From 2a6e71a753c14b2563714e69d2e97ea5472dc8b0 Mon Sep 17 00:00:00 2001 From: Adrian Brink Date: Thu, 31 Aug 2017 13:41:28 +0200 Subject: [PATCH 11/92] Reformat tests to extract common setup --- state/state_test.go | 59 +++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 31 deletions(-) diff --git a/state/state_test.go b/state/state_test.go index 713d76f80..2576b35d5 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -8,7 +8,9 @@ import ( "github.com/stretchr/testify/assert" abci "github.com/tendermint/abci/types" + crypto "github.com/tendermint/go-crypto" + cmn "github.com/tendermint/tmlibs/common" dbm "github.com/tendermint/tmlibs/db" "github.com/tendermint/tmlibs/log" @@ -17,31 +19,35 @@ import ( "github.com/tendermint/tendermint/types" ) -func TestStateCopyEquals(t *testing.T) { - assert := assert.New(t) - config := cfg.ResetTestRoot("state_") +func setupTestCase(t *testing.T) (func(t *testing.T), dbm.DB, *State) { - // Get State db + config := cfg.ResetTestRoot("state_") stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir()) state := GetState(stateDB, config.GenesisFile()) state.SetLogger(log.TestingLogger()) + tearDown := func(t *testing.T) {} + + return tearDown, stateDB, state +} + +func TestStateCopy(t *testing.T) { + tearDown, _, state := setupTestCase(t) + defer tearDown(t) + assert := assert.New(t) + stateCopy := state.Copy() assert.True(state.Equals(stateCopy), cmn.Fmt("expected state and its copy to be identical. got %v\n expected %v\n", stateCopy, state)) - stateCopy.LastBlockHeight += 1 + stateCopy.LastBlockHeight++ assert.False(state.Equals(stateCopy), cmn.Fmt("expected states to be different. got same %v", state)) } func TestStateSaveLoad(t *testing.T) { - assert := assert.New(t) - config := cfg.ResetTestRoot("state_") - // Get State db - stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir()) - state := GetState(stateDB, config.GenesisFile()) - state.SetLogger(log.TestingLogger()) + tearDown, stateDB, state := setupTestCase(t) + defer tearDown(t) - state.LastBlockHeight += 1 + state.LastBlockHeight++ state.Save() loadedState := LoadState(stateDB) @@ -49,14 +55,11 @@ func TestStateSaveLoad(t *testing.T) { } func TestABCIResponsesSaveLoad(t *testing.T) { + tearDown, _, state := setupTestCase(t) + defer tearDown(t) assert := assert.New(t) - config := cfg.ResetTestRoot("state_") - stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir()) - state := GetState(stateDB, config.GenesisFile()) - state.SetLogger(log.TestingLogger()) - - state.LastBlockHeight += 1 + state.LastBlockHeight++ // build mock responses block := makeBlock(2, state) @@ -77,12 +80,9 @@ func TestABCIResponsesSaveLoad(t *testing.T) { } func TestValidatorSimpleSaveLoad(t *testing.T) { + tearDown, _, state := setupTestCase(t) + defer tearDown(t) assert := assert.New(t) - config := cfg.ResetTestRoot("state_") - // Get State db - stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir()) - state := GetState(stateDB, config.GenesisFile()) - state.SetLogger(log.TestingLogger()) // cant load anything for height 0 v, err := state.LoadValidators(0) @@ -94,7 +94,7 @@ func TestValidatorSimpleSaveLoad(t *testing.T) { assert.Equal(v.Hash(), state.Validators.Hash(), "expected validator hashes to match") // increment height, save; should be able to load for next height - state.LastBlockHeight += 1 + state.LastBlockHeight++ state.saveValidatorsInfo() v, err = state.LoadValidators(state.LastBlockHeight + 1) assert.Nil(err, "expected no err") @@ -113,12 +113,9 @@ func TestValidatorSimpleSaveLoad(t *testing.T) { } func TestValidatorChangesSaveLoad(t *testing.T) { + tearDown, _, state := setupTestCase(t) + defer tearDown(t) assert := assert.New(t) - config := cfg.ResetTestRoot("state_") - // Get State db - stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir()) - state := GetState(stateDB, config.GenesisFile()) - state.SetLogger(log.TestingLogger()) // change vals at these heights changeHeights := []int{1, 2, 4, 5, 10, 15, 16, 17, 20} @@ -141,7 +138,7 @@ func TestValidatorChangesSaveLoad(t *testing.T) { // when we get to a change height, // use the next pubkey if changeIndex < len(changeHeights) && i == changeHeights[changeIndex] { - changeIndex += 1 + changeIndex++ pubkey = pubkeys[changeIndex] } header, parts, responses := makeHeaderPartsResponses(state, i, pubkey) @@ -157,7 +154,7 @@ func TestValidatorChangesSaveLoad(t *testing.T) { // we we get to the height after a change height // use the next pubkey (note our counter starts at 0 this time) if changeIndex < len(changeHeights) && i == changeHeights[changeIndex]+1 { - changeIndex += 1 + changeIndex++ pubkey = pubkeys[changeIndex] } testCases[i-1] = valChangeTestCase{i, pubkey} From e2c3cc9685cdb7cdac7b18780928ff3dba9ff1b3 Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Tue, 12 Sep 2017 10:57:34 -0400 Subject: [PATCH 12/92] docs: give index a Tools section --- docs/index.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 3d278f60c..56b1163c7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -23,9 +23,16 @@ Tendermint 101 introduction.rst install.rst getting-started.rst - deploy-testnets.rst using-tendermint.rst +Tendermint Tools +---------------- + +.. toctree:: + :maxdepth: 2 + + deploy-testnets.rst + Tendermint 102 -------------- From 8eda3efa28dd7d129ef403a73830a7519719adb6 Mon Sep 17 00:00:00 2001 From: Adrian Brink Date: Thu, 31 Aug 2017 13:54:08 +0200 Subject: [PATCH 13/92] Cleanup lines to fit within 72 characters --- state/state.go | 21 ++++++++++++--------- state/state_test.go | 15 ++++++++++----- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/state/state.go b/state/state.go index 72c3e64da..a9a2fee8c 100644 --- a/state/state.go +++ b/state/state.go @@ -42,17 +42,18 @@ type State struct { GenesisDoc *types.GenesisDoc ChainID string - // updated at end of SetBlockAndValidators - LastBlockHeight int // Genesis state has this set to 0. So, Block(H=0) does not exist. + // Updated at end of SetBlockAndValidators + // Genesis state has this set to 0. So, Block(H=0) does not exist + LastBlockHeight int LastBlockID types.BlockID LastBlockTime time.Time Validators *types.ValidatorSet - LastValidators *types.ValidatorSet // block.LastCommit validated against this - + // block.LastCommit validated against LastValidators + LastValidators *types.ValidatorSet // AppHash is updated after Commit AppHash []byte - TxIndexer txindex.TxIndexer `json:"-"` // Transaction indexer. + TxIndexer txindex.TxIndexer `json:"-"` // Transaction indexer // When a block returns a validator set change via EndBlock, // the change only applies to the next block. @@ -224,7 +225,8 @@ func (s *State) SetBlockAndValidators(header *types.Header, blockPartsHeader typ nextValSet.IncrementAccum(1) s.setBlockAndValidators(header.Height, - types.BlockID{header.Hash(), blockPartsHeader}, header.Time, + types.BlockID{header.Hash(), blockPartsHeader}, + header.Time, prevValSet, nextValSet) } @@ -258,7 +260,7 @@ func GetState(stateDB dbm.DB, genesisFile string) *State { return state } -//-------------------------------------------------- +//------------------------------------------------------------------------ // ABCIResponses retains the responses of the various ABCI calls during block processing. // It is persisted to disk before calling Commit. @@ -298,10 +300,11 @@ func (vi *ValidatorsInfo) Bytes() []byte { return wire.BinaryBytes(*vi) } -//----------------------------------------------------------------------------- +//------------------------------------------------------------------------ // Genesis -// MakeGenesisStateFromFile reads and unmarshals state from the given file. +// MakeGenesisStateFromFile reads and unmarshals state from the given +// file. // // Used during replay and in tests. func MakeGenesisStateFromFile(db dbm.DB, genDocFile string) *State { diff --git a/state/state_test.go b/state/state_test.go index 2576b35d5..a22e9958b 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -19,8 +19,8 @@ import ( "github.com/tendermint/tendermint/types" ) +// setupTestCase does setup common to all test cases func setupTestCase(t *testing.T) (func(t *testing.T), dbm.DB, *State) { - config := cfg.ResetTestRoot("state_") stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir()) state := GetState(stateDB, config.GenesisFile()) @@ -38,7 +38,8 @@ func TestStateCopy(t *testing.T) { stateCopy := state.Copy() - assert.True(state.Equals(stateCopy), cmn.Fmt("expected state and its copy to be identical. got %v\n expected %v\n", stateCopy, state)) + assert.True(state.Equals(stateCopy), + cmn.Fmt("exppppppected state and its copy to be identical. got %v\n expected %v\n", stateCopy, state)) stateCopy.LastBlockHeight++ assert.False(state.Equals(stateCopy), cmn.Fmt("expected states to be different. got same %v", state)) } @@ -51,7 +52,8 @@ func TestStateSaveLoad(t *testing.T) { state.Save() loadedState := LoadState(stateDB) - assert.True(state.Equals(loadedState), cmn.Fmt("expected state and its copy to be identical. got %v\n expected %v\n", loadedState, state)) + assert.True(state.Equals(loadedState), + cmn.Fmt("expected state and its copy to be identical. got %v\n expected %v\n", loadedState, state)) } func TestABCIResponsesSaveLoad(t *testing.T) { @@ -76,7 +78,8 @@ func TestABCIResponsesSaveLoad(t *testing.T) { state.SaveABCIResponses(abciResponses) abciResponses2 := state.LoadABCIResponses() - assert.Equal(abciResponses, abciResponses2, cmn.Fmt("ABCIResponses don't match: Got %v, Expected %v", abciResponses2, abciResponses)) + assert.Equal(abciResponses, abciResponses2, + cmn.Fmt("ABCIResponses don't match: Got %v, Expected %v", abciResponses2, abciResponses)) } func TestValidatorSimpleSaveLoad(t *testing.T) { @@ -170,7 +173,9 @@ func TestValidatorChangesSaveLoad(t *testing.T) { } } -func makeHeaderPartsResponses(state *State, height int, pubkey crypto.PubKey) (*types.Header, types.PartSetHeader, *ABCIResponses) { +func makeHeaderPartsResponses(state *State, height int, + pubkey crypto.PubKey) (*types.Header, types.PartSetHeader, *ABCIResponses) { + block := makeBlock(height, state) _, val := state.Validators.GetByIndex(0) abciResponses := &ABCIResponses{ From 870a98ccc3cd61b13d7c999d5bf6ac89755a113a Mon Sep 17 00:00:00 2001 From: Adrian Brink Date: Tue, 12 Sep 2017 17:12:19 +0200 Subject: [PATCH 14/92] Last fixes --- state/state_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/state/state_test.go b/state/state_test.go index a22e9958b..4ff367fa3 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -39,7 +39,7 @@ func TestStateCopy(t *testing.T) { stateCopy := state.Copy() assert.True(state.Equals(stateCopy), - cmn.Fmt("exppppppected state and its copy to be identical. got %v\n expected %v\n", stateCopy, state)) + cmn.Fmt("expected state and its copy to be identical. got %v\n expected %v\n", stateCopy, state)) stateCopy.LastBlockHeight++ assert.False(state.Equals(stateCopy), cmn.Fmt("expected states to be different. got same %v", state)) } @@ -47,6 +47,7 @@ func TestStateCopy(t *testing.T) { func TestStateSaveLoad(t *testing.T) { tearDown, stateDB, state := setupTestCase(t) defer tearDown(t) + assert := assert.New(t) state.LastBlockHeight++ state.Save() From bf576f0097b45d096f29b8de5f56e555993f11a9 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 12 Sep 2017 14:37:32 -0400 Subject: [PATCH 15/92] state: minor comment fixes --- state/state.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/state/state.go b/state/state.go index a9a2fee8c..7e22e04d1 100644 --- a/state/state.go +++ b/state/state.go @@ -42,14 +42,15 @@ type State struct { GenesisDoc *types.GenesisDoc ChainID string - // Updated at end of SetBlockAndValidators - // Genesis state has this set to 0. So, Block(H=0) does not exist + // These fields are updated by SetBlockAndValidators. + // LastBlockHeight=0 at genesis (ie. block(H=0) does not exist) + // LastValidators is used to validate block.LastCommit. LastBlockHeight int LastBlockID types.BlockID LastBlockTime time.Time Validators *types.ValidatorSet - // block.LastCommit validated against LastValidators - LastValidators *types.ValidatorSet + LastValidators *types.ValidatorSet + // AppHash is updated after Commit AppHash []byte From aea8629272b088486ba51a5e6784cd235c080205 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 12 Sep 2017 20:49:22 -0400 Subject: [PATCH 16/92] peer interface --- blockchain/reactor.go | 12 ++--- consensus/byzantine_test.go | 12 ++--- consensus/common_test.go | 4 +- consensus/reactor.go | 53 ++++++++++-------- mempool/reactor.go | 6 +-- p2p/peer.go | 104 ++++++++++++++++++++++++------------ p2p/peer_set.go | 32 +++++------ p2p/peer_set_test.go | 22 ++++---- p2p/peer_test.go | 6 +-- p2p/pex_reactor.go | 18 +++---- p2p/pex_reactor_test.go | 13 ++--- p2p/switch.go | 41 +++++++------- p2p/switch_test.go | 12 ++--- rpc/core/consensus.go | 4 +- rpc/core/net.go | 4 +- 15 files changed, 195 insertions(+), 148 deletions(-) diff --git a/blockchain/reactor.go b/blockchain/reactor.go index 813d8f6b7..efa6e2f05 100644 --- a/blockchain/reactor.go +++ b/blockchain/reactor.go @@ -111,19 +111,19 @@ func (bcR *BlockchainReactor) GetChannels() []*p2p.ChannelDescriptor { } // AddPeer implements Reactor by sending our state to peer. -func (bcR *BlockchainReactor) AddPeer(peer *p2p.Peer) { +func (bcR *BlockchainReactor) AddPeer(peer p2p.Peer) { if !peer.Send(BlockchainChannel, struct{ BlockchainMessage }{&bcStatusResponseMessage{bcR.store.Height()}}) { // doing nothing, will try later in `poolRoutine` } } // RemovePeer implements Reactor by removing peer from the pool. -func (bcR *BlockchainReactor) RemovePeer(peer *p2p.Peer, reason interface{}) { - bcR.pool.RemovePeer(peer.Key) +func (bcR *BlockchainReactor) RemovePeer(peer p2p.Peer, reason interface{}) { + bcR.pool.RemovePeer(peer.Key()) } // Receive implements Reactor by handling 4 types of messages (look below). -func (bcR *BlockchainReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte) { +func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) { _, msg, err := DecodeMessage(msgBytes) if err != nil { bcR.Logger.Error("Error decoding message", "err", err) @@ -148,7 +148,7 @@ func (bcR *BlockchainReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte) } case *bcBlockResponseMessage: // Got a block. - bcR.pool.AddBlock(src.Key, msg.Block, len(msgBytes)) + bcR.pool.AddBlock(src.Key(), msg.Block, len(msgBytes)) case *bcStatusRequestMessage: // Send peer our state. queued := src.TrySend(BlockchainChannel, struct{ BlockchainMessage }{&bcStatusResponseMessage{bcR.store.Height()}}) @@ -157,7 +157,7 @@ func (bcR *BlockchainReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte) } case *bcStatusResponseMessage: // Got a peer status. Unverified. - bcR.pool.SetPeerHeight(src.Key, msg.Height) + bcR.pool.SetPeerHeight(src.Key(), msg.Height) default: bcR.Logger.Error(cmn.Fmt("Unknown message type %v", reflect.TypeOf(msg))) } diff --git a/consensus/byzantine_test.go b/consensus/byzantine_test.go index 94a03c7aa..236ba2d40 100644 --- a/consensus/byzantine_test.go +++ b/consensus/byzantine_test.go @@ -188,7 +188,7 @@ func byzantineDecideProposalFunc(t *testing.T, height, round int, cs *ConsensusS } } -func sendProposalAndParts(height, round int, cs *ConsensusState, peer *p2p.Peer, proposal *types.Proposal, blockHash []byte, parts *types.PartSet) { +func sendProposalAndParts(height, round int, cs *ConsensusState, peer p2p.Peer, proposal *types.Proposal, blockHash []byte, parts *types.PartSet) { // proposal msg := &ProposalMessage{Proposal: proposal} peer.Send(DataChannel, struct{ ConsensusMessage }{msg}) @@ -231,14 +231,14 @@ func NewByzantineReactor(conR *ConsensusReactor) *ByzantineReactor { func (br *ByzantineReactor) SetSwitch(s *p2p.Switch) { br.reactor.SetSwitch(s) } func (br *ByzantineReactor) GetChannels() []*p2p.ChannelDescriptor { return br.reactor.GetChannels() } -func (br *ByzantineReactor) AddPeer(peer *p2p.Peer) { +func (br *ByzantineReactor) AddPeer(peer p2p.Peer) { if !br.reactor.IsRunning() { return } // Create peerState for peer - peerState := NewPeerState(peer) - peer.Data.Set(types.PeerStateKey, peerState) + peerState := NewPeerState(peer).SetLogger(br.reactor.Logger) + peer.Set(types.PeerStateKey, peerState) // Send our state to peer. // If we're fast_syncing, broadcast a RoundStepMessage later upon SwitchToConsensus(). @@ -246,10 +246,10 @@ func (br *ByzantineReactor) AddPeer(peer *p2p.Peer) { br.reactor.sendNewRoundStepMessages(peer) } } -func (br *ByzantineReactor) RemovePeer(peer *p2p.Peer, reason interface{}) { +func (br *ByzantineReactor) RemovePeer(peer p2p.Peer, reason interface{}) { br.reactor.RemovePeer(peer, reason) } -func (br *ByzantineReactor) Receive(chID byte, peer *p2p.Peer, msgBytes []byte) { +func (br *ByzantineReactor) Receive(chID byte, peer p2p.Peer, msgBytes []byte) { br.reactor.Receive(chID, peer, msgBytes) } diff --git a/consensus/common_test.go b/consensus/common_test.go index c59a6d969..84f47d021 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -380,9 +380,9 @@ func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerF return css } -func getSwitchIndex(switches []*p2p.Switch, peer *p2p.Peer) int { +func getSwitchIndex(switches []*p2p.Switch, peer p2p.Peer) int { for i, s := range switches { - if bytes.Equal(peer.NodeInfo.PubKey.Address(), s.NodeInfo().PubKey.Address()) { + if bytes.Equal(peer.NodeInfo().PubKey.Address(), s.NodeInfo().PubKey.Address()) { return i } } diff --git a/consensus/reactor.go b/consensus/reactor.go index f4107c653..59cbfea76 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -120,14 +120,14 @@ func (conR *ConsensusReactor) GetChannels() []*p2p.ChannelDescriptor { } // AddPeer implements Reactor -func (conR *ConsensusReactor) AddPeer(peer *p2p.Peer) { +func (conR *ConsensusReactor) AddPeer(peer p2p.Peer) { if !conR.IsRunning() { return } // Create peerState for peer - peerState := NewPeerState(peer) - peer.Data.Set(types.PeerStateKey, peerState) + peerState := NewPeerState(peer).SetLogger(conR.Logger) + peer.Set(types.PeerStateKey, peerState) // Begin routines for this peer. go conR.gossipDataRoutine(peer, peerState) @@ -142,12 +142,12 @@ func (conR *ConsensusReactor) AddPeer(peer *p2p.Peer) { } // RemovePeer implements Reactor -func (conR *ConsensusReactor) RemovePeer(peer *p2p.Peer, reason interface{}) { +func (conR *ConsensusReactor) RemovePeer(peer p2p.Peer, reason interface{}) { if !conR.IsRunning() { return } // TODO - //peer.Data.Get(PeerStateKey).(*PeerState).Disconnect() + //peer.Get(PeerStateKey).(*PeerState).Disconnect() } // Receive implements Reactor @@ -156,7 +156,7 @@ func (conR *ConsensusReactor) RemovePeer(peer *p2p.Peer, reason interface{}) { // Peer state updates can happen in parallel, but processing of // proposals, block parts, and votes are ordered by the receiveRoutine // NOTE: blocks on consensus state for proposals, block parts, and votes -func (conR *ConsensusReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte) { +func (conR *ConsensusReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) { if !conR.IsRunning() { conR.Logger.Debug("Receive", "src", src, "chId", chID, "bytes", msgBytes) return @@ -171,7 +171,7 @@ func (conR *ConsensusReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte) conR.Logger.Debug("Receive", "src", src, "chId", chID, "msg", msg) // Get peer states - ps := src.Data.Get(types.PeerStateKey).(*PeerState) + ps := src.Get(types.PeerStateKey).(*PeerState) switch chID { case StateChannel: @@ -191,7 +191,7 @@ func (conR *ConsensusReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte) return } // Peer claims to have a maj23 for some BlockID at H,R,S, - votes.SetPeerMaj23(msg.Round, msg.Type, ps.Peer.Key, msg.BlockID) + votes.SetPeerMaj23(msg.Round, msg.Type, ps.Peer.Key(), msg.BlockID) // Respond with a VoteSetBitsMessage showing which votes we have. // (and consequently shows which we don't have) var ourVotes *cmn.BitArray @@ -228,12 +228,12 @@ func (conR *ConsensusReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte) switch msg := msg.(type) { case *ProposalMessage: ps.SetHasProposal(msg.Proposal) - conR.conS.peerMsgQueue <- msgInfo{msg, src.Key} + conR.conS.peerMsgQueue <- msgInfo{msg, src.Key()} case *ProposalPOLMessage: ps.ApplyProposalPOLMessage(msg) case *BlockPartMessage: ps.SetHasProposalBlockPart(msg.Height, msg.Round, msg.Part.Index) - conR.conS.peerMsgQueue <- msgInfo{msg, src.Key} + conR.conS.peerMsgQueue <- msgInfo{msg, src.Key()} default: conR.Logger.Error(cmn.Fmt("Unknown message type %v", reflect.TypeOf(msg))) } @@ -253,7 +253,7 @@ func (conR *ConsensusReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte) ps.EnsureVoteBitArrays(height-1, lastCommitSize) ps.SetHasVote(msg.Vote) - cs.peerMsgQueue <- msgInfo{msg, src.Key} + cs.peerMsgQueue <- msgInfo{msg, src.Key()} default: // don't punish (leave room for soft upgrades) @@ -367,7 +367,7 @@ func (conR *ConsensusReactor) broadcastHasVoteMessage(vote *types.Vote) { /* // TODO: Make this broadcast more selective. for _, peer := range conR.Switch.Peers().List() { - ps := peer.Data.Get(PeerStateKey).(*PeerState) + ps := peer.Get(PeerStateKey).(*PeerState) prs := ps.GetRoundState() if prs.Height == vote.Height { // TODO: Also filter on round? @@ -399,7 +399,7 @@ func makeRoundStepMessages(rs *RoundState) (nrsMsg *NewRoundStepMessage, csMsg * return } -func (conR *ConsensusReactor) sendNewRoundStepMessages(peer *p2p.Peer) { +func (conR *ConsensusReactor) sendNewRoundStepMessages(peer p2p.Peer) { rs := conR.conS.GetRoundState() nrsMsg, csMsg := makeRoundStepMessages(rs) if nrsMsg != nil { @@ -410,7 +410,7 @@ func (conR *ConsensusReactor) sendNewRoundStepMessages(peer *p2p.Peer) { } } -func (conR *ConsensusReactor) gossipDataRoutine(peer *p2p.Peer, ps *PeerState) { +func (conR *ConsensusReactor) gossipDataRoutine(peer p2p.Peer, ps *PeerState) { logger := conR.Logger.With("peer", peer) OUTER_LOOP: @@ -492,7 +492,7 @@ OUTER_LOOP: } func (conR *ConsensusReactor) gossipDataForCatchup(logger log.Logger, rs *RoundState, - prs *PeerRoundState, ps *PeerState, peer *p2p.Peer) { + prs *PeerRoundState, ps *PeerState, peer p2p.Peer) { if index, ok := prs.ProposalBlockParts.Not().PickRandom(); ok { // Ensure that the peer's PartSetHeader is correct @@ -534,7 +534,7 @@ func (conR *ConsensusReactor) gossipDataForCatchup(logger log.Logger, rs *RoundS } } -func (conR *ConsensusReactor) gossipVotesRoutine(peer *p2p.Peer, ps *PeerState) { +func (conR *ConsensusReactor) gossipVotesRoutine(peer p2p.Peer, ps *PeerState) { logger := conR.Logger.With("peer", peer) // Simple hack to throttle logs upon sleep. @@ -644,7 +644,7 @@ func (conR *ConsensusReactor) gossipVotesForHeight(logger log.Logger, rs *RoundS // NOTE: `queryMaj23Routine` has a simple crude design since it only comes // into play for liveness when there's a signature DDoS attack happening. -func (conR *ConsensusReactor) queryMaj23Routine(peer *p2p.Peer, ps *PeerState) { +func (conR *ConsensusReactor) queryMaj23Routine(peer p2p.Peer, ps *PeerState) { logger := conR.Logger.With("peer", peer) OUTER_LOOP: @@ -743,7 +743,7 @@ func (conR *ConsensusReactor) StringIndented(indent string) string { s := "ConsensusReactor{\n" s += indent + " " + conR.conS.StringIndented(indent+" ") + "\n" for _, peer := range conR.Switch.Peers().List() { - ps := peer.Data.Get(types.PeerStateKey).(*PeerState) + ps := peer.Get(types.PeerStateKey).(*PeerState) s += indent + " " + ps.StringIndented(indent+" ") + "\n" } s += indent + "}" @@ -808,16 +808,18 @@ var ( // PeerState contains the known state of a peer, including its connection // and threadsafe access to its PeerRoundState. type PeerState struct { - Peer *p2p.Peer + Peer p2p.Peer + logger log.Logger mtx sync.Mutex PeerRoundState } // NewPeerState returns a new PeerState for the given Peer -func NewPeerState(peer *p2p.Peer) *PeerState { +func NewPeerState(peer p2p.Peer) *PeerState { return &PeerState{ - Peer: peer, + Peer: peer, + logger: log.NewNopLogger(), PeerRoundState: PeerRoundState{ Round: -1, ProposalPOLRound: -1, @@ -827,6 +829,11 @@ func NewPeerState(peer *p2p.Peer) *PeerState { } } +func (ps *PeerState) SetLogger(logger log.Logger) *PeerState { + ps.logger = logger + return ps +} + // GetRoundState returns an atomic snapshot of the PeerRoundState. // There's no point in mutating it since it won't change PeerState. func (ps *PeerState) GetRoundState() *PeerRoundState { @@ -1025,7 +1032,7 @@ func (ps *PeerState) SetHasVote(vote *types.Vote) { } func (ps *PeerState) setHasVote(height int, round int, type_ byte, index int) { - logger := ps.Peer.Logger.With("peerRound", ps.Round, "height", height, "round", round) + logger := ps.logger.With("peerRound", ps.Round, "height", height, "round", round) logger.Debug("setHasVote(LastCommit)", "lastCommit", ps.LastCommit, "index", index) // NOTE: some may be nil BitArrays -> no side effects. @@ -1163,7 +1170,7 @@ func (ps *PeerState) StringIndented(indent string) string { %s Key %v %s PRS %v %s}`, - indent, ps.Peer.Key, + indent, ps.Peer.Key(), indent, ps.PeerRoundState.StringIndented(indent+" "), indent) } diff --git a/mempool/reactor.go b/mempool/reactor.go index 7dbfa2924..87bac5d92 100644 --- a/mempool/reactor.go +++ b/mempool/reactor.go @@ -60,18 +60,18 @@ func (memR *MempoolReactor) GetChannels() []*p2p.ChannelDescriptor { // AddPeer implements Reactor. // It starts a broadcast routine ensuring all txs are forwarded to the given peer. -func (memR *MempoolReactor) AddPeer(peer *p2p.Peer) { +func (memR *MempoolReactor) AddPeer(peer p2p.Peer) { go memR.broadcastTxRoutine(peer) } // RemovePeer implements Reactor. -func (memR *MempoolReactor) RemovePeer(peer *p2p.Peer, reason interface{}) { +func (memR *MempoolReactor) RemovePeer(peer p2p.Peer, reason interface{}) { // broadcast routine checks if peer is gone and returns } // Receive implements Reactor. // It adds any received transactions to the mempool. -func (memR *MempoolReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte) { +func (memR *MempoolReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) { _, msg, err := DecodeMessage(msgBytes) if err != nil { memR.Logger.Error("Error decoding message", "err", err) diff --git a/p2p/peer.go b/p2p/peer.go index c15f61d32..2b986a7a7 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -12,12 +12,29 @@ import ( cmn "github.com/tendermint/tmlibs/common" ) +// Peer is an interface representing a peer connected on a reactor. +type Peer interface { + cmn.Service + + Key() string + IsOutbound() bool + IsPersistent() bool + NodeInfo() *NodeInfo + Status() ConnectionStatus + + Send(byte, interface{}) bool + TrySend(byte, interface{}) bool + + Set(string, interface{}) + Get(string) interface{} +} + // Peer could be marked as persistent, in which case you can use // Redial function to reconnect. Note that inbound peers can't be // made persistent. They should be made persistent on the other end. // // Before using a peer, you will need to perform a handshake on connection. -type Peer struct { +type peer struct { cmn.BaseService outbound bool @@ -28,9 +45,9 @@ type Peer struct { persistent bool config *PeerConfig - *NodeInfo - Key string - Data *cmn.CMap // User data. + nodeInfo *NodeInfo + key string + Data *cmn.CMap // User data. } // PeerConfig is a Peer configuration. @@ -60,7 +77,7 @@ func DefaultPeerConfig() *PeerConfig { } func newOutboundPeer(addr *NetAddress, reactorsByCh map[byte]Reactor, chDescs []*ChannelDescriptor, - onPeerError func(*Peer, interface{}), ourNodePrivKey crypto.PrivKeyEd25519, config *PeerConfig) (*Peer, error) { + onPeerError func(Peer, interface{}), ourNodePrivKey crypto.PrivKeyEd25519, config *PeerConfig) (*peer, error) { conn, err := dial(addr, config) if err != nil { @@ -76,13 +93,13 @@ func newOutboundPeer(addr *NetAddress, reactorsByCh map[byte]Reactor, chDescs [] } func newInboundPeer(conn net.Conn, reactorsByCh map[byte]Reactor, chDescs []*ChannelDescriptor, - onPeerError func(*Peer, interface{}), ourNodePrivKey crypto.PrivKeyEd25519, config *PeerConfig) (*Peer, error) { + onPeerError func(Peer, interface{}), ourNodePrivKey crypto.PrivKeyEd25519, config *PeerConfig) (*peer, error) { return newPeerFromConnAndConfig(conn, false, reactorsByCh, chDescs, onPeerError, ourNodePrivKey, config) } func newPeerFromConnAndConfig(rawConn net.Conn, outbound bool, reactorsByCh map[byte]Reactor, chDescs []*ChannelDescriptor, - onPeerError func(*Peer, interface{}), ourNodePrivKey crypto.PrivKeyEd25519, config *PeerConfig) (*Peer, error) { + onPeerError func(Peer, interface{}), ourNodePrivKey crypto.PrivKeyEd25519, config *PeerConfig) (*peer, error) { conn := rawConn @@ -104,7 +121,7 @@ func newPeerFromConnAndConfig(rawConn net.Conn, outbound bool, reactorsByCh map[ } // Key and NodeInfo are set after Handshake - p := &Peer{ + p := &peer{ outbound: outbound, conn: conn, config: config, @@ -119,12 +136,12 @@ func newPeerFromConnAndConfig(rawConn net.Conn, outbound bool, reactorsByCh map[ } // CloseConn should be used when the peer was created, but never started. -func (p *Peer) CloseConn() { +func (p *peer) CloseConn() { p.conn.Close() } // makePersistent marks the peer as persistent. -func (p *Peer) makePersistent() { +func (p *peer) makePersistent() { if !p.outbound { panic("inbound peers can't be made persistent") } @@ -133,13 +150,13 @@ func (p *Peer) makePersistent() { } // IsPersistent returns true if the peer is persitent, false otherwise. -func (p *Peer) IsPersistent() bool { +func (p *peer) IsPersistent() bool { return p.persistent } // HandshakeTimeout performs a handshake between a given node and the peer. // NOTE: blocking -func (p *Peer) HandshakeTimeout(ourNodeInfo *NodeInfo, timeout time.Duration) error { +func (p *peer) HandshakeTimeout(ourNodeInfo *NodeInfo, timeout time.Duration) error { // Set deadline for handshake so we don't block forever on conn.ReadFull p.conn.SetDeadline(time.Now().Add(timeout)) @@ -176,19 +193,19 @@ func (p *Peer) HandshakeTimeout(ourNodeInfo *NodeInfo, timeout time.Duration) er peerNodeInfo.RemoteAddr = p.Addr().String() - p.NodeInfo = peerNodeInfo - p.Key = peerNodeInfo.PubKey.KeyString() + p.nodeInfo = peerNodeInfo + p.key = peerNodeInfo.PubKey.KeyString() return nil } // Addr returns peer's remote network address. -func (p *Peer) Addr() net.Addr { +func (p *peer) Addr() net.Addr { return p.conn.RemoteAddr() } // PubKey returns peer's public key. -func (p *Peer) PubKey() crypto.PubKeyEd25519 { +func (p *peer) PubKey() crypto.PubKeyEd25519 { if p.config.AuthEnc { return p.conn.(*SecretConnection).RemotePubKey() } @@ -199,31 +216,31 @@ func (p *Peer) PubKey() crypto.PubKeyEd25519 { } // OnStart implements BaseService. -func (p *Peer) OnStart() error { +func (p *peer) OnStart() error { p.BaseService.OnStart() _, err := p.mconn.Start() return err } // OnStop implements BaseService. -func (p *Peer) OnStop() { +func (p *peer) OnStop() { p.BaseService.OnStop() p.mconn.Stop() } // Connection returns underlying MConnection. -func (p *Peer) Connection() *MConnection { +func (p *peer) Connection() *MConnection { return p.mconn } // IsOutbound returns true if the connection is outbound, false otherwise. -func (p *Peer) IsOutbound() bool { +func (p *peer) IsOutbound() bool { return p.outbound } // Send msg to the channel identified by chID byte. Returns false if the send // queue is full after timeout, specified by MConnection. -func (p *Peer) Send(chID byte, msg interface{}) bool { +func (p *peer) Send(chID byte, msg interface{}) bool { if !p.IsRunning() { // see Switch#Broadcast, where we fetch the list of peers and loop over // them - while we're looping, one peer may be removed and stopped. @@ -234,7 +251,7 @@ func (p *Peer) Send(chID byte, msg interface{}) bool { // TrySend msg to the channel identified by chID byte. Immediately returns // false if the send queue is full. -func (p *Peer) TrySend(chID byte, msg interface{}) bool { +func (p *peer) TrySend(chID byte, msg interface{}) bool { if !p.IsRunning() { return false } @@ -242,7 +259,7 @@ func (p *Peer) TrySend(chID byte, msg interface{}) bool { } // CanSend returns true if the send queue is not full, false otherwise. -func (p *Peer) CanSend(chID byte) bool { +func (p *peer) CanSend(chID byte) bool { if !p.IsRunning() { return false } @@ -250,32 +267,53 @@ func (p *Peer) CanSend(chID byte) bool { } // WriteTo writes the peer's public key to w. -func (p *Peer) WriteTo(w io.Writer) (n int64, err error) { +func (p *peer) WriteTo(w io.Writer) (n int64, err error) { var n_ int - wire.WriteString(p.Key, w, &n_, &err) + wire.WriteString(p.key, w, &n_, &err) n += int64(n_) return } // String representation. -func (p *Peer) String() string { +func (p *peer) String() string { if p.outbound { - return fmt.Sprintf("Peer{%v %v out}", p.mconn, p.Key[:12]) + return fmt.Sprintf("Peer{%v %v out}", p.mconn, p.key[:12]) } - return fmt.Sprintf("Peer{%v %v in}", p.mconn, p.Key[:12]) + return fmt.Sprintf("Peer{%v %v in}", p.mconn, p.key[:12]) } // Equals reports whenever 2 peers are actually represent the same node. -func (p *Peer) Equals(other *Peer) bool { - return p.Key == other.Key +func (p *peer) Equals(other Peer) bool { + return p.key == other.Key() } // Get the data for a given key. -func (p *Peer) Get(key string) interface{} { +func (p *peer) Get(key string) interface{} { return p.Data.Get(key) } +// Set sets the data for the given key. +func (p *peer) Set(key string, data interface{}) { + p.Data.Set(key, data) +} + +// Key returns the peer's id key. +func (p *peer) Key() string { + return p.key +} + +// NodeInfo returns a copy of the peer's NodeInfo. +func (p *peer) NodeInfo() *NodeInfo { + n := *p.nodeInfo // copy + return &n +} + +// Status returns the peer's ConnectionStatus. +func (p *peer) Status() ConnectionStatus { + return p.mconn.Status() +} + func dial(addr *NetAddress, config *PeerConfig) (net.Conn, error) { conn, err := addr.DialTimeout(config.DialTimeout * time.Second) if err != nil { @@ -284,8 +322,8 @@ func dial(addr *NetAddress, config *PeerConfig) (net.Conn, error) { return conn, nil } -func createMConnection(conn net.Conn, p *Peer, reactorsByCh map[byte]Reactor, chDescs []*ChannelDescriptor, - onPeerError func(*Peer, interface{}), config *MConnConfig) *MConnection { +func createMConnection(conn net.Conn, p *peer, reactorsByCh map[byte]Reactor, chDescs []*ChannelDescriptor, + onPeerError func(Peer, interface{}), config *MConnConfig) *MConnection { onReceive := func(chID byte, msgBytes []byte) { reactor := reactorsByCh[chID] diff --git a/p2p/peer_set.go b/p2p/peer_set.go index a5a443d87..c21748cf9 100644 --- a/p2p/peer_set.go +++ b/p2p/peer_set.go @@ -7,8 +7,8 @@ import ( // IPeerSet has a (immutable) subset of the methods of PeerSet. type IPeerSet interface { Has(key string) bool - Get(key string) *Peer - List() []*Peer + Get(key string) Peer + List() []Peer Size() int } @@ -19,11 +19,11 @@ type IPeerSet interface { type PeerSet struct { mtx sync.Mutex lookup map[string]*peerSetItem - list []*Peer + list []Peer } type peerSetItem struct { - peer *Peer + peer Peer index int } @@ -31,16 +31,16 @@ type peerSetItem struct { func NewPeerSet() *PeerSet { return &PeerSet{ lookup: make(map[string]*peerSetItem), - list: make([]*Peer, 0, 256), + list: make([]Peer, 0, 256), } } // Add adds the peer to the PeerSet. // It returns ErrSwitchDuplicatePeer if the peer is already present. -func (ps *PeerSet) Add(peer *Peer) error { +func (ps *PeerSet) Add(peer Peer) error { ps.mtx.Lock() defer ps.mtx.Unlock() - if ps.lookup[peer.Key] != nil { + if ps.lookup[peer.Key()] != nil { return ErrSwitchDuplicatePeer } @@ -48,7 +48,7 @@ func (ps *PeerSet) Add(peer *Peer) error { // Appending is safe even with other goroutines // iterating over the ps.list slice. ps.list = append(ps.list, peer) - ps.lookup[peer.Key] = &peerSetItem{peer, index} + ps.lookup[peer.Key()] = &peerSetItem{peer, index} return nil } @@ -62,7 +62,7 @@ func (ps *PeerSet) Has(peerKey string) bool { } // Get looks up a peer by the provided peerKey. -func (ps *PeerSet) Get(peerKey string) *Peer { +func (ps *PeerSet) Get(peerKey string) Peer { ps.mtx.Lock() defer ps.mtx.Unlock() item, ok := ps.lookup[peerKey] @@ -74,10 +74,10 @@ func (ps *PeerSet) Get(peerKey string) *Peer { } // Remove discards peer by its Key, if the peer was previously memoized. -func (ps *PeerSet) Remove(peer *Peer) { +func (ps *PeerSet) Remove(peer Peer) { ps.mtx.Lock() defer ps.mtx.Unlock() - item := ps.lookup[peer.Key] + item := ps.lookup[peer.Key()] if item == nil { return } @@ -85,23 +85,23 @@ func (ps *PeerSet) Remove(peer *Peer) { index := item.index // Create a new copy of the list but with one less item. // (we must copy because we'll be mutating the list). - newList := make([]*Peer, len(ps.list)-1) + newList := make([]Peer, len(ps.list)-1) copy(newList, ps.list) // If it's the last peer, that's an easy special case. if index == len(ps.list)-1 { ps.list = newList - delete(ps.lookup, peer.Key) + delete(ps.lookup, peer.Key()) return } // Replace the popped item with the last item in the old list. lastPeer := ps.list[len(ps.list)-1] - lastPeerKey := lastPeer.Key + lastPeerKey := lastPeer.Key() lastPeerItem := ps.lookup[lastPeerKey] newList[index] = lastPeer lastPeerItem.index = index ps.list = newList - delete(ps.lookup, peer.Key) + delete(ps.lookup, peer.Key()) } // Size returns the number of unique items in the peerSet. @@ -112,7 +112,7 @@ func (ps *PeerSet) Size() int { } // List returns the threadsafe list of peers. -func (ps *PeerSet) List() []*Peer { +func (ps *PeerSet) List() []Peer { ps.mtx.Lock() defer ps.mtx.Unlock() return ps.list diff --git a/p2p/peer_set_test.go b/p2p/peer_set_test.go index 309b7e970..e37455256 100644 --- a/p2p/peer_set_test.go +++ b/p2p/peer_set_test.go @@ -11,10 +11,10 @@ import ( ) // Returns an empty dummy peer -func randPeer() *Peer { - return &Peer{ - Key: cmn.RandStr(12), - NodeInfo: &NodeInfo{ +func randPeer() *peer { + return &peer{ + key: cmn.RandStr(12), + nodeInfo: &NodeInfo{ RemoteAddr: cmn.Fmt("%v.%v.%v.%v:46656", rand.Int()%256, rand.Int()%256, rand.Int()%256, rand.Int()%256), ListenAddr: cmn.Fmt("%v.%v.%v.%v:46656", rand.Int()%256, rand.Int()%256, rand.Int()%256, rand.Int()%256), }, @@ -25,7 +25,7 @@ func TestPeerSetAddRemoveOne(t *testing.T) { t.Parallel() peerSet := NewPeerSet() - var peerList []*Peer + var peerList []Peer for i := 0; i < 5; i++ { p := randPeer() peerSet.Add(p) @@ -38,7 +38,7 @@ func TestPeerSetAddRemoveOne(t *testing.T) { peerSet.Remove(peerAtFront) wantSize := n - i - 1 for j := 0; j < 2; j++ { - assert.Equal(t, false, peerSet.Has(peerAtFront.Key), "#%d Run #%d: failed to remove peer", i, j) + assert.Equal(t, false, peerSet.Has(peerAtFront.Key()), "#%d Run #%d: failed to remove peer", i, j) assert.Equal(t, wantSize, peerSet.Size(), "#%d Run #%d: failed to remove peer and decrement size", i, j) // Test the route of removing the now non-existent element peerSet.Remove(peerAtFront) @@ -55,7 +55,7 @@ func TestPeerSetAddRemoveOne(t *testing.T) { for i := n - 1; i >= 0; i-- { peerAtEnd := peerList[i] peerSet.Remove(peerAtEnd) - assert.Equal(t, false, peerSet.Has(peerAtEnd.Key), "#%d: failed to remove item at end", i) + assert.Equal(t, false, peerSet.Has(peerAtEnd.Key()), "#%d: failed to remove item at end", i) assert.Equal(t, i, peerSet.Size(), "#%d: differing sizes after peerSet.Remove(atEndPeer)", i) } } @@ -64,7 +64,7 @@ func TestPeerSetAddRemoveMany(t *testing.T) { t.Parallel() peerSet := NewPeerSet() - peers := []*Peer{} + peers := []Peer{} N := 100 for i := 0; i < N; i++ { peer := randPeer() @@ -79,7 +79,7 @@ func TestPeerSetAddRemoveMany(t *testing.T) { for i, peer := range peers { peerSet.Remove(peer) - if peerSet.Has(peer.Key) { + if peerSet.Has(peer.Key()) { t.Errorf("Failed to remove peer") } if peerSet.Size() != len(peers)-i-1 { @@ -126,7 +126,7 @@ func TestPeerSetGet(t *testing.T) { t.Parallel() peerSet := NewPeerSet() peer := randPeer() - assert.Nil(t, peerSet.Get(peer.Key), "expecting a nil lookup, before .Add") + assert.Nil(t, peerSet.Get(peer.Key()), "expecting a nil lookup, before .Add") if err := peerSet.Add(peer); err != nil { t.Fatalf("Failed to add new peer: %v", err) @@ -139,7 +139,7 @@ func TestPeerSetGet(t *testing.T) { wg.Add(1) go func(i int) { defer wg.Done() - got, want := peerSet.Get(peer.Key), peer + got, want := peerSet.Get(peer.Key()), peer assert.Equal(t, got, want, "#%d: got=%v want=%v", i, got, want) }(i) } diff --git a/p2p/peer_test.go b/p2p/peer_test.go index 347de784c..ba52b22a4 100644 --- a/p2p/peer_test.go +++ b/p2p/peer_test.go @@ -76,13 +76,13 @@ func TestPeerSend(t *testing.T) { assert.True(p.Send(0x01, "Asylum")) } -func createOutboundPeerAndPerformHandshake(addr *NetAddress, config *PeerConfig) (*Peer, error) { +func createOutboundPeerAndPerformHandshake(addr *NetAddress, config *PeerConfig) (*peer, error) { chDescs := []*ChannelDescriptor{ &ChannelDescriptor{ID: 0x01, Priority: 1}, } reactorsByCh := map[byte]Reactor{0x01: NewTestReactor(chDescs, true)} pk := crypto.GenPrivKeyEd25519() - p, err := newOutboundPeer(addr, reactorsByCh, chDescs, func(p *Peer, r interface{}) {}, pk, config) + p, err := newOutboundPeer(addr, reactorsByCh, chDescs, func(p Peer, r interface{}) {}, pk, config) if err != nil { return nil, err } @@ -133,7 +133,7 @@ func (p *remotePeer) accept(l net.Listener) { if err != nil { golog.Fatalf("Failed to accept conn: %+v", err) } - peer, err := newInboundPeer(conn, make(map[byte]Reactor), make([]*ChannelDescriptor, 0), func(p *Peer, r interface{}) {}, p.PrivKey, p.Config) + peer, err := newInboundPeer(conn, make(map[byte]Reactor), make([]*ChannelDescriptor, 0), func(p Peer, r interface{}) {}, p.PrivKey, p.Config) if err != nil { golog.Fatalf("Failed to create a peer: %+v", err) } diff --git a/p2p/pex_reactor.go b/p2p/pex_reactor.go index 30ebc3a7e..69ab55cc9 100644 --- a/p2p/pex_reactor.go +++ b/p2p/pex_reactor.go @@ -92,7 +92,7 @@ func (r *PEXReactor) GetChannels() []*ChannelDescriptor { // AddPeer implements Reactor by adding peer to the address book (if inbound) // or by requesting more addresses (if outbound). -func (r *PEXReactor) AddPeer(p *Peer) { +func (r *PEXReactor) AddPeer(p Peer) { if p.IsOutbound() { // For outbound peers, the address is already in the books. // Either it was added in DialSeeds or when we @@ -101,10 +101,10 @@ func (r *PEXReactor) AddPeer(p *Peer) { r.RequestPEX(p) } } else { // For inbound connections, the peer is its own source - addr, err := NewNetAddressString(p.ListenAddr) + addr, err := NewNetAddressString(p.NodeInfo().ListenAddr) if err != nil { // this should never happen - r.Logger.Error("Error in AddPeer: invalid peer address", "addr", p.ListenAddr, "err", err) + r.Logger.Error("Error in AddPeer: invalid peer address", "addr", p.NodeInfo().ListenAddr, "err", err) return } r.book.AddAddress(addr, addr) @@ -112,15 +112,15 @@ func (r *PEXReactor) AddPeer(p *Peer) { } // RemovePeer implements Reactor. -func (r *PEXReactor) RemovePeer(p *Peer, reason interface{}) { +func (r *PEXReactor) RemovePeer(p Peer, reason interface{}) { // If we aren't keeping track of local temp data for each peer here, then we // don't have to do anything. } // Receive implements Reactor by handling incoming PEX messages. -func (r *PEXReactor) Receive(chID byte, src *Peer, msgBytes []byte) { - srcAddr := src.Connection().RemoteAddress - srcAddrStr := srcAddr.String() +func (r *PEXReactor) Receive(chID byte, src Peer, msgBytes []byte) { + srcAddrStr := src.NodeInfo().RemoteAddr + srcAddr, _ := NewNetAddressString(srcAddrStr) r.IncrementMsgCountForPeer(srcAddrStr) if r.ReachedMaxMsgCountForPeer(srcAddrStr) { @@ -154,12 +154,12 @@ func (r *PEXReactor) Receive(chID byte, src *Peer, msgBytes []byte) { } // RequestPEX asks peer for more addresses. -func (r *PEXReactor) RequestPEX(p *Peer) { +func (r *PEXReactor) RequestPEX(p Peer) { p.Send(PexChannel, struct{ PexMessage }{&pexRequestMessage{}}) } // SendAddrs sends addrs to the peer. -func (r *PEXReactor) SendAddrs(p *Peer, addrs []*NetAddress) { +func (r *PEXReactor) SendAddrs(p Peer, addrs []*NetAddress) { p.Send(PexChannel, struct{ PexMessage }{&pexAddrsMessage{Addrs: addrs}}) } diff --git a/p2p/pex_reactor_test.go b/p2p/pex_reactor_test.go index 0748486e5..b2c15ed89 100644 --- a/p2p/pex_reactor_test.go +++ b/p2p/pex_reactor_test.go @@ -129,7 +129,7 @@ func TestPEXReactorReceive(t *testing.T) { peer := createRandomPeer(false) size := book.Size() - netAddr, _ := NewNetAddressString(peer.ListenAddr) + netAddr, _ := NewNetAddressString(peer.NodeInfo().ListenAddr) addrs := []*NetAddress{netAddr} msg := wire.BinaryBytes(struct{ PexMessage }{&pexAddrsMessage{Addrs: addrs}}) r.Receive(PexChannel, peer, msg) @@ -159,16 +159,17 @@ func TestPEXReactorAbuseFromPeer(t *testing.T) { r.Receive(PexChannel, peer, msg) } - assert.True(r.ReachedMaxMsgCountForPeer(peer.ListenAddr)) + assert.True(r.ReachedMaxMsgCountForPeer(peer.NodeInfo().ListenAddr)) } -func createRandomPeer(outbound bool) *Peer { +func createRandomPeer(outbound bool) *peer { addr := cmn.Fmt("%v.%v.%v.%v:46656", rand.Int()%256, rand.Int()%256, rand.Int()%256, rand.Int()%256) netAddr, _ := NewNetAddressString(addr) - p := &Peer{ - Key: cmn.RandStr(12), - NodeInfo: &NodeInfo{ + p := &peer{ + key: cmn.RandStr(12), + nodeInfo: &NodeInfo{ ListenAddr: addr, + RemoteAddr: netAddr.String(), }, outbound: outbound, mconn: &MConnection{RemoteAddress: netAddr}, diff --git a/p2p/switch.go b/p2p/switch.go index d92dd6370..d1d615398 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -22,9 +22,9 @@ type Reactor interface { SetSwitch(*Switch) GetChannels() []*ChannelDescriptor - AddPeer(peer *Peer) - RemovePeer(peer *Peer, reason interface{}) - Receive(chID byte, peer *Peer, msgBytes []byte) + AddPeer(peer Peer) + RemovePeer(peer Peer, reason interface{}) + Receive(chID byte, peer Peer, msgBytes []byte) } //-------------------------------------- @@ -44,10 +44,10 @@ func NewBaseReactor(name string, impl Reactor) *BaseReactor { func (br *BaseReactor) SetSwitch(sw *Switch) { br.Switch = sw } -func (_ *BaseReactor) GetChannels() []*ChannelDescriptor { return nil } -func (_ *BaseReactor) AddPeer(peer *Peer) {} -func (_ *BaseReactor) RemovePeer(peer *Peer, reason interface{}) {} -func (_ *BaseReactor) Receive(chID byte, peer *Peer, msgBytes []byte) {} +func (_ *BaseReactor) GetChannels() []*ChannelDescriptor { return nil } +func (_ *BaseReactor) AddPeer(peer Peer) {} +func (_ *BaseReactor) RemovePeer(peer Peer, reason interface{}) {} +func (_ *BaseReactor) Receive(chID byte, peer Peer, msgBytes []byte) {} //----------------------------------------------------------------------------- @@ -213,7 +213,8 @@ func (sw *Switch) OnStop() { // and to all registered reactors. // NOTE: This performs a blocking handshake before the peer is added. // CONTRACT: If error is returned, peer is nil, and conn is immediately closed. -func (sw *Switch) AddPeer(peer *Peer) error { +func (sw *Switch) AddPeer(peer *peer) error { + if err := sw.FilterConnByAddr(peer.Addr()); err != nil { return err } @@ -232,12 +233,12 @@ func (sw *Switch) AddPeer(peer *Peer) error { } // Check version, chain id - if err := sw.nodeInfo.CompatibleWith(peer.NodeInfo); err != nil { + if err := sw.nodeInfo.CompatibleWith(peer.NodeInfo()); err != nil { return err } // Check for duplicate peer - if sw.peers.Has(peer.Key) { + if sw.peers.Has(peer.Key()) { return ErrSwitchDuplicatePeer } @@ -285,7 +286,7 @@ func (sw *Switch) SetPubKeyFilter(f func(crypto.PubKeyEd25519) error) { sw.filterConnByPubKey = f } -func (sw *Switch) startInitPeer(peer *Peer) { +func (sw *Switch) startInitPeer(peer *peer) { peer.Start() // spawn send/recv routines for _, reactor := range sw.reactors { reactor.AddPeer(peer) @@ -337,7 +338,7 @@ func (sw *Switch) dialSeed(addr *NetAddress) { // DialPeerWithAddress dials the given peer and runs sw.AddPeer if it connects successfully. // If `persistent == true`, the switch will always try to reconnect to this peer if the connection ever fails. -func (sw *Switch) DialPeerWithAddress(addr *NetAddress, persistent bool) (*Peer, error) { +func (sw *Switch) DialPeerWithAddress(addr *NetAddress, persistent bool) (Peer, error) { sw.dialing.Set(addr.IP.String(), addr) defer sw.dialing.Delete(addr.IP.String()) @@ -375,7 +376,7 @@ func (sw *Switch) Broadcast(chID byte, msg interface{}) chan bool { successChan := make(chan bool, len(sw.peers.List())) sw.Logger.Debug("Broadcast", "channel", chID, "msg", msg) for _, peer := range sw.peers.List() { - go func(peer *Peer) { + go func(peer Peer) { success := peer.Send(chID, msg) successChan <- success }(peer) @@ -387,7 +388,7 @@ func (sw *Switch) Broadcast(chID byte, msg interface{}) chan bool { func (sw *Switch) NumPeers() (outbound, inbound, dialing int) { peers := sw.peers.List() for _, peer := range peers { - if peer.outbound { + if peer.IsOutbound() { outbound++ } else { inbound++ @@ -405,8 +406,8 @@ func (sw *Switch) Peers() IPeerSet { // StopPeerForError disconnects from a peer due to external error. // If the peer is persistent, it will attempt to reconnect. // TODO: make record depending on reason. -func (sw *Switch) StopPeerForError(peer *Peer, reason interface{}) { - addr := NewNetAddress(peer.Addr()) +func (sw *Switch) StopPeerForError(peer Peer, reason interface{}) { + addr, _ := NewNetAddressString(peer.NodeInfo().RemoteAddr) sw.Logger.Error("Stopping peer for error", "peer", peer, "err", reason) sw.stopAndRemovePeer(peer, reason) @@ -438,12 +439,12 @@ func (sw *Switch) StopPeerForError(peer *Peer, reason interface{}) { // StopPeerGracefully disconnects from a peer gracefully. // TODO: handle graceful disconnects. -func (sw *Switch) StopPeerGracefully(peer *Peer) { +func (sw *Switch) StopPeerGracefully(peer Peer) { sw.Logger.Info("Stopping peer gracefully") sw.stopAndRemovePeer(peer, nil) } -func (sw *Switch) stopAndRemovePeer(peer *Peer, reason interface{}) { +func (sw *Switch) stopAndRemovePeer(peer Peer, reason interface{}) { sw.peers.Remove(peer) peer.Stop() for _, reactor := range sw.reactors { @@ -483,11 +484,11 @@ func (sw *Switch) listenerRoutine(l Listener) { //----------------------------------------------------------------------------- type SwitchEventNewPeer struct { - Peer *Peer + Peer Peer } type SwitchEventDonePeer struct { - Peer *Peer + Peer Peer Error interface{} } diff --git a/p2p/switch_test.go b/p2p/switch_test.go index c686f9e46..d2435dfff 100644 --- a/p2p/switch_test.go +++ b/p2p/switch_test.go @@ -37,8 +37,8 @@ type TestReactor struct { mtx sync.Mutex channels []*ChannelDescriptor - peersAdded []*Peer - peersRemoved []*Peer + peersAdded []Peer + peersRemoved []Peer logMessages bool msgsCounter int msgsReceived map[byte][]PeerMessage @@ -59,24 +59,24 @@ func (tr *TestReactor) GetChannels() []*ChannelDescriptor { return tr.channels } -func (tr *TestReactor) AddPeer(peer *Peer) { +func (tr *TestReactor) AddPeer(peer Peer) { tr.mtx.Lock() defer tr.mtx.Unlock() tr.peersAdded = append(tr.peersAdded, peer) } -func (tr *TestReactor) RemovePeer(peer *Peer, reason interface{}) { +func (tr *TestReactor) RemovePeer(peer Peer, reason interface{}) { tr.mtx.Lock() defer tr.mtx.Unlock() tr.peersRemoved = append(tr.peersRemoved, peer) } -func (tr *TestReactor) Receive(chID byte, peer *Peer, msgBytes []byte) { +func (tr *TestReactor) Receive(chID byte, peer Peer, msgBytes []byte) { if tr.logMessages { tr.mtx.Lock() defer tr.mtx.Unlock() //fmt.Printf("Received: %X, %X\n", chID, msgBytes) - tr.msgsReceived[chID] = append(tr.msgsReceived[chID], PeerMessage{peer.Key, msgBytes, tr.msgsCounter}) + tr.msgsReceived[chID] = append(tr.msgsReceived[chID], PeerMessage{peer.Key(), msgBytes, tr.msgsCounter}) tr.msgsCounter++ } } diff --git a/rpc/core/consensus.go b/rpc/core/consensus.go index 848288447..0429c8d45 100644 --- a/rpc/core/consensus.go +++ b/rpc/core/consensus.go @@ -86,9 +86,9 @@ func DumpConsensusState() (*ctypes.ResultDumpConsensusState, error) { peerRoundStates := []string{} for _, peer := range p2pSwitch.Peers().List() { // TODO: clean this up? - peerState := peer.Data.Get(types.PeerStateKey).(*cm.PeerState) + peerState := peer.Get(types.PeerStateKey).(*cm.PeerState) peerRoundState := peerState.GetRoundState() - peerRoundStateStr := peer.Key + ":" + string(wire.JSONBytes(peerRoundState)) + peerRoundStateStr := peer.Key() + ":" + string(wire.JSONBytes(peerRoundState)) peerRoundStates = append(peerRoundStates, peerRoundStateStr) } return &ctypes.ResultDumpConsensusState{roundState.String(), peerRoundStates}, nil diff --git a/rpc/core/net.go b/rpc/core/net.go index 6c2dc587e..c6749941c 100644 --- a/rpc/core/net.go +++ b/rpc/core/net.go @@ -42,9 +42,9 @@ func NetInfo() (*ctypes.ResultNetInfo, error) { peers := []ctypes.Peer{} for _, peer := range p2pSwitch.Peers().List() { peers = append(peers, ctypes.Peer{ - NodeInfo: *peer.NodeInfo, + NodeInfo: *peer.NodeInfo(), IsOutbound: peer.IsOutbound(), - ConnectionStatus: peer.Connection().Status(), + ConnectionStatus: peer.Status(), }) } return &ctypes.ResultNetInfo{ From ede4f818fdc2d188ec3a5efd88fa8cc9e8ae7879 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 15 Sep 2017 19:31:37 -0400 Subject: [PATCH 17/92] docs: update and clean up adr --- docs/architecture/ABCI.md | 16 -- docs/architecture/README.md | 13 +- .../{adr-001.md => adr-001-logging.md} | 0 ...r-002.md => adr-002-event-subscription.md} | 2 +- .../{adr-003.md => adr-003-abci-app-rpc.md} | 2 +- .../adr-004-historical-validators.md | 38 +++ docs/architecture/adr-template.md | 16 ++ docs/architecture/merkle-frey.md | 240 ------------------ docs/architecture/merkle.md | 17 -- 9 files changed, 57 insertions(+), 287 deletions(-) delete mode 100644 docs/architecture/ABCI.md rename docs/architecture/{adr-001.md => adr-001-logging.md} (100%) rename docs/architecture/{adr-002.md => adr-002-event-subscription.md} (98%) rename docs/architecture/{adr-003.md => adr-003-abci-app-rpc.md} (95%) create mode 100644 docs/architecture/adr-004-historical-validators.md create mode 100644 docs/architecture/adr-template.md delete mode 100644 docs/architecture/merkle-frey.md delete mode 100644 docs/architecture/merkle.md diff --git a/docs/architecture/ABCI.md b/docs/architecture/ABCI.md deleted file mode 100644 index 04d62c1ce..000000000 --- a/docs/architecture/ABCI.md +++ /dev/null @@ -1,16 +0,0 @@ -# ABCI - -ABCI is an interface between the consensus/blockchain engine known as tendermint, and the application-specific business logic, known as an ABCi app. - -The tendermint core should run unchanged for all apps. Each app can customize it, the supported transactions, queries, even the validator sets and how to handle staking / slashing stake. This customization is achieved by implementing the ABCi app to send the proper information to the tendermint engine to perform as directed. - -To understand this decision better, think of the design of the tendermint engine. - -* A blockchain is simply consensus on a unique global ordering of events. -* This consensus can efficiently be implemented using BFT and PoS -* This code can be generalized to easily support a large number of blockchains -* The block-chain specific code, the interpretation of the individual events, can be implemented by a 3rd party app without touching the consensus engine core -* Use an efficient, language-agnostic layer to implement this (ABCi) - - -Bucky, please make this doc real. diff --git a/docs/architecture/README.md b/docs/architecture/README.md index dc9c62a9e..9e41d3064 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -2,15 +2,4 @@ This is a location to record all high-level architecture decisions in the tendermint project. Not the implementation details, but the reasoning that happened. This should be refered to for guidance of the "right way" to extend the application. And if we notice that the original decisions were lacking, we should have another open discussion, record the new decisions here, and then modify the code to match. -This is like our guide and mentor when Jae and Bucky are offline.... The concept comes from a [blog post](https://product.reverb.com/documenting-architecture-decisions-the-reverb-way-a3563bb24bd0#.78xhdix6t) that resonated among the team when Anton shared it. - -Each section of the code can have it's own markdown file in this directory, and please add a link to the readme. - -## Sections - -* [ABCI](./ABCI.md) -* [go-merkle / merkleeyes](./merkle.md) -* [Frey's thoughts on the data store](./merkle-frey.md) -* basecoin -* tendermint core (multiple sections) -* ??? +Read up on the concept in this [blog post](https://product.reverb.com/documenting-architecture-decisions-the-reverb-way-a3563bb24bd0#.78xhdix6t). diff --git a/docs/architecture/adr-001.md b/docs/architecture/adr-001-logging.md similarity index 100% rename from docs/architecture/adr-001.md rename to docs/architecture/adr-001-logging.md diff --git a/docs/architecture/adr-002.md b/docs/architecture/adr-002-event-subscription.md similarity index 98% rename from docs/architecture/adr-002.md rename to docs/architecture/adr-002-event-subscription.md index 3c1f51b6e..cc207c4af 100644 --- a/docs/architecture/adr-002.md +++ b/docs/architecture/adr-002-event-subscription.md @@ -1,4 +1,4 @@ -# ADR 2: Indexing +# ADR 2: Event Subscription ## Context diff --git a/docs/architecture/adr-003.md b/docs/architecture/adr-003-abci-app-rpc.md similarity index 95% rename from docs/architecture/adr-003.md rename to docs/architecture/adr-003-abci-app-rpc.md index 7384a5d93..2775db077 100644 --- a/docs/architecture/adr-003.md +++ b/docs/architecture/adr-003-abci-app-rpc.md @@ -1,4 +1,4 @@ -# ADR 1: Must an ABCI-app have an RPC server? +# ADR 3: Must an ABCI-app have an RPC server? ## Context diff --git a/docs/architecture/adr-004-historical-validators.md b/docs/architecture/adr-004-historical-validators.md new file mode 100644 index 000000000..be0de22c1 --- /dev/null +++ b/docs/architecture/adr-004-historical-validators.md @@ -0,0 +1,38 @@ +# ADR 004: Historical Validators + +## Context + +Right now, we can query the present validator set, but there is no history. +If you were offline for a long time, there is no way to reconstruct past validators. This is needed for the light client and we agreed needs enhancement of the API. + +## Decision + +For every block, store a new structure that contains either the latest validator set, +or the height of the last block for which the validator set changed. Note this is not +the height of the block which returned the validator set change itself, but the next block, +ie. the first block it comes into effect for. + +Storing the validators will be handled by the `state` package. + +At some point in the future, we may consider more efficient storage in the case where the validators +are updated frequently - for instance by only saving the diffs, rather than the whole set. + +An alternative approach suggested keeping the validator set, or diffs of it, in a merkle IAVL tree. +While it might afford cheaper proofs that a validator set has not changed, it would be more complex, +and likely less efficient. + +## Status + +Accepted. + +## Consequences + +### Positive + +- Can query old validator sets, with proof. + +### Negative + +- Writes an extra structure to disk with every block. + +### Neutral diff --git a/docs/architecture/adr-template.md b/docs/architecture/adr-template.md new file mode 100644 index 000000000..2303490ad --- /dev/null +++ b/docs/architecture/adr-template.md @@ -0,0 +1,16 @@ +# ADR 000: Template for an ADR + +## Context + +## Decision + +## Status + + +## Consequences + +### Positive + +### Negative + +### Neutral diff --git a/docs/architecture/merkle-frey.md b/docs/architecture/merkle-frey.md deleted file mode 100644 index aca4ce32e..000000000 --- a/docs/architecture/merkle-frey.md +++ /dev/null @@ -1,240 +0,0 @@ -# Merkle data stores - Frey's proposal - -## TL;DR - -To allow the efficient creation of an ABCi app, tendermint wishes to provide a reference implementation of a key-value store that provides merkle proofs of the data. These proofs then quickly allow the ABCi app to provide an app hash to the consensus engine, as well as a full proof to any client. - -This is equivalent to building a database, and I would propose designing it from the API first, then looking how to implement this (or make an adapter from the API to existing implementations). Once we agree on the functionality and the interface, we can implement the API bindings, and then work on building adapters to existence merkle-ized data stores, or modifying the stores to support this interface. - -We need to consider the API (both in-process and over the network), language bindings, maintaining handles to old state (and garbage collecting), persistence, security, providing merkle proofs, and general key-value store operations. To stay consistent with the blockchains "single global order of operations", this data store should only allow one connection at a time to have write access. - -## Overview - -* **State** - * There are two concepts of state, "committed state" and "working state" - * The working state is only accessible from the ABCi app, allows writing, but does not need to support proofs. - * When we commit the "working state", it becomes a new "committed state" and has an immutable root hash, provides proofs, and can be exposed to external clients. -* **Transactions** - * The database always allows creating a read-only transaction at the last "committed state", this transaction can serve read queries and proofs. - * The database maintains all data to serve these read transactions until they are closed by the client (or time out). This allows the client(s) to determine how much old info is needed - * The database can only support *maximal* one writable transaction at a time. This makes it easy to enforce serializability, and attempting to start a second writable transaction may trigger a panic. -* **Functionality** - * It must support efficient key-value operations (get/set/delete) - * It must support returning merkle proofs for any "committed state" - * It should support range queries on subsets of the key space if possible (ie. if the db doesn't hash keys) - * It should also support listening to changes to a desired key via pub-sub or similar method, so I can quickly notify you on a change to your balance without constant polling. - * It may support other db-specific query types as an extension to this interface, as long as all specified actions maintain their meaning. -* **Interface** - * This interface should be domain-specific - ie. designed just for this use case - * It should present a simple go interface for embedding the data store in-process - * It should create a gRPC/protobuf API for calling from any client - * It should provide and maintain client adapters from our in-process interface to gRPC client calls for at least golang and Java (maybe more languages?) - * It should provide and maintain server adapters from our gRPC calls to the in-process interface for golang at least (unless there is another server we wish to support) -* **Persistence** - * It must support atomic persistence upon committing a new block. That is, upon crash recovery, the state is guaranteed to represent the state at the end of a complete block (along with a note of which height it was). - * It must delay deletion of old data as long as there are open read-only transactions referring to it, thus we must maintain some sort of WAL to keep track of pending cleanup. - * When a transaction is closed, or when we recover from a crash, it should clean up all no longer needed data to avoid memory/storage leaks. -* **Security and Auth** - * If we allow connections over gRPC, we must consider this issues and allow both encryption (SSL), and some basic auth rules to prevent undesired access to the DB - * This is client-specific and does not need to be supported in the in-process, embedded version. - -## Details - -Here we go more in-depth in each of the sections, explaining the reasoning and more details on the desired behavior. This document is only the high-level architecture and should support multiple implementations. When building out a specific implementation, a similar document should be provided for that repo, showing how it implements these concepts, and details about memory usage, storage, efficiency, etc. - - -### State - -The current ABCi interface avoids this question a bit and that has brought confusion. If I use `merkleeyes` to store data, which state is returned from `Query`? The current "working" state, which I would like to refer to in my ABCi application? Or the last committed state, which I would like to return to a client's query? Or an old state, which I may select based on height? - -Right now, `merkleeyes` implements `Query` like a normal ABCi app and only returns committed state, which has lead to problems and confusion. Thus, we need to be explicit about which state we want to view. Each viewer can then specify which state it wants to view. This allows the app to query the working state in DeliverTx, but the committed state in Query. - -We can easily provide two global references for "last committed" and "current working" states. However, if we want to also allow querying of older commits... then we need some way to keep track of which ones are still in use, so we can garbage collect the unneeded ones. There is a non-trivial overhead in holding references to all past states, but also a hard-coded solution (hold onto the last 5 commits) may not support all clients. We should let the client define this somehow. - -### Transactions - -Transactions (in the typical database sense) are a clean and established solution to this issue. We can look at the [isolations levels](https://en.wikipedia.org/wiki/Isolation_(database_systems)#Serializable) which attempt to provide us things like "repeatable reads". That means if we open a transaction, and query some data 100 times while other processes are writing to the db, we get the same result each time. This transaction has a reference to its own local state from the time the transaction started. (We are referring to the highest isolation levels here, which correlate well this the blockchain use case). - -If we implement a read-only transaction as a reference to state at the time of creation of that transaction, we can then hold these references to various snapshots, one per block that we are interested, and allow the client to multiplex queries and proofs from these various blocks. - -If we continue using these concepts (which have informed 30+ years of server side design), we can add a few nice features to our write transactions. The first of which is `Rollback` and `Commit`. That means all the changes we make in this transaction have no effect on the database until they are committed. And until they are committed, we can always abort if we detect an anomaly, returning to the last committed state with a rollback. - -There is also a nice extension to this available on some database servers, basically, "nested" transactions or "savepoints". This means that within one transaction, you can open a subtransaction/savepoint and continue work. Later you have the option to commit or rollback all work since the savepoint/subtransaction. And then continue with the main transaction. - -If you don't understand why this is useful, look at how basecoin needs to [hold cached state for AppTx](https://github.com/tendermint/basecoin/blob/master/state/execution.go#L126-L149), meaning that it rolls back all modifications if the AppTx returns an error. This was implemented as a wrapper in basecoin, but it is a reasonable thing to support in the DB interface itself (especially since the implementation becomes quite non-trivial as soon as you support range queries). - -To give a bit more reference to this concept in practice, read about [Savepoints in Postgresql](https://www.postgresql.org/docs/current/static/tutorial-transactions.html) ([reference](https://www.postgresql.org/docs/current/static/sql-savepoint.html)) or [Nesting transactions in SQL Server](http://dba-presents.com/index.php/databases/sql-server/43-nesting-transactions-and-save-transaction-command) (TL;DR: scroll to the bottom, section "Real nesting transactions with SAVE TRANSACTION") - -### Functionality - -Merkle trees work with key-value pairs, so we should most importantly focus on the basic Key-Value operations. That is `Get`, `Set`, and `Remove`. We also need to return a merkle proof for any key, along with a root hash of the tree for committing state to the blockchain. This is just the basic merkle-tree stuff. - -If it is possible with the implementation, it is nice to provide access to Range Queries. That is, return all values where the key is between X and Y. If you construct your keys wisely, it is possible to store lists (1:N) relations this way. Eg, storing blog posts and the key is blog:`poster_id`:`sequence`, then I could search for all blog posts by a given `poster_id`, or even return just posts 10-19 from the given poster. - -The construction of a tree that supports range queries was one of the [design decisions of go-merkle](https://github.com/tendermint/go-merkle/blob/master/README.md). It is also kind of possible with [ethereum's patricia trie](https://github.com/ethereum/wiki/wiki/Patricia-Tree) as long as the key is less than 32 bytes. - -In addition to range queries, there is one more nice feature that we could add to our data store - listening to events. Depending on your context, this is "reactive programming", "event emitters", "notifications", etc... But the basic concept is that a client can listen for all changes to a given key (or set of keys), and receive a notification when this happens. This is very important to avoid [repeated polling and wasted queries](http://resthooks.org/) when a client simply wants to [detect changes](https://www.rethinkdb.com/blog/realtime-web/). - -If the database provides access to some "listener" functionality, the app can choose to expose this to the external client via websockets, web hooks, http2 push events, android push notifications, etc, etc etc.... But if we want to support modern client functionality, let's add support for this reactive paradigm in our DB interface. - -**TODO** support for more advanced backends, eg. Bolt.... - -### Go Interface - -I will start with a simple go interface to illustrate the in-process interface. Once there is agreement on how this looks, we can work out the gRPC bindings to support calling out of process. These interfaces are not finalized code, but I think the demonstrate the concepts better than text and provide a strawman to get feedback. - -``` -// DB represents the committed state of a merkle-ized key-value store -type DB interface { - // Snapshot returns a reference to last committed state to use for - // providing proofs, you must close it at the end to garbage collect - // the historical state we hold on to to make these proofs - Snapshot() Prover - - // Start a transaction - only way to change state - // This will return an error if there is an open Transaction - Begin() (Transaction, error) - - // These callbacks are triggered when the Transaction is Committed - // to the DB. They can be used to eg. notify clients via websockets when - // their account balance changes. - AddListener(key []byte, listener Listener) - RemoveListener(listener Listener) -} - -// DBReader represents a read-only connection to a snapshot of the db -type DBReader interface { - // Queries on my local view - Has(key []byte) (bool, error) - Get(key []byte) (Model, error) - GetRange(start, end []byte, ascending bool, limit int) ([]Model, error) - Closer -} - -// Prover is an interface that lets one query for Proofs, holding the -// data at a specific location in memory -type Prover interface { - DBReader - - // Hash is the AppHash (RootHash) for this block - Hash() (hash []byte) - - // Prove returns the data along with a merkle Proof - // Model and Proof are nil if not found - Prove(key []byte) (Model, Proof, error) -} - -// Transaction is a set of state changes to the DB to be applied atomically. -// There can only be one open transaction at a time, which may only have -// maximum one subtransaction at a time. -// In short, at any time, there is exactly one object that can write to the -// DB, and we can use Subtransactions to group operations and roll them back -// together (kind of like `types.KVCache` from basecoin) -type Transaction interface { - DBReader - - // Change the state - will raise error immediately if this Transaction - // is not holding the exclusive write lock - Set(model Model) (err error) - Remove(key []byte) (removed bool, err error) - - // Subtransaction starts a new subtransaction, rollback will not affect the - // parent. Only on Commit are the changes applied to this transaction. - // While the subtransaction exists, no write allowed on the parent. - // (You must Commit or Rollback the child to continue) - Subtransaction() Transaction - - // Commit this transaction (or subtransaction), the parent reference is - // now updated. - // This only updates persistant store if the top level transaction commits - // (You may have any number of nested sub transactions) - Commit() error - - // Rollback ends the transaction and throw away all transaction-local state, - // allowing the tree to prune those elements. - // The parent transaction now recovers the write lock. - Rollback() -} - -// Listener registers callbacks on changes to the data store -type Listener interface { - OnSet(key, value, oldValue []byte) - OnRemove(key, oldValue []byte) -} - -// Proof represents a merkle proof for a key -type Proof interface { - RootHash() []byte - Verify(key, value, root []byte) bool -} - -type Model interface { - Key() []byte - Value() []byte -} - -// Closer releases the reference to this state, allowing us to garbage collect -// Make sure to call it before discarding. -type Closer interface { - Close() -} -``` - -### Remote Interface - -The use-case of allowing out-of-process calls is very powerful. Not just to provide a powerful merkle-ready data store to non-go applications. - -It we allow the ABCi app to maintain the only writable connections, we can guarantee that all transactions are only processed through the tendermint consensus engine. We could then allow multiple "web server" machines "read-only" access and scale out the database reads, assuming the consensus engine, ABCi logic, and public key cryptography is more the bottleneck than the database. We could even place the consensus engine, ABCi app, and data store on one machine, connected with unix sockets for security, and expose a tcp/ssl interface for reading the data, to scale out query processing over multiple machines. - -But returning our focus directly to the ABCi app (which is the most important use case). An app may well want to maintain 100 or 1000 snapshots of different heights to allow people to easily query many proofs at a given height without race conditions (very important for IBC, ask Jae). Thus, we should not require a separate TCP connection for each height, as this gets quite awkward with so many connections. Also, if we want to use gRPC, we should consider the connections potentially transient (although they are more efficient with keep-alive). - -Thus, the wire encoding of a transaction or a snapshot should simply return a unique id. All methods on a `Prover` or `Transaction` over the wire can send this id along with the arguments for the method call. And we just need a hash map on the server to map this id to a state. - -The only negative of not requiring a persistent tcp connection for each snapshot is there is no auto-detection if the client crashes without explicitly closing the connections. Thus, I would suggest adding a `Ping` thread in the gRPC interface which keeps the Snapshot alive. If no ping is received within a server-defined time, it may automatically close those transactions. And if we consider a client with 500 snapshots that needs to ping each every 10 seconds, that is a lot of overhead, so we should design the ping to accept a list of IDs for the client and update them all. Or associate all snapshots with a clientID and then just send the clientID in the ping. (Please add other ideas on how to detect client crashes without persistent connections). - -To encourage adoption, we should provide a nice client that uses this gRPC interface (like we do with ABCi). For go, the client may have the exact same interface as the in-process version, just that the error call may return network errors, not just illegal operations. We should also add a client with a clean API for Java, since that seems to be popular among app developers in the current tendermint community. Other bindings as we see the need in the server space. - -### Persistence - -Any data store worth it's name should not lose all data on a crash. Even [redis provides some persistence](https://redis.io/topics/persistence) these days. Ideally, if the system crashes and restarts, it should have the data at the last block N that was committed. If the system crash during the commit of block N+1, then the recovered state should either be block N or completely committed block N+1, but no partial state between the two. Basically, the commit must be an atomic operation (even if updating 100's of records). - -To avoid a lot of headaches ourselves, we can use an existing data store, such as leveldb, which provides `WriteBatch` to group all operations. - -The other issue is cleaning up old state. We cannot delete any information from our persistent store, as long as any snapshot holds a reference to it (or else we get some panics when the data we query is not there). So, we need to store the outstanding deletions that we can perform when the snapshot is `Close`d. In addition, we must consider the case that the data store crashes with open snapshots. Thus, the info on outstanding deletions must also be persisted somewhere. Something like a "delete-behind log" (the opposite of a "write ahead log"). - -This is not a concern of the generic interface, but each implementation should take care to handle this well to avoid accumulation of unused references in the data store and eventual data bloat. - -#### Backing stores - -It is way outside the scope of this project to build our own database that is capable of efficiently storing the data, provide multiple read-only snapshots at once, and save it atomically. The best approach seems to select an existing database (best a simple one) that provides this functionality and build upon it, much like the current `go-merkle` implementation builds upon `leveldb`. After some research here are winners and losers: - -**Winners** - -* Leveldb - [provides consistent snapshots](https://ayende.com/blog/161705/reviewing-leveldb-part-xiii-smile-and-here-is-your-snapshot), and [provides tooling for building ACID compliance](http://codeofrob.com/entries/writing-a-transaction-manager-on-top-of-leveldb.html) - * Note there are at least two solid implementations available in go - [goleveldb](https://github.com/syndtr/goleveldb) - a pure go implementation, and [levigo](https://github.com/jmhodges/levigo) - a go wrapper around leveldb. - * Goleveldb is much easier to compile and cross-compile (not requiring cgo), while levigo (or cleveldb) seems to provide a significant performance boosts (but I had trouble even running benchmarks) -* PostgreSQL - fully supports these ACID semantics if you call `SET TRANSACTION ISOLATION LEVEL SERIALIZABLE` at the beginning of a transaction (tested) - * This may be total overkill unless we also want to make use of other features, like storing data in multiple columns with secondary indexes. - * Trillian can show an example of [how to store a merkle tree in sql](https://github.com/google/trillian/blob/master/storage/mysql/tree_storage.go) - -**Losers** - -* Bolt - open [read-only snapshots can block writing](https://github.com/boltdb/bolt/issues/378) -* Mongo - [barely even supports atomic operations](https://docs.mongodb.com/manual/core/write-operations-atomicity/), much less multiple snapshots - -**To investigate** - -* [Trillian](https://github.com/google/trillian) - has a [persistent merkle tree interface](https://github.com/google/trillian/blob/master/storage/tree_storage.go) along with [backend storage with mysql](https://github.com/google/trillian/blob/master/storage/mysql/tree_storage.go), good inspiration for our design if not directly using it -* [Moss](https://github.com/couchbase/moss) - another key-value store in go, seems similar to leveldb, maybe compare with performance tests? - -### Security - -When allowing access out-of-process, we should provide different mechanisms to secure it. The first is the choice of binding to a local unix socket or a tcp port. The second is the optional use of ssl to encrypt the connection (very important over tcp). The third is authentication to control access to the database. - -We may also want to consider the case of two server connections with different permissions, eg. a local unix socket that allows write access with no more credentials, and a public TCP connection with ssl and authentication that only provides read-only access. - -The use of ssl is quite easy in go, we just need to generate and sign a certificate, so it is nice to be able to disable it for dev machines, but it is very important for production. - -For authentication, let me sketch out a minimal solution. The server could just have a simple config file with key/bcrypt(password) pairs along with read/write permission level, and read that upon startup. The client must provide a username and password in the HTTP headers when making the original HTTPS gRPC connection. - -This is super minimal to provide some protection. Things like LDAP, OAuth and single-sign on seem overkill and even potential security holes. Maybe there is another solution somewhere in the middle. diff --git a/docs/architecture/merkle.md b/docs/architecture/merkle.md deleted file mode 100644 index 4e769aed6..000000000 --- a/docs/architecture/merkle.md +++ /dev/null @@ -1,17 +0,0 @@ -# Merkle data stores - -To allow the efficient creation of an ABCi app, tendermint wishes to provide a reference implemention of a key-value store that provides merkle proofs of the data. These proofs then quickly allow the ABCi app to provide an apphash to the consensus engine, as well as a full proof to any client. - -This engine is currently implemented in `go-merkle` with `merkleeyes` providing a language-agnostic binding via ABCi. It uses `tmlibs/db` bindings internally to persist data to leveldb. - -What are some of the requirements of this store: - -* It must support efficient key-value operations (get/set/delete) -* It must support persistance. -* We must only persist complete blocks, so when we come up after a crash we are at the state of block N or N+1, but not in-between these two states. -* It must allow us to read/write from one uncommited state (working state), while serving other queries from the last commited state. And a way to determine which one to serve for each client. -* It must allow us to hold references to old state, to allow providing proofs from 20 blocks ago. We can define some limits as to the maximum time to hold this data. -* We provide in process binding in Go -* We provide language-agnostic bindings when running the data store as it's own process. - - From 90c0267bc1a490d0501c7fa1c5916132f4d6be1e Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 16 Sep 2017 01:07:04 -0400 Subject: [PATCH 18/92] types: privVal.Sign returns an error --- consensus/byzantine_test.go | 14 +++++++------- types/priv_validator.go | 16 ++++++++++------ types/proposal_test.go | 2 +- types/vote_set_test.go | 6 +++++- 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/consensus/byzantine_test.go b/consensus/byzantine_test.go index 236ba2d40..81a4ab42c 100644 --- a/consensus/byzantine_test.go +++ b/consensus/byzantine_test.go @@ -275,30 +275,30 @@ func (privVal *ByzantinePrivValidator) GetAddress() []byte { return privVal.Address } -func (privVal *ByzantinePrivValidator) SignVote(chainID string, vote *types.Vote) error { +func (privVal *ByzantinePrivValidator) SignVote(chainID string, vote *types.Vote) (err error) { privVal.mtx.Lock() defer privVal.mtx.Unlock() // Sign - vote.Signature = privVal.Sign(types.SignBytes(chainID, vote)) - return nil + vote.Signature, err = privVal.Sign(types.SignBytes(chainID, vote)) + return err } -func (privVal *ByzantinePrivValidator) SignProposal(chainID string, proposal *types.Proposal) error { +func (privVal *ByzantinePrivValidator) SignProposal(chainID string, proposal *types.Proposal) (err error) { privVal.mtx.Lock() defer privVal.mtx.Unlock() // Sign - proposal.Signature = privVal.Sign(types.SignBytes(chainID, proposal)) + proposal.Signature, err = privVal.Sign(types.SignBytes(chainID, proposal)) return nil } -func (privVal *ByzantinePrivValidator) SignHeartbeat(chainID string, heartbeat *types.Heartbeat) error { +func (privVal *ByzantinePrivValidator) SignHeartbeat(chainID string, heartbeat *types.Heartbeat) (err error) { privVal.mtx.Lock() defer privVal.mtx.Unlock() // Sign - heartbeat.Signature = privVal.Sign(types.SignBytes(chainID, heartbeat)) + heartbeat.Signature, err = privVal.Sign(types.SignBytes(chainID, heartbeat)) return nil } diff --git a/types/priv_validator.go b/types/priv_validator.go index 690824934..b96e988db 100644 --- a/types/priv_validator.go +++ b/types/priv_validator.go @@ -59,7 +59,7 @@ type PrivValidator struct { // Currently, the only callers are SignVote and SignProposal type Signer interface { PubKey() crypto.PubKey - Sign(msg []byte) crypto.Signature + Sign(msg []byte) (crypto.Signature, error) } // Implements Signer @@ -72,8 +72,8 @@ func NewDefaultSigner(priv crypto.PrivKey) *DefaultSigner { } // Implements Signer -func (ds *DefaultSigner) Sign(msg []byte) crypto.Signature { - return ds.priv.Sign(msg) +func (ds *DefaultSigner) Sign(msg []byte) (crypto.Signature, error) { + return ds.priv.Sign(msg), nil } // Implements Signer @@ -238,7 +238,10 @@ func (privVal *PrivValidator) signBytesHRS(height, round int, step int8, signByt } // Sign - sig = privVal.Sign(signBytes) + sig, err := privVal.Sign(signBytes) + if err != nil { + return sig, err + } // Persist height/round/step privVal.LastHeight = height @@ -255,8 +258,9 @@ func (privVal *PrivValidator) signBytesHRS(height, round int, step int8, signByt func (privVal *PrivValidator) SignHeartbeat(chainID string, heartbeat *Heartbeat) error { privVal.mtx.Lock() defer privVal.mtx.Unlock() - heartbeat.Signature = privVal.Sign(SignBytes(chainID, heartbeat)) - return nil + var err error + heartbeat.Signature, err = privVal.Sign(SignBytes(chainID, heartbeat)) + return err } func (privVal *PrivValidator) String() string { diff --git a/types/proposal_test.go b/types/proposal_test.go index 622236b60..6f6189595 100644 --- a/types/proposal_test.go +++ b/types/proposal_test.go @@ -37,7 +37,7 @@ func BenchmarkProposalSign(b *testing.B) { func BenchmarkProposalVerifySignature(b *testing.B) { signBytes := SignBytes("test_chain_id", testProposal) privVal := GenPrivValidator() - signature := privVal.Sign(signBytes) + signature, _ := privVal.Sign(signBytes) pubKey := privVal.PubKey for i := 0; i < b.N; i++ { diff --git a/types/vote_set_test.go b/types/vote_set_test.go index 84e13ac17..ddc055489 100644 --- a/types/vote_set_test.go +++ b/types/vote_set_test.go @@ -60,7 +60,11 @@ func withBlockPartsHeader(vote *Vote, blockPartsHeader PartSetHeader) *Vote { } func signAddVote(privVal *PrivValidator, vote *Vote, voteSet *VoteSet) (bool, error) { - vote.Signature = privVal.Sign(SignBytes(voteSet.ChainID(), vote)) + var err error + vote.Signature, err = privVal.Sign(SignBytes(voteSet.ChainID(), vote)) + if err != nil { + return false, err + } added, err := voteSet.AddVote(vote) return added, err } From 12c85c4e601fa3de99d0463d38e5581a1cdbce16 Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Wed, 13 Sep 2017 09:28:37 -0400 Subject: [PATCH 19/92] docs: add original README's from tools repo --- docs/mintnet-kubernetes.md | 229 +++++++++++++++++++++++++++++++++ docs/terraform-digitalocean.md | 66 ++++++++++ docs/using-ansible.md | 202 +++++++++++++++++++++++++++++ docs/using-docker.md | 95 ++++++++++++++ 4 files changed, 592 insertions(+) create mode 100644 docs/mintnet-kubernetes.md create mode 100644 docs/terraform-digitalocean.md create mode 100644 docs/using-ansible.md create mode 100644 docs/using-docker.md diff --git a/docs/mintnet-kubernetes.md b/docs/mintnet-kubernetes.md new file mode 100644 index 000000000..c2928a8e0 --- /dev/null +++ b/docs/mintnet-kubernetes.md @@ -0,0 +1,229 @@ +# Tendermint network powered by Kubernetes + +![Tendermint plus Kubernetes](img/t_plus_k.png) + +* [QuickStart (MacOS)](#quickstart-macos) +* [QuickStart (Linux)](#quickstart-linux) +* [Usage](#usage) +* [Security](#security) +* [Fault tolerance](#fault-tolerance) +* [Starting process](#starting-process) + +This should primarily be used for testing purposes or for tightly-defined +chains operated by a single stakeholder (see [the security +precautions](#security)). If your desire is to launch an application with many +stakeholders, consider using our set of Ansible scripts. + +## QuickStart (MacOS) + +[Requirements](https://github.com/kubernetes/minikube#requirements) + +``` +curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/darwin/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/kubectl +curl -Lo minikube https://storage.googleapis.com/minikube/releases/v0.18.0/minikube-darwin-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/ +minikube start + +git clone https://github.com/tendermint/tools.git && cd tools/mintnet-kubernetes/examples/basecoin && make create +``` + +## QuickStart (Linux) + +[Requirements](https://github.com/kubernetes/minikube#requirements) + +``` +curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/kubectl +curl -Lo minikube https://storage.googleapis.com/minikube/releases/v0.18.0/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/ +minikube start + +git clone https://github.com/tendermint/tools.git && cd tools/mintnet-kubernetes/examples/basecoin && make create +``` + +### Verify everything works + +**Using a shell:** + +1. wait until all the pods are `Running`. + + ``` + kubectl get pods -w -o wide -L tm + ``` + +2. query the Tendermint app logs from the first pod. + + ``` + kubectl logs -c tm -f tm-0 + ``` + +3. use [Rest API](https://tendermint.com/docs/internals/rpc) to fetch the + status of the second pod's Tendermint app. Note we are using `kubectl exec` + because pods are not exposed (and should not be) to the outer network. + + ``` + kubectl exec -c tm tm-0 -- curl -s http://tm-1.basecoin:46657/status | json_pp + ``` + +**Using the dashboard:** + +``` +minikube dashboard +``` + +### Clean up + +``` +make destroy +``` + +## Usage + +### (1/4) Setup a Kubernetes cluster + +- locally using [Minikube](https://github.com/kubernetes/minikube) +- on GCE with a single click in the web UI +- on AWS using [Kubernetes Operations](https://github.com/kubernetes/kops/blob/master/docs/aws.md) +- on Linux machines (Digital Ocean) using [kubeadm](https://kubernetes.io/docs/getting-started-guides/kubeadm/) +- on AWS, Azure, GCE or bare metal using [Kargo (Ansible)](https://kubernetes.io/docs/getting-started-guides/kargo/) + +Please refer to [the official +documentation](https://kubernetes.io/docs/getting-started-guides/) for overview +and comparison of different options. See our guides for [Google Cloud +Engine](docs/SETUP_K8S_ON_GCE.md) or [Digital Ocean](docs/SETUP_K8S_ON_DO.md). + +**Make sure you have Kubernetes >= 1.5, because you will be using StatefulSets, +which is a beta feature in 1.5.** + +### (2/4) Create a configuration file + +Download a template: + +``` +curl -Lo app.yaml https://github.com/tendermint/tools/raw/master/mintnet-kubernetes/app.template.yaml +``` + +Open `app.yaml` in your favorite editor and configure your app container +(navigate to `- name: app`). Kubernetes DSL (Domain Specific Language) is very +simple, so it should be easy. You will need to set Docker image, command and/or +run arguments. Replace variables prefixed with `YOUR_APP` with corresponding +values. Set genesis time to now and preferable chain ID in ConfigMap. + +Please note if you are changing `replicas` number, do not forget to update +`validators` set in ConfigMap. You will be able to scale the cluster up or down +later, but new pods (nodes) won't become validators automatically. + +### (3/4) Deploy your application + +``` +kubectl create -f ./app.yaml +``` + +### (4/4) Observe your cluster + +**web UI** <-> https://github.com/kubernetes/dashboard + +The easiest way to access Dashboard is to use kubectl. Run the following command in your desktop environment: + +``` +kubectl proxy +``` + +kubectl will handle authentication with apiserver and make Dashboard available at [http://localhost:8001/ui](http://localhost:8001/ui) + +**shell** + +List all the pods: + +``` +kubectl get pods -o wide -L tm +``` + +StatefulSet details: + +``` +kubectl describe statefulsets tm +``` + +First pod details: + +``` +kubectl describe pod tm-0 +``` + +Tendermint app logs from the first pod: + +``` +kubectl logs tm-0 -c tm -f +``` + +App logs from the first pod: + +``` +kubectl logs tm-0 -c app -f +``` + +Status of the second pod's Tendermint app: + +``` +kubectl exec -c tm tm-0 -- curl -s http://tm-1.:46657/status | json_pp +``` + +## Security + +Due to the nature of Kubernetes, where you typically have a single master, the +master could be a SPOF (Single Point Of Failure). Therefore, you need to make +sure only authorized people can access it. And these people themselves had +taken basic measures in order not to get hacked. + +These are the best practices: + +- all access to the master is over TLS +- access to the API Server is X.509 certificate or token based +- etcd is not exposed directly to the cluster +- ensure that images are free of vulnerabilities ([1](https://github.com/coreos/clair)) +- ensure that only authorized images are used in your environment +- disable direct access to Kubernetes nodes (no SSH) +- define resource quota + +Resources: + +- https://kubernetes.io/docs/admin/accessing-the-api/ +- http://blog.kubernetes.io/2016/08/security-best-practices-kubernetes-deployment.html +- https://blog.openshift.com/securing-kubernetes/ + +## Fault tolerance + +Having a single master (API server) is a bad thing also because if something +happens to it, you risk being left without an access to the application. + +To avoid that you can [run Kubernetes in multiple +zones](https://kubernetes.io/docs/admin/multiple-zones/), each zone running an +[API server](https://kubernetes.io/docs/admin/high-availability/) and load +balance requests between them. Do not forget to make sure only one instance of +scheduler and controller-manager are running at once. + +Running in multiple zones is a lightweight version of a broader [Cluster +Federation feature](https://kubernetes.io/docs/admin/federation/). Federated +deployments could span across multiple regions (not zones). We haven't tried +this feature yet, so any feedback is highly appreciated! Especially, related to +additional latency and cost of exchanging data between the regions. + +Resources: + +- https://kubernetes.io/docs/admin/high-availability/ + +## Starting process + +![StatefulSet](img/statefulset.png) + +Init containers (`tm-gen-validator`) are run before all other containers, +creating public-private key pair for each pod. Every `tm` container then asks +other pods for their public keys, which are served with nginx (`pub-key` +container). When `tm` container have all the keys, it forms a genesis file and +starts Tendermint process. + +## TODO + +- [ ] run tendermint from tmuser + ``` + securityContext: + fsGroup: 999 + ``` diff --git a/docs/terraform-digitalocean.md b/docs/terraform-digitalocean.md new file mode 100644 index 000000000..7e8bbd959 --- /dev/null +++ b/docs/terraform-digitalocean.md @@ -0,0 +1,66 @@ +# Terraform for Digital Ocean + +This is a generic [Terraform](https://www.terraform.io/) configuration that sets up DigitalOcean droplets. + +# Prerequisites + +* Install [HashiCorp Terraform](https://www.terraform.io) on a linux machine. +* Create a [DigitalOcean API token](https://cloud.digitalocean.com/settings/api/tokens) with read and write capability. +* Create a private/public key pair for SSH. This is needed to log onto your droplets as well as by Ansible to connect for configuration changes. +* Set up the public SSH key at the [DigitalOcean security page](https://cloud.digitalocean.com/settings/security). [Here](https://www.digitalocean.com/community/tutorials/how-to-use-ssh-keys-with-digitalocean-droplets)'s a tutorial. +* Find out your SSH key ID at DigitalOcean by querying the below command on your linux box: + +``` +DO_API_TOKEN="" +curl -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $DO_API_TOKEN" "https://api.digitalocean.com/v2/account/keys" +``` + +# How to run + +## Initialization +If this is your first time using terraform, you have to initialize it by running the below command. (Note: initialization can be run multiple times) +``` +terraform init +``` + +After initialization it's good measure to create a new Terraform environment for the droplets so they are always managed together. + +``` +TESTNET_NAME="testnet-servers" +terraform env new "$TESTNET_NAME" +``` + +Note this `terraform env` command is only available in terraform `v0.9` and up. + +## Execution + +The below command will create 4 nodes in DigitalOcean. They will be named `testnet-servers-node0` to `testnet-servers-node3` and they will be tagged as `testnet-servers`. +``` +DO_API_TOKEN="" +SSH_IDS="[ \"\" ]" +terraform apply -var TESTNET_NAME="testnet-servers" -var servers=4 -var DO_API_TOKEN="$DO_API_TOKEN" -var ssh_keys="$SSH_IDS" +``` + +Note: `ssh_keys` is a list of strings. You can add multiple keys. For example: `["1234567","9876543"]`. + +Alternatively you can use the default settings. The number of default servers is 4 and the testnet name is `tf-testnet1`. Variables can also be defined as environment variables instead of the command-line. Environment variables that start with `TF_VAR_` will be translated into the Terraform configuration. For example the number of servers can be overriden by setting the `TF_VAR_servers` variable. + +``` +TF_VAR_DO_API_TOKEN="" +TF_VAR_TESTNET_NAME="testnet-servers" +terraform-apply +``` + +## Security + +DigitalOcean uses the root user by default on its droplets. This is fine as long as SSH keys are used. However some people still would like to disable root and use an alternative user to connect to the droplets - then `sudo` from there. +Terraform can do this but it requires SSH agent running on the machine where terraform is run, with one of the SSH keys of the droplets added to the agent. (This will be neede for ansible too, so it's worth setting it up here. Check out the [ansible](https://github.com/tendermint/tools/tree/master/ansible) page for more information.) +After setting up the SSH key, run `terraform apply` with `-var noroot=true` to create your droplets. Terraform will create a user called `ec2-user` and move the SSH keys over, this way disabling SSH login for root. It also adds the `ec2-user` to the sudoers file, so after logging in as ec2-user you can `sudo` to `root`. + +DigitalOcean announced firewalls but the current version of Terraform (0.9.8 as of this writing) does not support it yet. Fortunately it is quite easy to set it up through the web interface (and not that bad through the [RESTful API](https://developers.digitalocean.com/documentation/v2/#firewalls) either). When adding droplets to a firewall rule, you can add tags. All droplets in a testnet are tagged with the testnet name so it's enough to define the testnet name in the firewall rule. It is not necessary to add the nodes one-by-one. Also, the firewall rule "remembers" the testnet name tag so if you change the servers but keep the name, the firewall rules will still apply. + +# What's next + +After setting up the nodes, head over to the [ansible folder](https://github.com/tendermint/tools/tree/master/ansible) to set up tendermint and basecoin. + + diff --git a/docs/using-ansible.md b/docs/using-ansible.md new file mode 100644 index 000000000..03a32f55d --- /dev/null +++ b/docs/using-ansible.md @@ -0,0 +1,202 @@ +# Ansible playbook for Tendermint applications + +![Ansible plus Tendermint](img/a_plus_t.png) + +* [Prerequisites](#Prerequisites) +* [Ansible setup](#Ansible setup) +* [Running the playbook](#Running the playbook) + +The playbooks in this folder run [ansible](http://www.ansible.com/) roles which: + +* install and configure basecoin or ethermint +* start/stop basecoin or ethermint and reset their configuration + +## Prerequisites + +* Ansible 2.0 or higher +* SSH key to the servers + +Optional for DigitalOcean droplets: +* DigitalOcean API Token +* python dopy package + +Head over to the [Terraform folder](https://github.com/tendermint/tools/tree/master/terraform-digitalocean) for a description on how to get a DigitalOcean API Token. + +Optional for Amazon AWS instances: +* Amazon AWS API access key ID and secret access key. + +The cloud inventory scripts come from the ansible team at their [GitHub](https://github.com/ansible/ansible) page. You can get the latest version from the contrib/inventory folder. + +## Ansible setup + +Ansible requires a "command machine" or "local machine" or "orchestrator machine" to run on. This can be your laptop or any machine that can run ansible. (It does not have to be part of the cloud network that hosts your servers.) + +Use the official [Ansible installation guide](http://docs.ansible.com/ansible/intro_installation.html) to install Ansible. Here are a few examples on basic installation commands: + +Ubuntu/Debian: +``` +sudo apt-get install ansible +``` + +CentOS/RedHat: +``` +sudo yum install epel-release +sudo yum install ansible +``` + +Mac OSX: +If you have (Homebrew)[https://brew.sh] installed, then it's simply +``` +brew install ansible +``` + +If not, you can install it using `pip`: +``` +sudo easy_install pip +sudo pip install ansible +``` + + +To make life easier, you can start an SSH Agent and load your SSH key(s). This way ansible will have an uninterrupted way of connecting to your servers. + +``` +ssh-agent > ~/.ssh/ssh.env +source ~/.ssh/ssh.env + +ssh-add private.key +``` + +Subsequently, as long as the agent is running, you can use `source ~/.ssh/ssh.env` to load the keys to the current session. +Note: On Mac OSX, you can add the `-K` option to ssh-add to store the passphrase in your keychain. The security of this feature is debated but it is convenient. + +### Optional cloud dependencies + +If you are using a cloud provider to host your servers, you need the below dependencies installed on your local machine. + +#### DigitalOcean inventory dependencies: + +Ubuntu/Debian: +``` +sudo apt-get install python-pip +sudo pip install dopy +``` + +CentOS/RedHat: +``` +sudo yum install python-pip +sudo pip install dopy +``` + +Mac OSX: +``` +sudo pip install dopy +``` + +#### Amazon AWS inventory dependencies: + +Ubuntu/Debian: +``` +sudo apt-get install python-boto +``` + +CentOS/RedHat: +``` +sudo yum install python-boto +``` + +Mac OSX: +``` +sudo pip install boto +``` + +## Refreshing the DigitalOcean inventory + +If you just finished creating droplets, the local DigitalOcean inventory cache is not up-to-date. To refresh it, run: + +``` +DO_API_TOKEN="" +python -u inventory/digital_ocean.py --refresh-cache 1> /dev/null +``` + +## Refreshing the Amazon AWS inventory + +If you just finished creating Amazon AWS EC2 instances, the local AWS inventory cache is not up-to-date. To refresh it, run: + +``` +AWS_ACCESS_KEY_ID='' +AWS_SECRET_ACCESS_KEY='' +python -u inventory/ec2.py --refresh-cache 1> /dev/null +``` + +Note: you don't need the access key and secret key set, if you are running ansible on an Amazon AMI instance with the proper IAM permissions set. + +## Running the playbooks + +The playbooks are locked down to only run if the environment variable `TF_VAR_TESTNET_NAME` is populated. This is a precaution so you don't accidentally run the playbook on all your servers. + +The variable `TF_VAR_TESTNET_NAME` contains the testnet name which ansible translates into an ansible group. If you used Terraform to create the servers, it was the testnet name used there. + +If the playbook cannot connect to the servers because of public key denial, your SSH Agent is not set up properly. Alternatively you can add the SSH key to ansible using the `--private-key` option. + +If you need to connect to the nodes as root but your local username is different, use the ansible option `-u root` to tell ansible to connect to the servers and authenticate as the root user. + +If you secured your server and you need to `sudo` for root access, use the the `-b` or `--become` option to tell ansible to sudo to root after connecting to the server. In the Terraform-DigitalOcean example, if you created the ec2-user by adding the `noroot=true` option (or if you are simply on Amazon AWS), you need to add the options `-u ec2-user -b` to ansible to tell it to connect as the ec2-user and then sudo to root to run the playbook. + +### DigitalOcean +``` +DO_API_TOKEN="" +TF_VAR_TESTNET_NAME="testnet-servers" +ansible-playbook -i inventory/digital_ocean.py install.yml -e service=basecoin +``` + +### Amazon AWS +``` +AWS_ACCESS_KEY_ID='' +AWS_SECRET_ACCESS_KEY='' +TF_VAR_TESTNET_NAME="testnet-servers" +ansible-playbook -i inventory/ec2.py install.yml -e service=basecoin +``` + +### Installing custom versions + +By default ansible installs the tendermint, basecoin or ethermint binary versions from the latest release in the repository. If you build your own version of the binaries, you can tell ansible to install that instead. + +``` +GOPATH="" +go get -u github.com/tendermint/basecoin/cmd/basecoin + +DO_API_TOKEN="" +TF_VAR_TESTNET_NAME="testnet-servers" +ansible-playbook -i inventory/digital_ocean.py install.yml -e service=basecoin -e release_install=false +``` + +Alternatively you can change the variable settings in `group_vars/all`. + +## Other commands and roles + +There are few extra playbooks to make life easier managing your servers. + +* install.yml - Install basecoin or ethermint applications. (Tendermint gets installed automatically.) Use the `service` parameter to define which application to install. Defaults to `basecoin`. +* reset.yml - Stop the application, reset the configuration and data, then start the application again. You need to pass `-e service=`, like `-e service=basecoin`. It will restart the underlying tendermint application too. +* restart.yml - Restart a service on all nodes. You need to pass `-e service=`, like `-e service=basecoin`. It will restart the underlying tendermint application too. +* stop.yml - Stop the application. You need to pass `-e service=`. +* status.yml - Check the service status and print it. You need to pass `-e service=`. +* start.yml - Start the application. You need to pass `-e service=`. +* ubuntu16-patch.yml - Ubuntu 16.04 does not have the minimum required python package installed to be able to run ansible. If you are using ubuntu, run this playbook first on the target machines. This will install the python pacakge that is required for ansible to work correctly on the remote nodes. +* upgrade.yml - Upgrade the `service` on your testnet. It will stop the service and restart it at the end. It will only work if the upgraded version is backward compatible with the installed version. +* upgrade-reset.yml - Upgrade the `service` on your testnet and reset the database. It will stop the service and restart it at the end. It will work for upgrades where the new version is not backward-compatible with the installed version - however it will reset the testnet to its default. + +The roles are self-sufficient under the `roles/` folder. + +* install - install the application defined in the `service` parameter. It can install release packages and update them with custom-compiled binaries. +* unsafe_reset - delete the database for a service, including the tendermint database. +* config - configure the application defined in `service`. It also configures the underlying tendermint service. Check `group_vars/all` for options. +* stop - stop an application. Requires the `service` parameter set. +* status - check the status of an application. Requires the `service` parameter set. +* start - start an application. Requires the `service` parameter set. + +## Default variables + +Default variables are documented under `group_vars/all`. You can the parameters there to deploy a previously created genesis.json file (instead of dynamically creating it) or if you want to deploy custom built binaries instead of deploying a released version. + + diff --git a/docs/using-docker.md b/docs/using-docker.md new file mode 100644 index 000000000..fc8679739 --- /dev/null +++ b/docs/using-docker.md @@ -0,0 +1,95 @@ +# Docker container description for Tendermint applications + +* Overview (#Overview) +* Tendermint (#Tendermint) +* Basecoin (#Basecoin) +* Ethermint (#Ethermint) + +## Overview + +This folder contains Docker container descriptions. Using this folder you can build your own Docker images with the tendermint application. + +It is assumed that you set up docker already. + +If you don't want to build the images yourself, you should be able to download them from Docker Hub. + +## Tendermint + +Build the container: +Copy the `tendermint` binary to the `tendermint` folder. +``` +docker build -t tendermint tendermint +``` + +The application configuration will be stored at `/tendermint` in the container. The ports 46656 and 46657 will be open for ABCI applications to connect. + +Initialize tendermint configuration and keep it after the container is finished in a docker volume called `data`: +``` +docker run --rm -v data:/tendermint tendermint init +``` +If you want the docker volume to be a physical directory on your filesystem, you have to give an absolute path to docker and make sure the permissions allow the application to write it. + +Get the public key of tendermint: +``` +docker run --rm -v data:/tendermint tendermint show_validator +``` + +Run the docker tendermint application with: +``` +docker run --rm -d -v data:/tendermint tendermint node +``` + +## Basecoin + +Build the container: +Copy the `basecoin` binary to the `basecoin` folder. +``` +docker build -t basecoin basecoin +``` +The application configuration will be stored at `/basecoin`. + +Initialize basecoin configuration and keep it after the container is finished: +``` +docker run --rm -v basecoindata:/basecoin basecoin init deadbeef +``` +Use your own basecoin account instead of `deadbeef` in the `init` command. + +Get the public key of basecoin: +We use a trick here: since the basecoin and the tendermint configuration folders are similar, the `tendermint` command can extract the public key for us if we feed the basecoin configuration folder to tendermint. +``` +docker run --rm -v basecoindata:/tendermint tendermint show_validator +``` + +Run the docker tendermint application with: +This is a two-step process: +* Run the basecoin container. +* Run the tendermint container and expose the ports that allow clients to connect. The --proxy_app should contain the basecoin application's IP address and port. +``` +docker run --rm -d -v basecoindata:/basecoin basecoin start --without-tendermint +docker run --rm -d -v data:/tendermint -p 46656-46657:46656-46657 tendermint node --proxy_app tcp://172.17.0.2:46658 +``` + +## Ethermint + +Build the container: +Copy the `ethermint` binary and the setup folder to the `ethermint` folder. +``` +docker build -t ethermint ethermint +``` +The application configuration will be stored at `/ethermint`. +The files required for initializing ethermint (the files in the source `setup` folder) are under `/setup`. + +Initialize ethermint configuration: +``` +docker run --rm -v ethermintdata:/ethermint ethermint init /setup/genesis.json +``` + +Start ethermint as a validator node: +This is a two-step process: +* Run the ethermint container. You will have to define where tendermint runs as the ethermint binary connects to it explicitly. +* Run the tendermint container and expose the ports that allow clients to connect. The --proxy_app should contain the ethermint application's IP address and port. +``` +docker run --rm -d -v ethermintdata:/ethermint ethermint --tendermint_addr tcp://172.17.0.3:46657 +docker run --rm -d -v data:/tendermint -p 46656-46657:46656-46657 tendermint node --proxy_app tcp://172.17.0.2:46658 +``` + From d4ccc88676f173712604b01df85462246ed8c5cc Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Wed, 13 Sep 2017 09:35:02 -0400 Subject: [PATCH 20/92] docs: convert from md to rst --- docs/mintnet-kubernetes.md | 229 ------------------------- docs/mintnet-kubernetes.rst | 253 +++++++++++++++++++++++++++ docs/terraform-digitalocean.md | 66 ------- docs/terraform-digitalocean.rst | 119 +++++++++++++ docs/using-ansible.md | 202 ---------------------- docs/using-ansible.rst | 294 ++++++++++++++++++++++++++++++++ docs/using-docker.md | 95 ----------- docs/using-docker.rst | 128 ++++++++++++++ 8 files changed, 794 insertions(+), 592 deletions(-) delete mode 100644 docs/mintnet-kubernetes.md create mode 100644 docs/mintnet-kubernetes.rst delete mode 100644 docs/terraform-digitalocean.md create mode 100644 docs/terraform-digitalocean.rst delete mode 100644 docs/using-ansible.md create mode 100644 docs/using-ansible.rst delete mode 100644 docs/using-docker.md create mode 100644 docs/using-docker.rst diff --git a/docs/mintnet-kubernetes.md b/docs/mintnet-kubernetes.md deleted file mode 100644 index c2928a8e0..000000000 --- a/docs/mintnet-kubernetes.md +++ /dev/null @@ -1,229 +0,0 @@ -# Tendermint network powered by Kubernetes - -![Tendermint plus Kubernetes](img/t_plus_k.png) - -* [QuickStart (MacOS)](#quickstart-macos) -* [QuickStart (Linux)](#quickstart-linux) -* [Usage](#usage) -* [Security](#security) -* [Fault tolerance](#fault-tolerance) -* [Starting process](#starting-process) - -This should primarily be used for testing purposes or for tightly-defined -chains operated by a single stakeholder (see [the security -precautions](#security)). If your desire is to launch an application with many -stakeholders, consider using our set of Ansible scripts. - -## QuickStart (MacOS) - -[Requirements](https://github.com/kubernetes/minikube#requirements) - -``` -curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/darwin/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/kubectl -curl -Lo minikube https://storage.googleapis.com/minikube/releases/v0.18.0/minikube-darwin-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/ -minikube start - -git clone https://github.com/tendermint/tools.git && cd tools/mintnet-kubernetes/examples/basecoin && make create -``` - -## QuickStart (Linux) - -[Requirements](https://github.com/kubernetes/minikube#requirements) - -``` -curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/kubectl -curl -Lo minikube https://storage.googleapis.com/minikube/releases/v0.18.0/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/ -minikube start - -git clone https://github.com/tendermint/tools.git && cd tools/mintnet-kubernetes/examples/basecoin && make create -``` - -### Verify everything works - -**Using a shell:** - -1. wait until all the pods are `Running`. - - ``` - kubectl get pods -w -o wide -L tm - ``` - -2. query the Tendermint app logs from the first pod. - - ``` - kubectl logs -c tm -f tm-0 - ``` - -3. use [Rest API](https://tendermint.com/docs/internals/rpc) to fetch the - status of the second pod's Tendermint app. Note we are using `kubectl exec` - because pods are not exposed (and should not be) to the outer network. - - ``` - kubectl exec -c tm tm-0 -- curl -s http://tm-1.basecoin:46657/status | json_pp - ``` - -**Using the dashboard:** - -``` -minikube dashboard -``` - -### Clean up - -``` -make destroy -``` - -## Usage - -### (1/4) Setup a Kubernetes cluster - -- locally using [Minikube](https://github.com/kubernetes/minikube) -- on GCE with a single click in the web UI -- on AWS using [Kubernetes Operations](https://github.com/kubernetes/kops/blob/master/docs/aws.md) -- on Linux machines (Digital Ocean) using [kubeadm](https://kubernetes.io/docs/getting-started-guides/kubeadm/) -- on AWS, Azure, GCE or bare metal using [Kargo (Ansible)](https://kubernetes.io/docs/getting-started-guides/kargo/) - -Please refer to [the official -documentation](https://kubernetes.io/docs/getting-started-guides/) for overview -and comparison of different options. See our guides for [Google Cloud -Engine](docs/SETUP_K8S_ON_GCE.md) or [Digital Ocean](docs/SETUP_K8S_ON_DO.md). - -**Make sure you have Kubernetes >= 1.5, because you will be using StatefulSets, -which is a beta feature in 1.5.** - -### (2/4) Create a configuration file - -Download a template: - -``` -curl -Lo app.yaml https://github.com/tendermint/tools/raw/master/mintnet-kubernetes/app.template.yaml -``` - -Open `app.yaml` in your favorite editor and configure your app container -(navigate to `- name: app`). Kubernetes DSL (Domain Specific Language) is very -simple, so it should be easy. You will need to set Docker image, command and/or -run arguments. Replace variables prefixed with `YOUR_APP` with corresponding -values. Set genesis time to now and preferable chain ID in ConfigMap. - -Please note if you are changing `replicas` number, do not forget to update -`validators` set in ConfigMap. You will be able to scale the cluster up or down -later, but new pods (nodes) won't become validators automatically. - -### (3/4) Deploy your application - -``` -kubectl create -f ./app.yaml -``` - -### (4/4) Observe your cluster - -**web UI** <-> https://github.com/kubernetes/dashboard - -The easiest way to access Dashboard is to use kubectl. Run the following command in your desktop environment: - -``` -kubectl proxy -``` - -kubectl will handle authentication with apiserver and make Dashboard available at [http://localhost:8001/ui](http://localhost:8001/ui) - -**shell** - -List all the pods: - -``` -kubectl get pods -o wide -L tm -``` - -StatefulSet details: - -``` -kubectl describe statefulsets tm -``` - -First pod details: - -``` -kubectl describe pod tm-0 -``` - -Tendermint app logs from the first pod: - -``` -kubectl logs tm-0 -c tm -f -``` - -App logs from the first pod: - -``` -kubectl logs tm-0 -c app -f -``` - -Status of the second pod's Tendermint app: - -``` -kubectl exec -c tm tm-0 -- curl -s http://tm-1.:46657/status | json_pp -``` - -## Security - -Due to the nature of Kubernetes, where you typically have a single master, the -master could be a SPOF (Single Point Of Failure). Therefore, you need to make -sure only authorized people can access it. And these people themselves had -taken basic measures in order not to get hacked. - -These are the best practices: - -- all access to the master is over TLS -- access to the API Server is X.509 certificate or token based -- etcd is not exposed directly to the cluster -- ensure that images are free of vulnerabilities ([1](https://github.com/coreos/clair)) -- ensure that only authorized images are used in your environment -- disable direct access to Kubernetes nodes (no SSH) -- define resource quota - -Resources: - -- https://kubernetes.io/docs/admin/accessing-the-api/ -- http://blog.kubernetes.io/2016/08/security-best-practices-kubernetes-deployment.html -- https://blog.openshift.com/securing-kubernetes/ - -## Fault tolerance - -Having a single master (API server) is a bad thing also because if something -happens to it, you risk being left without an access to the application. - -To avoid that you can [run Kubernetes in multiple -zones](https://kubernetes.io/docs/admin/multiple-zones/), each zone running an -[API server](https://kubernetes.io/docs/admin/high-availability/) and load -balance requests between them. Do not forget to make sure only one instance of -scheduler and controller-manager are running at once. - -Running in multiple zones is a lightweight version of a broader [Cluster -Federation feature](https://kubernetes.io/docs/admin/federation/). Federated -deployments could span across multiple regions (not zones). We haven't tried -this feature yet, so any feedback is highly appreciated! Especially, related to -additional latency and cost of exchanging data between the regions. - -Resources: - -- https://kubernetes.io/docs/admin/high-availability/ - -## Starting process - -![StatefulSet](img/statefulset.png) - -Init containers (`tm-gen-validator`) are run before all other containers, -creating public-private key pair for each pod. Every `tm` container then asks -other pods for their public keys, which are served with nginx (`pub-key` -container). When `tm` container have all the keys, it forms a genesis file and -starts Tendermint process. - -## TODO - -- [ ] run tendermint from tmuser - ``` - securityContext: - fsGroup: 999 - ``` diff --git a/docs/mintnet-kubernetes.rst b/docs/mintnet-kubernetes.rst new file mode 100644 index 000000000..583b58248 --- /dev/null +++ b/docs/mintnet-kubernetes.rst @@ -0,0 +1,253 @@ +Tendermint network powered by Kubernetes +======================================== + +.. figure:: img/t_plus_k.png + :alt: Tendermint plus Kubernetes + + Tendermint plus Kubernetes + +- `QuickStart (MacOS) <#quickstart-macos>`__ +- `QuickStart (Linux) <#quickstart-linux>`__ +- `Usage <#usage>`__ +- `Security <#security>`__ +- `Fault tolerance <#fault-tolerance>`__ +- `Starting process <#starting-process>`__ + +This should primarily be used for testing purposes or for +tightly-defined chains operated by a single stakeholder (see `the +security precautions <#security>`__). If your desire is to launch an +application with many stakeholders, consider using our set of Ansible +scripts. + +QuickStart (MacOS) +------------------ + +`Requirements `__ + +:: + + curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/darwin/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/kubectl + curl -Lo minikube https://storage.googleapis.com/minikube/releases/v0.18.0/minikube-darwin-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/ + minikube start + + git clone https://github.com/tendermint/tools.git && cd tools/mintnet-kubernetes/examples/basecoin && make create + +QuickStart (Linux) +------------------ + +`Requirements `__ + +:: + + curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/kubectl + curl -Lo minikube https://storage.googleapis.com/minikube/releases/v0.18.0/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/ + minikube start + + git clone https://github.com/tendermint/tools.git && cd tools/mintnet-kubernetes/examples/basecoin && make create + +Verify everything works +~~~~~~~~~~~~~~~~~~~~~~~ + +**Using a shell:** + +1. wait until all the pods are ``Running``. + +``kubectl get pods -w -o wide -L tm`` + +2. query the Tendermint app logs from the first pod. + +``kubectl logs -c tm -f tm-0`` + +3. use `Rest API `__ to fetch + the status of the second pod's Tendermint app. Note we are using + ``kubectl exec`` because pods are not exposed (and should not be) to + the outer network. + +``kubectl exec -c tm tm-0 -- curl -s http://tm-1.basecoin:46657/status | json_pp`` + +**Using the dashboard:** + +:: + + minikube dashboard + +Clean up +~~~~~~~~ + +:: + + make destroy + +Usage +----- + +(1/4) Setup a Kubernetes cluster +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- locally using `Minikube `__ +- on GCE with a single click in the web UI +- on AWS using `Kubernetes + Operations `__ +- on Linux machines (Digital Ocean) using + `kubeadm `__ +- on AWS, Azure, GCE or bare metal using `Kargo + (Ansible) `__ + +Please refer to `the official +documentation `__ +for overview and comparison of different options. See our guides for +`Google Cloud Engine `__ or `Digital +Ocean `__. + +**Make sure you have Kubernetes >= 1.5, because you will be using +StatefulSets, which is a beta feature in 1.5.** + +(2/4) Create a configuration file +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Download a template: + +:: + + curl -Lo app.yaml https://github.com/tendermint/tools/raw/master/mintnet-kubernetes/app.template.yaml + +Open ``app.yaml`` in your favorite editor and configure your app +container (navigate to ``- name: app``). Kubernetes DSL (Domain Specific +Language) is very simple, so it should be easy. You will need to set +Docker image, command and/or run arguments. Replace variables prefixed +with ``YOUR_APP`` with corresponding values. Set genesis time to now and +preferable chain ID in ConfigMap. + +Please note if you are changing ``replicas`` number, do not forget to +update ``validators`` set in ConfigMap. You will be able to scale the +cluster up or down later, but new pods (nodes) won't become validators +automatically. + +(3/4) Deploy your application +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + kubectl create -f ./app.yaml + +(4/4) Observe your cluster +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**web UI** <-> https://github.com/kubernetes/dashboard + +The easiest way to access Dashboard is to use kubectl. Run the following +command in your desktop environment: + +:: + + kubectl proxy + +kubectl will handle authentication with apiserver and make Dashboard +available at http://localhost:8001/ui + +**shell** + +List all the pods: + +:: + + kubectl get pods -o wide -L tm + +StatefulSet details: + +:: + + kubectl describe statefulsets tm + +First pod details: + +:: + + kubectl describe pod tm-0 + +Tendermint app logs from the first pod: + +:: + + kubectl logs tm-0 -c tm -f + +App logs from the first pod: + +:: + + kubectl logs tm-0 -c app -f + +Status of the second pod's Tendermint app: + +:: + + kubectl exec -c tm tm-0 -- curl -s http://tm-1.:46657/status | json_pp + +Security +-------- + +Due to the nature of Kubernetes, where you typically have a single +master, the master could be a SPOF (Single Point Of Failure). Therefore, +you need to make sure only authorized people can access it. And these +people themselves had taken basic measures in order not to get hacked. + +These are the best practices: + +- all access to the master is over TLS +- access to the API Server is X.509 certificate or token based +- etcd is not exposed directly to the cluster +- ensure that images are free of vulnerabilities + (`1 `__) +- ensure that only authorized images are used in your environment +- disable direct access to Kubernetes nodes (no SSH) +- define resource quota + +Resources: + +- https://kubernetes.io/docs/admin/accessing-the-api/ +- http://blog.kubernetes.io/2016/08/security-best-practices-kubernetes-deployment.html +- https://blog.openshift.com/securing-kubernetes/ + +Fault tolerance +--------------- + +Having a single master (API server) is a bad thing also because if +something happens to it, you risk being left without an access to the +application. + +To avoid that you can `run Kubernetes in multiple +zones `__, each zone +running an `API +server `__ and load +balance requests between them. Do not forget to make sure only one +instance of scheduler and controller-manager are running at once. + +Running in multiple zones is a lightweight version of a broader `Cluster +Federation feature `__. +Federated deployments could span across multiple regions (not zones). We +haven't tried this feature yet, so any feedback is highly appreciated! +Especially, related to additional latency and cost of exchanging data +between the regions. + +Resources: + +- https://kubernetes.io/docs/admin/high-availability/ + +Starting process +---------------- + +.. figure:: img/statefulset.png + :alt: StatefulSet + + StatefulSet + +Init containers (``tm-gen-validator``) are run before all other +containers, creating public-private key pair for each pod. Every ``tm`` +container then asks other pods for their public keys, which are served +with nginx (``pub-key`` container). When ``tm`` container have all the +keys, it forms a genesis file and starts Tendermint process. + +TODO +---- + +- [ ] run tendermint from tmuser ``securityContext: fsGroup: 999`` diff --git a/docs/terraform-digitalocean.md b/docs/terraform-digitalocean.md deleted file mode 100644 index 7e8bbd959..000000000 --- a/docs/terraform-digitalocean.md +++ /dev/null @@ -1,66 +0,0 @@ -# Terraform for Digital Ocean - -This is a generic [Terraform](https://www.terraform.io/) configuration that sets up DigitalOcean droplets. - -# Prerequisites - -* Install [HashiCorp Terraform](https://www.terraform.io) on a linux machine. -* Create a [DigitalOcean API token](https://cloud.digitalocean.com/settings/api/tokens) with read and write capability. -* Create a private/public key pair for SSH. This is needed to log onto your droplets as well as by Ansible to connect for configuration changes. -* Set up the public SSH key at the [DigitalOcean security page](https://cloud.digitalocean.com/settings/security). [Here](https://www.digitalocean.com/community/tutorials/how-to-use-ssh-keys-with-digitalocean-droplets)'s a tutorial. -* Find out your SSH key ID at DigitalOcean by querying the below command on your linux box: - -``` -DO_API_TOKEN="" -curl -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $DO_API_TOKEN" "https://api.digitalocean.com/v2/account/keys" -``` - -# How to run - -## Initialization -If this is your first time using terraform, you have to initialize it by running the below command. (Note: initialization can be run multiple times) -``` -terraform init -``` - -After initialization it's good measure to create a new Terraform environment for the droplets so they are always managed together. - -``` -TESTNET_NAME="testnet-servers" -terraform env new "$TESTNET_NAME" -``` - -Note this `terraform env` command is only available in terraform `v0.9` and up. - -## Execution - -The below command will create 4 nodes in DigitalOcean. They will be named `testnet-servers-node0` to `testnet-servers-node3` and they will be tagged as `testnet-servers`. -``` -DO_API_TOKEN="" -SSH_IDS="[ \"\" ]" -terraform apply -var TESTNET_NAME="testnet-servers" -var servers=4 -var DO_API_TOKEN="$DO_API_TOKEN" -var ssh_keys="$SSH_IDS" -``` - -Note: `ssh_keys` is a list of strings. You can add multiple keys. For example: `["1234567","9876543"]`. - -Alternatively you can use the default settings. The number of default servers is 4 and the testnet name is `tf-testnet1`. Variables can also be defined as environment variables instead of the command-line. Environment variables that start with `TF_VAR_` will be translated into the Terraform configuration. For example the number of servers can be overriden by setting the `TF_VAR_servers` variable. - -``` -TF_VAR_DO_API_TOKEN="" -TF_VAR_TESTNET_NAME="testnet-servers" -terraform-apply -``` - -## Security - -DigitalOcean uses the root user by default on its droplets. This is fine as long as SSH keys are used. However some people still would like to disable root and use an alternative user to connect to the droplets - then `sudo` from there. -Terraform can do this but it requires SSH agent running on the machine where terraform is run, with one of the SSH keys of the droplets added to the agent. (This will be neede for ansible too, so it's worth setting it up here. Check out the [ansible](https://github.com/tendermint/tools/tree/master/ansible) page for more information.) -After setting up the SSH key, run `terraform apply` with `-var noroot=true` to create your droplets. Terraform will create a user called `ec2-user` and move the SSH keys over, this way disabling SSH login for root. It also adds the `ec2-user` to the sudoers file, so after logging in as ec2-user you can `sudo` to `root`. - -DigitalOcean announced firewalls but the current version of Terraform (0.9.8 as of this writing) does not support it yet. Fortunately it is quite easy to set it up through the web interface (and not that bad through the [RESTful API](https://developers.digitalocean.com/documentation/v2/#firewalls) either). When adding droplets to a firewall rule, you can add tags. All droplets in a testnet are tagged with the testnet name so it's enough to define the testnet name in the firewall rule. It is not necessary to add the nodes one-by-one. Also, the firewall rule "remembers" the testnet name tag so if you change the servers but keep the name, the firewall rules will still apply. - -# What's next - -After setting up the nodes, head over to the [ansible folder](https://github.com/tendermint/tools/tree/master/ansible) to set up tendermint and basecoin. - - diff --git a/docs/terraform-digitalocean.rst b/docs/terraform-digitalocean.rst new file mode 100644 index 000000000..399a0a76d --- /dev/null +++ b/docs/terraform-digitalocean.rst @@ -0,0 +1,119 @@ +Terraform for Digital Ocean +=========================== + +This is a generic `Terraform `__ +configuration that sets up DigitalOcean droplets. + +Prerequisites +============= + +- Install `HashiCorp Terraform `__ on a linux + machine. +- Create a `DigitalOcean API + token `__ with + read and write capability. +- Create a private/public key pair for SSH. This is needed to log onto + your droplets as well as by Ansible to connect for configuration + changes. +- Set up the public SSH key at the `DigitalOcean security + page `__. + `Here `__'s + a tutorial. +- Find out your SSH key ID at DigitalOcean by querying the below + command on your linux box: + +:: + + DO_API_TOKEN="" + curl -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $DO_API_TOKEN" "https://api.digitalocean.com/v2/account/keys" + +How to run +========== + +Initialization +-------------- + +If this is your first time using terraform, you have to initialize it by +running the below command. (Note: initialization can be run multiple +times) + +:: + + terraform init + +After initialization it's good measure to create a new Terraform +environment for the droplets so they are always managed together. + +:: + + TESTNET_NAME="testnet-servers" + terraform env new "$TESTNET_NAME" + +Note this ``terraform env`` command is only available in terraform +``v0.9`` and up. + +Execution +--------- + +The below command will create 4 nodes in DigitalOcean. They will be +named ``testnet-servers-node0`` to ``testnet-servers-node3`` and they +will be tagged as ``testnet-servers``. + +:: + + DO_API_TOKEN="" + SSH_IDS="[ \"\" ]" + terraform apply -var TESTNET_NAME="testnet-servers" -var servers=4 -var DO_API_TOKEN="$DO_API_TOKEN" -var ssh_keys="$SSH_IDS" + +Note: ``ssh_keys`` is a list of strings. You can add multiple keys. For +example: ``["1234567","9876543"]``. + +Alternatively you can use the default settings. The number of default +servers is 4 and the testnet name is ``tf-testnet1``. Variables can also +be defined as environment variables instead of the command-line. +Environment variables that start with ``TF_VAR_`` will be translated +into the Terraform configuration. For example the number of servers can +be overriden by setting the ``TF_VAR_servers`` variable. + +:: + + TF_VAR_DO_API_TOKEN="" + TF_VAR_TESTNET_NAME="testnet-servers" + terraform-apply + +Security +-------- + +DigitalOcean uses the root user by default on its droplets. This is fine +as long as SSH keys are used. However some people still would like to +disable root and use an alternative user to connect to the droplets - +then ``sudo`` from there. Terraform can do this but it requires SSH +agent running on the machine where terraform is run, with one of the SSH +keys of the droplets added to the agent. (This will be neede for ansible +too, so it's worth setting it up here. Check out the +`ansible `__ +page for more information.) After setting up the SSH key, run +``terraform apply`` with ``-var noroot=true`` to create your droplets. +Terraform will create a user called ``ec2-user`` and move the SSH keys +over, this way disabling SSH login for root. It also adds the +``ec2-user`` to the sudoers file, so after logging in as ec2-user you +can ``sudo`` to ``root``. + +DigitalOcean announced firewalls but the current version of Terraform +(0.9.8 as of this writing) does not support it yet. Fortunately it is +quite easy to set it up through the web interface (and not that bad +through the `RESTful +API `__ +either). When adding droplets to a firewall rule, you can add tags. All +droplets in a testnet are tagged with the testnet name so it's enough to +define the testnet name in the firewall rule. It is not necessary to add +the nodes one-by-one. Also, the firewall rule "remembers" the testnet +name tag so if you change the servers but keep the name, the firewall +rules will still apply. + +What's next +=========== + +After setting up the nodes, head over to the `ansible +folder `__ to +set up tendermint and basecoin. diff --git a/docs/using-ansible.md b/docs/using-ansible.md deleted file mode 100644 index 03a32f55d..000000000 --- a/docs/using-ansible.md +++ /dev/null @@ -1,202 +0,0 @@ -# Ansible playbook for Tendermint applications - -![Ansible plus Tendermint](img/a_plus_t.png) - -* [Prerequisites](#Prerequisites) -* [Ansible setup](#Ansible setup) -* [Running the playbook](#Running the playbook) - -The playbooks in this folder run [ansible](http://www.ansible.com/) roles which: - -* install and configure basecoin or ethermint -* start/stop basecoin or ethermint and reset their configuration - -## Prerequisites - -* Ansible 2.0 or higher -* SSH key to the servers - -Optional for DigitalOcean droplets: -* DigitalOcean API Token -* python dopy package - -Head over to the [Terraform folder](https://github.com/tendermint/tools/tree/master/terraform-digitalocean) for a description on how to get a DigitalOcean API Token. - -Optional for Amazon AWS instances: -* Amazon AWS API access key ID and secret access key. - -The cloud inventory scripts come from the ansible team at their [GitHub](https://github.com/ansible/ansible) page. You can get the latest version from the contrib/inventory folder. - -## Ansible setup - -Ansible requires a "command machine" or "local machine" or "orchestrator machine" to run on. This can be your laptop or any machine that can run ansible. (It does not have to be part of the cloud network that hosts your servers.) - -Use the official [Ansible installation guide](http://docs.ansible.com/ansible/intro_installation.html) to install Ansible. Here are a few examples on basic installation commands: - -Ubuntu/Debian: -``` -sudo apt-get install ansible -``` - -CentOS/RedHat: -``` -sudo yum install epel-release -sudo yum install ansible -``` - -Mac OSX: -If you have (Homebrew)[https://brew.sh] installed, then it's simply -``` -brew install ansible -``` - -If not, you can install it using `pip`: -``` -sudo easy_install pip -sudo pip install ansible -``` - - -To make life easier, you can start an SSH Agent and load your SSH key(s). This way ansible will have an uninterrupted way of connecting to your servers. - -``` -ssh-agent > ~/.ssh/ssh.env -source ~/.ssh/ssh.env - -ssh-add private.key -``` - -Subsequently, as long as the agent is running, you can use `source ~/.ssh/ssh.env` to load the keys to the current session. -Note: On Mac OSX, you can add the `-K` option to ssh-add to store the passphrase in your keychain. The security of this feature is debated but it is convenient. - -### Optional cloud dependencies - -If you are using a cloud provider to host your servers, you need the below dependencies installed on your local machine. - -#### DigitalOcean inventory dependencies: - -Ubuntu/Debian: -``` -sudo apt-get install python-pip -sudo pip install dopy -``` - -CentOS/RedHat: -``` -sudo yum install python-pip -sudo pip install dopy -``` - -Mac OSX: -``` -sudo pip install dopy -``` - -#### Amazon AWS inventory dependencies: - -Ubuntu/Debian: -``` -sudo apt-get install python-boto -``` - -CentOS/RedHat: -``` -sudo yum install python-boto -``` - -Mac OSX: -``` -sudo pip install boto -``` - -## Refreshing the DigitalOcean inventory - -If you just finished creating droplets, the local DigitalOcean inventory cache is not up-to-date. To refresh it, run: - -``` -DO_API_TOKEN="" -python -u inventory/digital_ocean.py --refresh-cache 1> /dev/null -``` - -## Refreshing the Amazon AWS inventory - -If you just finished creating Amazon AWS EC2 instances, the local AWS inventory cache is not up-to-date. To refresh it, run: - -``` -AWS_ACCESS_KEY_ID='' -AWS_SECRET_ACCESS_KEY='' -python -u inventory/ec2.py --refresh-cache 1> /dev/null -``` - -Note: you don't need the access key and secret key set, if you are running ansible on an Amazon AMI instance with the proper IAM permissions set. - -## Running the playbooks - -The playbooks are locked down to only run if the environment variable `TF_VAR_TESTNET_NAME` is populated. This is a precaution so you don't accidentally run the playbook on all your servers. - -The variable `TF_VAR_TESTNET_NAME` contains the testnet name which ansible translates into an ansible group. If you used Terraform to create the servers, it was the testnet name used there. - -If the playbook cannot connect to the servers because of public key denial, your SSH Agent is not set up properly. Alternatively you can add the SSH key to ansible using the `--private-key` option. - -If you need to connect to the nodes as root but your local username is different, use the ansible option `-u root` to tell ansible to connect to the servers and authenticate as the root user. - -If you secured your server and you need to `sudo` for root access, use the the `-b` or `--become` option to tell ansible to sudo to root after connecting to the server. In the Terraform-DigitalOcean example, if you created the ec2-user by adding the `noroot=true` option (or if you are simply on Amazon AWS), you need to add the options `-u ec2-user -b` to ansible to tell it to connect as the ec2-user and then sudo to root to run the playbook. - -### DigitalOcean -``` -DO_API_TOKEN="" -TF_VAR_TESTNET_NAME="testnet-servers" -ansible-playbook -i inventory/digital_ocean.py install.yml -e service=basecoin -``` - -### Amazon AWS -``` -AWS_ACCESS_KEY_ID='' -AWS_SECRET_ACCESS_KEY='' -TF_VAR_TESTNET_NAME="testnet-servers" -ansible-playbook -i inventory/ec2.py install.yml -e service=basecoin -``` - -### Installing custom versions - -By default ansible installs the tendermint, basecoin or ethermint binary versions from the latest release in the repository. If you build your own version of the binaries, you can tell ansible to install that instead. - -``` -GOPATH="" -go get -u github.com/tendermint/basecoin/cmd/basecoin - -DO_API_TOKEN="" -TF_VAR_TESTNET_NAME="testnet-servers" -ansible-playbook -i inventory/digital_ocean.py install.yml -e service=basecoin -e release_install=false -``` - -Alternatively you can change the variable settings in `group_vars/all`. - -## Other commands and roles - -There are few extra playbooks to make life easier managing your servers. - -* install.yml - Install basecoin or ethermint applications. (Tendermint gets installed automatically.) Use the `service` parameter to define which application to install. Defaults to `basecoin`. -* reset.yml - Stop the application, reset the configuration and data, then start the application again. You need to pass `-e service=`, like `-e service=basecoin`. It will restart the underlying tendermint application too. -* restart.yml - Restart a service on all nodes. You need to pass `-e service=`, like `-e service=basecoin`. It will restart the underlying tendermint application too. -* stop.yml - Stop the application. You need to pass `-e service=`. -* status.yml - Check the service status and print it. You need to pass `-e service=`. -* start.yml - Start the application. You need to pass `-e service=`. -* ubuntu16-patch.yml - Ubuntu 16.04 does not have the minimum required python package installed to be able to run ansible. If you are using ubuntu, run this playbook first on the target machines. This will install the python pacakge that is required for ansible to work correctly on the remote nodes. -* upgrade.yml - Upgrade the `service` on your testnet. It will stop the service and restart it at the end. It will only work if the upgraded version is backward compatible with the installed version. -* upgrade-reset.yml - Upgrade the `service` on your testnet and reset the database. It will stop the service and restart it at the end. It will work for upgrades where the new version is not backward-compatible with the installed version - however it will reset the testnet to its default. - -The roles are self-sufficient under the `roles/` folder. - -* install - install the application defined in the `service` parameter. It can install release packages and update them with custom-compiled binaries. -* unsafe_reset - delete the database for a service, including the tendermint database. -* config - configure the application defined in `service`. It also configures the underlying tendermint service. Check `group_vars/all` for options. -* stop - stop an application. Requires the `service` parameter set. -* status - check the status of an application. Requires the `service` parameter set. -* start - start an application. Requires the `service` parameter set. - -## Default variables - -Default variables are documented under `group_vars/all`. You can the parameters there to deploy a previously created genesis.json file (instead of dynamically creating it) or if you want to deploy custom built binaries instead of deploying a released version. - - diff --git a/docs/using-ansible.rst b/docs/using-ansible.rst new file mode 100644 index 000000000..6cbe96ac8 --- /dev/null +++ b/docs/using-ansible.rst @@ -0,0 +1,294 @@ +Ansible playbook for Tendermint applications +============================================ + +.. figure:: img/a_plus_t.png + :alt: Ansible plus Tendermint + + Ansible plus Tendermint + +- `Prerequisites <#Prerequisites>`__ +- `Ansible setup <#Ansible%20setup>`__ +- `Running the playbook <#Running%20the%20playbook>`__ + +The playbooks in this folder run `ansible `__ +roles which: + +- install and configure basecoin or ethermint +- start/stop basecoin or ethermint and reset their configuration + +Prerequisites +------------- + +- Ansible 2.0 or higher +- SSH key to the servers + +Optional for DigitalOcean droplets: \* DigitalOcean API Token \* python +dopy package + +Head over to the `Terraform +folder `__ +for a description on how to get a DigitalOcean API Token. + +Optional for Amazon AWS instances: \* Amazon AWS API access key ID and +secret access key. + +The cloud inventory scripts come from the ansible team at their +`GitHub `__ page. You can get the +latest version from the contrib/inventory folder. + +Ansible setup +------------- + +Ansible requires a "command machine" or "local machine" or "orchestrator +machine" to run on. This can be your laptop or any machine that can run +ansible. (It does not have to be part of the cloud network that hosts +your servers.) + +Use the official `Ansible installation +guide `__ to +install Ansible. Here are a few examples on basic installation commands: + +Ubuntu/Debian: + +:: + + sudo apt-get install ansible + +CentOS/RedHat: + +:: + + sudo yum install epel-release + sudo yum install ansible + +Mac OSX: If you have (Homebrew)[https://brew.sh] installed, then it's +simply + +:: + + brew install ansible + +If not, you can install it using ``pip``: + +:: + + sudo easy_install pip + sudo pip install ansible + +To make life easier, you can start an SSH Agent and load your SSH +key(s). This way ansible will have an uninterrupted way of connecting to +your servers. + +:: + + ssh-agent > ~/.ssh/ssh.env + source ~/.ssh/ssh.env + + ssh-add private.key + +Subsequently, as long as the agent is running, you can use +``source ~/.ssh/ssh.env`` to load the keys to the current session. Note: +On Mac OSX, you can add the ``-K`` option to ssh-add to store the +passphrase in your keychain. The security of this feature is debated but +it is convenient. + +Optional cloud dependencies +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you are using a cloud provider to host your servers, you need the +below dependencies installed on your local machine. + +DigitalOcean inventory dependencies: +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Ubuntu/Debian: + +:: + + sudo apt-get install python-pip + sudo pip install dopy + +CentOS/RedHat: + +:: + + sudo yum install python-pip + sudo pip install dopy + +Mac OSX: + +:: + + sudo pip install dopy + +Amazon AWS inventory dependencies: +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Ubuntu/Debian: + +:: + + sudo apt-get install python-boto + +CentOS/RedHat: + +:: + + sudo yum install python-boto + +Mac OSX: + +:: + + sudo pip install boto + +Refreshing the DigitalOcean inventory +------------------------------------- + +If you just finished creating droplets, the local DigitalOcean inventory +cache is not up-to-date. To refresh it, run: + +:: + + DO_API_TOKEN="" + python -u inventory/digital_ocean.py --refresh-cache 1> /dev/null + +Refreshing the Amazon AWS inventory +----------------------------------- + +If you just finished creating Amazon AWS EC2 instances, the local AWS +inventory cache is not up-to-date. To refresh it, run: + +:: + + AWS_ACCESS_KEY_ID='' + AWS_SECRET_ACCESS_KEY='' + python -u inventory/ec2.py --refresh-cache 1> /dev/null + +Note: you don't need the access key and secret key set, if you are +running ansible on an Amazon AMI instance with the proper IAM +permissions set. + +Running the playbooks +--------------------- + +The playbooks are locked down to only run if the environment variable +``TF_VAR_TESTNET_NAME`` is populated. This is a precaution so you don't +accidentally run the playbook on all your servers. + +The variable ``TF_VAR_TESTNET_NAME`` contains the testnet name which +ansible translates into an ansible group. If you used Terraform to +create the servers, it was the testnet name used there. + +If the playbook cannot connect to the servers because of public key +denial, your SSH Agent is not set up properly. Alternatively you can add +the SSH key to ansible using the ``--private-key`` option. + +If you need to connect to the nodes as root but your local username is +different, use the ansible option ``-u root`` to tell ansible to connect +to the servers and authenticate as the root user. + +If you secured your server and you need to ``sudo`` for root access, use +the the ``-b`` or ``--become`` option to tell ansible to sudo to root +after connecting to the server. In the Terraform-DigitalOcean example, +if you created the ec2-user by adding the ``noroot=true`` option (or if +you are simply on Amazon AWS), you need to add the options +``-u ec2-user -b`` to ansible to tell it to connect as the ec2-user and +then sudo to root to run the playbook. + +DigitalOcean +~~~~~~~~~~~~ + +:: + + DO_API_TOKEN="" + TF_VAR_TESTNET_NAME="testnet-servers" + ansible-playbook -i inventory/digital_ocean.py install.yml -e service=basecoin + +Amazon AWS +~~~~~~~~~~ + +:: + + AWS_ACCESS_KEY_ID='' + AWS_SECRET_ACCESS_KEY='' + TF_VAR_TESTNET_NAME="testnet-servers" + ansible-playbook -i inventory/ec2.py install.yml -e service=basecoin + +Installing custom versions +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +By default ansible installs the tendermint, basecoin or ethermint binary +versions from the latest release in the repository. If you build your +own version of the binaries, you can tell ansible to install that +instead. + +:: + + GOPATH="" + go get -u github.com/tendermint/basecoin/cmd/basecoin + + DO_API_TOKEN="" + TF_VAR_TESTNET_NAME="testnet-servers" + ansible-playbook -i inventory/digital_ocean.py install.yml -e service=basecoin -e release_install=false + +Alternatively you can change the variable settings in +``group_vars/all``. + +Other commands and roles +------------------------ + +There are few extra playbooks to make life easier managing your servers. + +- install.yml - Install basecoin or ethermint applications. (Tendermint + gets installed automatically.) Use the ``service`` parameter to + define which application to install. Defaults to ``basecoin``. +- reset.yml - Stop the application, reset the configuration and data, + then start the application again. You need to pass + ``-e service=``, like ``-e service=basecoin``. It will + restart the underlying tendermint application too. +- restart.yml - Restart a service on all nodes. You need to pass + ``-e service=``, like ``-e service=basecoin``. It will + restart the underlying tendermint application too. +- stop.yml - Stop the application. You need to pass + ``-e service=``. +- status.yml - Check the service status and print it. You need to pass + ``-e service=``. +- start.yml - Start the application. You need to pass + ``-e service=``. +- ubuntu16-patch.yml - Ubuntu 16.04 does not have the minimum required + python package installed to be able to run ansible. If you are using + ubuntu, run this playbook first on the target machines. This will + install the python pacakge that is required for ansible to work + correctly on the remote nodes. +- upgrade.yml - Upgrade the ``service`` on your testnet. It will stop + the service and restart it at the end. It will only work if the + upgraded version is backward compatible with the installed version. +- upgrade-reset.yml - Upgrade the ``service`` on your testnet and reset + the database. It will stop the service and restart it at the end. It + will work for upgrades where the new version is not + backward-compatible with the installed version - however it will + reset the testnet to its default. + +The roles are self-sufficient under the ``roles/`` folder. + +- install - install the application defined in the ``service`` + parameter. It can install release packages and update them with + custom-compiled binaries. +- unsafe\_reset - delete the database for a service, including the + tendermint database. +- config - configure the application defined in ``service``. It also + configures the underlying tendermint service. Check + ``group_vars/all`` for options. +- stop - stop an application. Requires the ``service`` parameter set. +- status - check the status of an application. Requires the ``service`` + parameter set. +- start - start an application. Requires the ``service`` parameter set. + +Default variables +----------------- + +Default variables are documented under ``group_vars/all``. You can the +parameters there to deploy a previously created genesis.json file +(instead of dynamically creating it) or if you want to deploy custom +built binaries instead of deploying a released version. diff --git a/docs/using-docker.md b/docs/using-docker.md deleted file mode 100644 index fc8679739..000000000 --- a/docs/using-docker.md +++ /dev/null @@ -1,95 +0,0 @@ -# Docker container description for Tendermint applications - -* Overview (#Overview) -* Tendermint (#Tendermint) -* Basecoin (#Basecoin) -* Ethermint (#Ethermint) - -## Overview - -This folder contains Docker container descriptions. Using this folder you can build your own Docker images with the tendermint application. - -It is assumed that you set up docker already. - -If you don't want to build the images yourself, you should be able to download them from Docker Hub. - -## Tendermint - -Build the container: -Copy the `tendermint` binary to the `tendermint` folder. -``` -docker build -t tendermint tendermint -``` - -The application configuration will be stored at `/tendermint` in the container. The ports 46656 and 46657 will be open for ABCI applications to connect. - -Initialize tendermint configuration and keep it after the container is finished in a docker volume called `data`: -``` -docker run --rm -v data:/tendermint tendermint init -``` -If you want the docker volume to be a physical directory on your filesystem, you have to give an absolute path to docker and make sure the permissions allow the application to write it. - -Get the public key of tendermint: -``` -docker run --rm -v data:/tendermint tendermint show_validator -``` - -Run the docker tendermint application with: -``` -docker run --rm -d -v data:/tendermint tendermint node -``` - -## Basecoin - -Build the container: -Copy the `basecoin` binary to the `basecoin` folder. -``` -docker build -t basecoin basecoin -``` -The application configuration will be stored at `/basecoin`. - -Initialize basecoin configuration and keep it after the container is finished: -``` -docker run --rm -v basecoindata:/basecoin basecoin init deadbeef -``` -Use your own basecoin account instead of `deadbeef` in the `init` command. - -Get the public key of basecoin: -We use a trick here: since the basecoin and the tendermint configuration folders are similar, the `tendermint` command can extract the public key for us if we feed the basecoin configuration folder to tendermint. -``` -docker run --rm -v basecoindata:/tendermint tendermint show_validator -``` - -Run the docker tendermint application with: -This is a two-step process: -* Run the basecoin container. -* Run the tendermint container and expose the ports that allow clients to connect. The --proxy_app should contain the basecoin application's IP address and port. -``` -docker run --rm -d -v basecoindata:/basecoin basecoin start --without-tendermint -docker run --rm -d -v data:/tendermint -p 46656-46657:46656-46657 tendermint node --proxy_app tcp://172.17.0.2:46658 -``` - -## Ethermint - -Build the container: -Copy the `ethermint` binary and the setup folder to the `ethermint` folder. -``` -docker build -t ethermint ethermint -``` -The application configuration will be stored at `/ethermint`. -The files required for initializing ethermint (the files in the source `setup` folder) are under `/setup`. - -Initialize ethermint configuration: -``` -docker run --rm -v ethermintdata:/ethermint ethermint init /setup/genesis.json -``` - -Start ethermint as a validator node: -This is a two-step process: -* Run the ethermint container. You will have to define where tendermint runs as the ethermint binary connects to it explicitly. -* Run the tendermint container and expose the ports that allow clients to connect. The --proxy_app should contain the ethermint application's IP address and port. -``` -docker run --rm -d -v ethermintdata:/ethermint ethermint --tendermint_addr tcp://172.17.0.3:46657 -docker run --rm -d -v data:/tendermint -p 46656-46657:46656-46657 tendermint node --proxy_app tcp://172.17.0.2:46658 -``` - diff --git a/docs/using-docker.rst b/docs/using-docker.rst new file mode 100644 index 000000000..85a302611 --- /dev/null +++ b/docs/using-docker.rst @@ -0,0 +1,128 @@ +Docker container description for Tendermint applications +======================================================== + +- Overview (#Overview) +- Tendermint (#Tendermint) +- Basecoin (#Basecoin) +- Ethermint (#Ethermint) + +Overview +-------- + +This folder contains Docker container descriptions. Using this folder +you can build your own Docker images with the tendermint application. + +It is assumed that you set up docker already. + +If you don't want to build the images yourself, you should be able to +download them from Docker Hub. + +Tendermint +---------- + +Build the container: Copy the ``tendermint`` binary to the +``tendermint`` folder. + +:: + + docker build -t tendermint tendermint + +The application configuration will be stored at ``/tendermint`` in the +container. The ports 46656 and 46657 will be open for ABCI applications +to connect. + +Initialize tendermint configuration and keep it after the container is +finished in a docker volume called ``data``: + +:: + + docker run --rm -v data:/tendermint tendermint init + +If you want the docker volume to be a physical directory on your +filesystem, you have to give an absolute path to docker and make sure +the permissions allow the application to write it. + +Get the public key of tendermint: + +:: + + docker run --rm -v data:/tendermint tendermint show_validator + +Run the docker tendermint application with: + +:: + + docker run --rm -d -v data:/tendermint tendermint node + +Basecoin +-------- + +Build the container: Copy the ``basecoin`` binary to the ``basecoin`` +folder. + +:: + + docker build -t basecoin basecoin + +The application configuration will be stored at ``/basecoin``. + +Initialize basecoin configuration and keep it after the container is +finished: + +:: + + docker run --rm -v basecoindata:/basecoin basecoin init deadbeef + +Use your own basecoin account instead of ``deadbeef`` in the ``init`` +command. + +Get the public key of basecoin: We use a trick here: since the basecoin +and the tendermint configuration folders are similar, the ``tendermint`` +command can extract the public key for us if we feed the basecoin +configuration folder to tendermint. + +:: + + docker run --rm -v basecoindata:/tendermint tendermint show_validator + +Run the docker tendermint application with: This is a two-step process: +\* Run the basecoin container. \* Run the tendermint container and +expose the ports that allow clients to connect. The --proxy\_app should +contain the basecoin application's IP address and port. + +:: + + docker run --rm -d -v basecoindata:/basecoin basecoin start --without-tendermint + docker run --rm -d -v data:/tendermint -p 46656-46657:46656-46657 tendermint node --proxy_app tcp://172.17.0.2:46658 + +Ethermint +--------- + +Build the container: Copy the ``ethermint`` binary and the setup folder +to the ``ethermint`` folder. + +:: + + docker build -t ethermint ethermint + +The application configuration will be stored at ``/ethermint``. The +files required for initializing ethermint (the files in the source +``setup`` folder) are under ``/setup``. + +Initialize ethermint configuration: + +:: + + docker run --rm -v ethermintdata:/ethermint ethermint init /setup/genesis.json + +Start ethermint as a validator node: This is a two-step process: \* Run +the ethermint container. You will have to define where tendermint runs +as the ethermint binary connects to it explicitly. \* Run the tendermint +container and expose the ports that allow clients to connect. The +--proxy\_app should contain the ethermint application's IP address and +port. + +:: + + docker run --rm -d -v ethermintdata:/ethermint ethermint --tendermint_addr tcp://172.17.0.3:46657 + docker run --rm -d -v data:/tendermint -p 46656-46657:46656-46657 tendermint node --proxy_app tcp://172.17.0.2:46658 From a917ec9ea2764424cd8a04ad44ee366606f628ac Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Wed, 13 Sep 2017 09:36:28 -0400 Subject: [PATCH 21/92] docs: update index.rst --- docs/index.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 56b1163c7..afeec1d94 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -32,6 +32,11 @@ Tendermint Tools :maxdepth: 2 deploy-testnets.rst + using-ansible.rst + using-docker.rst + mintnet-kubernetes.rst + terraform-digitalocean.rst + Tendermint 102 -------------- From 044c500cc0ed59d675a7474117c844cbcb5f0f0a Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Wed, 13 Sep 2017 09:54:07 -0400 Subject: [PATCH 22/92] docs: move images in from tools repo --- docs/assets/a_plus_t.png | Bin 0 -> 13830 bytes docs/assets/statefulset.png | Bin 0 -> 17367 bytes docs/assets/t_plus_k.png | Bin 0 -> 18476 bytes docs/mintnet-kubernetes.rst | 4 ++-- docs/using-ansible.rst | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 docs/assets/a_plus_t.png create mode 100644 docs/assets/statefulset.png create mode 100644 docs/assets/t_plus_k.png diff --git a/docs/assets/a_plus_t.png b/docs/assets/a_plus_t.png new file mode 100644 index 0000000000000000000000000000000000000000..8f5bc5e950cb4ee31a0983f004f780aeb398c6ea GIT binary patch literal 13830 zcmaL8b95%bw>BEv<{M3Bl8J5Gwr#($J<-J08(R}w6HRQ}&cx2mobUY3cmKHO_Uf*# zs$Or@oU|?X#Qj(&|pZDj_%M=dgv#bfAL;1Yn1I09fDh}pA zcOz#rFd41IJ0lA-WiumFPp1(xUNA7oFIK9WKutMWZW9MPdZT|b^d5GOpJ*^J zUI7nBBNH1lAgQsLg_S)Y`DJ@IIjNN?AGrpb9HX40h?%98q?faqikG~qiI)GjBW0s! zp)+A*Vj^W{r)OeiV`pcgC1qx0VrF3cJlW}(Sh?BQxtTdg|N9{S9L?F(oLgB`{J+Qg zZ1Is>0)dX)3=Hn>?)2^~^bXDz3`|^HT>m(jndv?y=v+MQfkqy5_AV6vVGuQQF>$tX z1X?-Rlm25gGInqU@{xaP`hQHYbNnA#dzb&7rq2mu@Gx>@V4`RIXVQNJ<>dbVMeXeV zN7@CbZ1#Wk{r@C(QT22*V^B78ad34u`E;B)#lN8(xka4KjDQZ#styje|CvQaO9!BX zi=~4jsfY?YDV3a&iIx4o4(k6x$jNa_*}DLZ>`lz1MES@+Iq0pdOu3na*tpmkIanCQ z*o2sv#5jZ)IoQQmImKB;ghUxRSw;SX6?HIiwKKB^{s(LNf3d9pEB0S7*g1ZVENbR# zzKTmA34F#WH5|BW^M-*sXBU$G3I$}s$M?f=Ww|LOXSpMTZ=3Ea=d z|AfAo{b#&8e+IU_EQK@}7_*0zsF141%9$^0fUes7N9#k^gSlH%aXdLZ8CshtDR?{% zbiO1yw2+`El<4D>0u(luz$-0WegJeRsx2IXdgRBK0ANw5K1JRasF~HKbkF^b7lFer zj?8op7M}WL_V)C8uJrqZ#@6oHi`GNNDmGQ}LA*ID%d(8kB6-*H6puHYxiWcI(BfxN z4%zux%u15lVCjlLc#>Zi9@Rm&qA&mCe~537=ZVu0>wsCV`gr{(=r-xDu=f^cL3a6* z7t&K0P)idZ-huLjyq6kpAn3>Im2^j@R5C4T3njWfL`o9#ioS;`rtQu6gZ1rg7%~1M zu)~Ryp$*_#GrItB8%v&x&Bra`W6OMcNU<5v0YbUuo)6@&os0HCx%Q_}D2w;LrN3qy zUL%-00PE!H%;*1;XecUbUh^45c-3Ed_?i1FN-% zVnR)Y1IlO8F(612CxqJyAVchg46V=re(8u)j~3MC07?~3(9<^3(ogL&gB?WB_PR^^ z-%yMe%AA7Xou-z}lqRPhu3&3Fj3`K3M~;AICdB2YFPhOJe=N|bpyd}JW4?Xo%7zHG zRTfDAE)mMb{pHNI*7CYqeS7h>raZ|Jen+{s-lMUG+Y`MM>SAU-|M}o1U5OJQ&ykhCwJB_0y6r7?a^g>61au>2pqL5ZLk)k;eB9&Qyqs58@sh19KGcvlcA zuU^b6()9$ze54~yBH~Bf_a${&{4$gg{P2kT zw{?%IM2QZRBQE3nd_r!;_%J>AeP?KxMo5US1m;pW!VxaTCQ1QMbIwsqK_W7#G$OqeWPe__UXIM-fUB!ku&n15EML1@8Qpm5^s(pwo@C*cN zvS8^K6Jo+w&MagMF31vF6RDY5_$E93*3?Vey7d*)5RcAlsul;lOY3@igs3RCAMw@= zJlV}9khLu;Diy^rm`5D;MWmC{_``e1eqAUV{8Y}byGJGtmX{DR8U?tGBii+(f}|%} zlrgGRhb$AZw9PD7V22csxHd{eu8WVDe{8R6Py4I2uLtrYZ#gG0486Zd%HnL4!aTE0 z3vRANIN$NaGn8&do=C!jOI-+W!N$w!qaT7=9)p{f0xO|PuXZ!jlsE~#Tkf4sCmDo# zx9F*~(S!qRf?N;@AU(#gy~ti}1Bj3NVfgqGT7jLq%sDXh8g% z`uO$IdF-MbJj-BiM@Q<)A81rhuGITd1Jb&I%Fh}tQ7qK4IckjU@X^d=7gvd|6n(RZ zp`)amNkbZT+`WJ^`+AGa!UqJ)k~eBzdu9ru6$k|&5d1BT3Qw&>_!&JU9e8uvoKM7z zXG{i$U0Tk{_0&i7NOZf$5L+N`1@Fe}X~Ue%ts;fIT#W+7hk%Y|HV)A)(|6J&uF1yd z`A}%KY*K7{)HW*;lZDA1I6xXd{USR7C4kmLFMe+DM3l@f)Ox|rf7bgF2*r&Z@F-Fh zow@G5NzDH@LZRoE+2~f&sy*Z)@%_|r6&lXCrnZEu?#s*xb4Vp!d8B$Gk)oxJOwr|z z7!KFD4DTP`!!OBsz6Brjm~cSUFZM)>X9pg#0R#@czwHCy8QLt%Puus>e*0S)=#O5r zHZUD0w^6MczR6Mk&M)7?MWCPW>9tQSfH~?8BN)*Vcvy4UUa_e;_w=3MI_&9%B|+-C zL_M^c##l2TBhe0b%fpO$XOl!_Yk>cykvBd9#BmJ9ur(%Xc$D&%UJ1WOf*{nz%&4Ds z*Z7H?y~YyQpdEpBm7g|IRqGdq=A^_QOXs;;PzNdVP^+h7A*!@j z+{n>uV&uWWvITzq{UWKGF_(T2bx_u}L?lRZRwIID(BJJ5A*{lQLSp&(pC7yKpLk)u ze^tawhsXW%C9^mIZcEfGg`C%fT^3pl2I5GdJUK8?_Ura+?mKW~vT4)LoAbCZ(RyFC zrm~+XdQz5)DFO*{)nuy$n^Okc@)rL`$~hvj3&L2h%-HUx3O?&>edU)^R%@r!;NY&7 z6e5UGh~qDU+DHAV#)JZx_sm7Vp)0A}0@#C$yAL91k@TTi5CGygCnBi#?->^s^N}O+ zw<8LE7@u+3PG#7S(7@gKer6$n$g%0;LYMVD{VF$Vhd`&wfqrUk;)~wyH;t_76Q$xa z>pZP^_GZc#PK)J@V9$7M7atV-%)rZG#$omCCFOp#M@RNQMcD{H|19NMCSHz@C7fFK z$e1WY9K}NepY7%7wm8olX7j7;iJ&32MD7drnrOvx}IoR|d>Abf}7b`x)20z$8q z9578#erMAz3$DoEpG8Ns5e3gqFm|g<4y?cIdLbZvSes3o8w-4&Q&FhG2B~aE-o*Jg zvE^q)5M6nw88FJoU=MJxON{qZGhDT2ET;yh!g|V&mb6X%e(1Aeel9>#gJ{D9!IcqF zdL!t?cwaf`Z@28gGWEKfXm={dXc=QzQ8l{%SnRQEaXS$4>M8nZxX@VsDp(7#{< zf8*JFLaEg6Y{QYyR5mFq1iIPG9>7DT)}}?|Yp{Eit^edE17mA4w2n@| z+ta&t!3mX3>}7(TO2`$ek^@=n*El)<{>GHCzFNCkJ3Tls=iEk*SW<`CcWq?2Kb&SzxCfEOBN>C5dFvH5UmAbf&#Ktdo*weGX2?YLqZO0h}ss-^{_;?tlCf_QTJxwRiyK8zIGuf^6I zu?_v)i+0anAyv<}$x#yBvg**?Ic$;3W`jfII;Yg={QN>eQahh^&K=F`!deh2ynvnN zJlFVc8m~o!lkwOp9#drofF)BY&B7VEm4XYD3KK#k2cwB1z$Nhnu$b}=eu(te6?1A2 zDJZWAa|o)MnrWed9GR(G+uQ>EjOmcG`cVal4#v_|3deIjcHlLCnZncRw$vWh`_!sH z$RSTh=)jNlCM8Mk2s#MM-%TU{2%#;(hX&TQUa4_y268mS9ru`!!E!2)zG>rhkLM1u2nkKy@XD}ztAiykVS_&sg@(_}(+v>AfWF1YmKWtE z){NiuzK08hoZ7rUP5SrI)O09+fwSx#rEfn|rb!ign8QD3OvMTjzGfKs`MT zsuEO}_>Vt^{+11pdT}D-lt3m_<~HV%Mw!?f;r$VWR~P7Oh6HG|U##)fjQMfl4HL)Qe3uX2tA#U!3bb{It>pJVnV*%n)49Cu9z=AUZIjGb@#NTBDLGD;kD0GU`H#adz8R&+}Bv3i9 z1o+=;5qxUf_-b`S`FSe1KqRU%f|F)_1K>moF21K#%?PYMU1DgtDTLzO9!o@r!hphH z%X*MVzW8k-+?c3qGzGA`FO;+eHNd+lqR<73l>aNtA6*8Qi=&!VOLGq3?5n`@F@_!H z1j(G|ky2Iz_u1OsbwpK8zyZPLv`52LR%7=@p&gc*OnJ83h}k0p%%u@9BPUZ;Z_EyDHtI_CncXSLSA zfUvHo7WN~eV@{9J85icfh@LFTt4)Iwe>h~`0Zz7I{7Fk(TY3fX8^nOT2?d^Gq2%nE za)kYGxQL|z$Pp7gw;T_|nqiA|%93f7hzKf2>y9;u<#jQo#aW?U0=r&Qi>tf-dnD47+zJhWJt2b+?hj8~KcQLL5{N#<4QMGzLry`MJl+DYlD-1o1ag+5A> zmQF(DkqCx95cH$mkvB&Z+)=k zIdC!*GCSb!V<3$(y^7bTo07V@R78XV{vd2+^#kA=OY(&!S8R6gS@ z5A0$`j;Uaos%1CLzUx^8NFEE3`o;xn^k|p2K2p$0hzFdak^ypl8=QL&(D&pgs-fC>pbawwCR2H{jmvZQ+bOkaO92TjJnA>ke_nL z=v#*J=wcUFExmRF9Kd5MFmjUt*P|l-)xB+~?9(?4BU{J)gxez&1lz~`!WTqQ#Ww2c zg-HQYkYEayj`8&n$p91FfVo+;J%ydi*ZNuoOdm{lj^geT%)cp!^wpft?+&fUH5?W%#nw9YWR4>Q%A)ZeaEo0pNTn*Be2OPWhqq zle{-bXPxfbPT(k`ANhHFrB;_!1)|GX_uR8&3}h9^4s3H1>|Zq7|ubJd?`uN z5Ra0S9OsMezIsBt=em^Pjc6-WW;~|^scIE(3a?cq;26XP73f%=*VUE<2Xoqq;BSeL5nj*B2!<* zwLAf_@VyF>z^bt)lE`LoIOLR}VREc)`abBj{2e5$*gzexKMcs}Cdo)8F$bxb z4C3s$8_|$z3ztXBr5jw+Wc}I!{rv6jALO9A&%C3d-*b+;({&+H%U?Z~q-8|xuI(^! zO~7#LlTW_NE}K+hqM}nIzUG5+u@n4w&-=RYax>;4X{LsN?V75wmf0)B(t$qijp~ef z&iw~RIX;OL*+FdUf(hNaA4sBs2pRmn{r%m{?4`7yet; zl!}WzURjj^ziGN{bC+q)-NVZpw||CR!n2ppbL{sW;R%K?~F!iB#r^heO@7NWehz)f-2&^H#?=F0I}Uy<@ej zIMtoe+Wd>Pc&U1;-i>1})x1r~PV8TBtFv_`ldYmu{qKY^v6x~9F6qusT2e_UI*?sn zvx2|7o_lnj*TH`-;76YaA^6t8-5z=))@~Qqke7D%Q1?~Iop+XSyi9;KRpjMS(4L{4 z00GtFiAvk$rT7KWkt3Ey1Vy2n78uf1SVN0Z(9jV0tLVA8+syM9+>C5Fr@~J>Gseq3 zM!AAyLM~S%L{fqz_d!Rji{o4jRM^RrqNZI2?(42)tye7e{avM;4^S!$x>4iwXshBMY?h}q~O>ErDG0rp&*V5S94 zGcLUkbGX6ta#p`xR+1A0kNNesZ899-HC(9`J3GO1X`51uLQz%)v&IdJ;!FKJ>2JQD z8IhRn%Wmw3Yt@^}R&Hl4PaxXO^YkRIEpnOxramUyC-N;OF!hjKguF_&K;&7=sor46 z(I58Vmb^T|Td9_IOI9E-KsS#qGxpnc9U2;v&l=~h%oT)P&qP7~PlyLf+32|PA!+G# z8`|N%#jn3MJXTZqJiXg*RH^KTYtQ)rthdT#Q?+S>*a|jdN_W zzAboBNQ0=0=3VLnjPl1Me^+i!xo4Bv&9s~x{>XQD__YlA*!~5L^kYa!FSMsq11>7w zbqM+wC21(Bdm<8muNoV`V;?`F86nHYinFUFTR&3VcoEI10j9w-R`27(B>a_n0-)slv9hG4 z1sQHR&OKDKpLAJq!YcW7-uvXAFzw}DHgEcg<+8D{;aB;-KON%U8IPjNjp%tD7kxGQ znoHH0l7G#L%z-q=h8E^X(z#3bW{(=Ml2ZNe3hp=ou?IIff(q+*=xk0LhBUPrEj;kmg=V*Gu^Cs6FoHHRL`Toe#Wy;&OeiN4~DR z{fM}5diinNY!%~~SvqVV1??=mBkWM=N>3Sr9jM#yoVNt%D!6nbW`}Kj!e*jkWgK^B zI;L+(S>1tiM8VudG8~hJH&Lg;YgX>;HcR|4Q?pLax^RkJ8ATLK3;X&puS)bY?Vue} zCpets(T9Zw#k;m8apI20Xw{4lH}-DQ*dKaw6$M4ZWWHt&YHbJ#C^@Kckas=3D72xy zo|oU$#11*rq9BhoY4ff7x#MPn=6csfYmS(f@OoC(LDaze~ibsnrCPtcv94 z3ooe?8_zTiB%d=ZM$8FbMQ^(@HBSAJm1~m^Cds|%{DZjQvhi5HIlLcHiR@{ol^S8m z_h0ewXT?(r4uUFRBrIiH68b?x=x~jGVmfD>KEJAFPWW0beVr;Yqx4{TOOp(4eYn`K zrj4(Xq#Dr8Phy@Mm0HcEWDF5~_#@#Tb-VT`iOLISk);9jtvL)gsDhs7D?zsEe$_PR z`bmF(31(IIh(H6aZLaSXQ31@&-{iA23+=49zEgJoKK|T93l%}(`dz6Q1-taaGUv6r z{o?yc;`__yN0+H$^@`^ccwuva;q@?N8_DgTnFsW4YwwUzVsjyPq&@!nlwvJo2cNo# zq>lA*(rM)LM;TrNB0)n~EZL((0RYu(L?|^IzW*TPjyeJ6j=o!5;@nAHQC-}BzQ;Ex>XUJ3_Skj#zb{2Ti=Ko4 zAzH@byg@8MYE1ur5cwu(@z$Rd^)Q^=d94VZZ8@Tdn&XVEZw0O0x?hSa@XC1BvitMu zdXxcI6XL1(l`>Tq!hO@1gHvA9Bl-1~&1M11%JurCEbA6+#)eV2EW*rtA7(W)B)gM@ z%_!EOyugZPRqfPMdu(BS#O)%Vi>7oBs|~_P;qQ>l(n2_k9b_uIH4ssiVg;hga)*3W zoRjyguy3dC$1Yqud5v@^HM13h%t_6*f9OMwyliYyMetH5ue zUIEIjP+OMO0Hkyr^Wv?q_DPIQDu6PzE6uW*iD;f1JELaKqnd|75%}BGaObVDY@X(C zHGI|SyMC@&l5l5rLx4_togy$2SEpsW=5@o}!5A~#0i?gGt27kYN5qL3X00yD7>ecZ zBqw#pPhN>*7f5~$=D(H%_?X)Dj(&@G#x}=*~oYz*A2*79|2@bfMN5<7%}`K zqaay7n_akZ0)2Hr2W?qnBH=++I09*8B1>ZJd!|p3yh5uv3oW%E)pus0(c>hmBGDK! zv;_$)ILwooKer-U+ah5&)gUk@x`5Osss2opdR7)k(B!esrd5J*g`4wKJvXfp$y=u(V%XUd1(Nbn zKchkjJ}{*O%KG7QemK}0t-snw5O14KdS?Z@x^;zO%ZxDRrs2}U^X=tqh)!OOY9zIkeh%oMd+ zuUDINPG4Y|Kl9pSt3qOrK?fz^JR+0tVQIslyvPWR z9A3Gn4?=i?Nyt@GV%KbKIVHy5D+`5kL9~S&%WM1omXeX45%TA$lpckIUyp%ckCuts z;tC$13|aV?W)`e~06%S=YN19P&(!6KPyRNVwi!)vjM8w?*^~b$tZ7PaBefI&YhwYh zWj<5>dAu32*Sx0ypO$lM>E-dWLtJ^>qy3-7Ul-W0g1-vX@R6iph3M2758#|9TDM;| z%*tOicfVB)i+-zBp@94ID+>1Nhvhy|(KZS*9~4Cz)*WtaU5yz+=llnq^cYi=#1hj; z3_oj)Tb@}X?05yQ<$%Ptr)A2^qwl0+0Fv%ovU=_ScJ=-V!)mo?3@!4Uo&&P)i{f^?+iZmW@vHVfSzm*=~XP+xZX)o&;}pzLh0n(X`i!vu~|WN4Vm2>3CV3iEZ6nC@Y-c2ocu1hsgJK z`m3ONWO)_2%1^kKJ(qvWAOi+@(XZZX*dw$Chc@q)B&X=+ABvvXwU}u*{hRZD&t(2k z|9xT`QyZ0qKjP(x5Y}yqGWW`*gVG z5Eln(yTNL5ZJ)W`f6C5U|Momzs}UddT|PU0osZQ^g}29~$OaB1qf;DtJ==@m~pP7_}nxXewn|K9KS zdB5cZ_%oRWl)gKmQM*RvyC;l_^tP#`;P` zE6{06cjYoP)~j;|-mW*hoB>&a>~Gq^j4}zJPc_e(hnJi5wcVgO6l<|nT{QBPVJD^S zDWZ-uF&4T4)oGwz+gXK0<=#~E${a4hgW5#sm0TZbrUjh4WyZ~d;aRDzDcJ=3-j~y4)vW_bow1jkLM4Y{{Hcf z)Qzvn*PGOI?r29%@hHIUn{gsVYSH)2fvTTIjdaV%W%S@>7sB?xA4+2w=WTXN7sgEZ zGx6Eq8uQhqppc6E%~bT;Lx=3G1DIB$Lk|v`PL)#q>T6Pz77|e9j;$Z3jA|PD3h%#2 zHlVGrGg&T!mB1-nR*L0;dQH~)bp2!8_<+NRTIPJW>*!ZMp;i$PmQe(kt{es*XVGjH zsC+yLMjz#nLdAZDLmt%#Yfjn-M&zGC6)Kyb8hku2(xLRJkuSQ+-7Wsx9v0K)^8+OP zd6Af1mo){ums%r-Mf-;# z3=KD0Jk=D8D_&2SdA3&m2AVSn?z>^nHqz4U`aT$s(aH&6A{#2nBjjIQ8_$CeHB@QF z!vSIHaZEvm7S9#SrYp~wUYxEyrA&q5>T4zysvulBEgFLzr7|K=X&;&4!Jf88Vh2Ik z6-2|Hd6gHZQTJ*=Ii%P)(%r_=;q&3t4=v$~U4XqeHjN6z7ID3>5zJJqkY&3DBHnDD zx6C8$xj5dy<~bs;Oc_ZXk?h3sVm7y?wPil@23Ry72UOC@k&f3dU0&S``q%^=?6qeO zFf7P7saI#(3Q76J#+VBPtcUhZhLRVz@H}kmudO>1$6|6ireyY<%1QS3qeq`cppues zDnjnZO8^2PQF$Gwf*q58K-ho&F;-AW{PFn2`+QX3*aHrWG`DTJl}roZoC$Iw>2$J_ zca0UPYNrRJ^^La5TlvX*CJJU|$hD1}u-z-*fKS3a{37L~?Jgxuj!^Gvp@chA$jrc` za;h{9(AGGBCzAglnBL9pD^;WTZqG`&u+)X%YsYdKFoi;kmIJYh)d=^~nebLm8;&Y% zbaAVQP^%tm%*3pU2#QM!w<}D4alj@FLR=TeU&o3M@*LD``SFmo-koCfmJVt+1_j7# zCkX0Gm#Y4;A+hwjU$&52{*JJqWNrKEv-9pFaEyRQSvbv6BAATvtjW9?1>4;rUkggD z1H9j>w^uHS;Td`U+Od^0`bC5QrjyeL)jl-wE3<{rT4_`r?MsUmBkp;sdKDctTe{^6 zd7dE%OE*O{m2_YTe(*y+-c|jJ$3FgeW|GE*t_z}DLVloES*qmDx~=l=-D^|SOX{&E zvDfWp?9fLI(U1K0H*oH!<0L^BM}LK#+I|$4SeX>sz_lbv7Z(=Wm4>o!bgonp)?KsC zixt&vf0FV~oQf}7AiM6W9FPmy4LG#}0AGaJhJ1aoX2nRg|AcsSX;j3&qba5z1SBX6 zJ}7eljOuaj?)>A#4*QY(E0 zic@9ZlUZkKLztuPCU4N5Ut4QSEMldJaqQ<2_zkM($N31;bQ&0Sb-q*m38#VWYdVcR zMu`{qFEkK>fMO5iUQ{IW(=7^Xrutqq;kN}z!sxn7^0`n1uCXnD#HKWxN~9e74PP$K z^c+M~ZUh{-+Pnu~cfB*A>><)zOA|hOjQnj472`uF-8wjUYiDPt zIV?|={b-dPiQu!C11U*F(i!WU!aYos1B@r@aQoLKL^bR{lSu1t9Mp~*)8ZY zI{mD(ZN+K9YQ<&&W`#8XEX?-EH{yJ1u60uvD$z>I$fB6?vn*nky0X&h349=5VTe5+?LGApSh+{NeR#nC zkxYO~rs7*{mnT0zwK~m5s3v+d%9-^*LsT+&daFwhNX~b5qZ?0MEln^hfL>veLvzM9 zlO{)KUipD@A1-t-Z_d&M>&J$=Q7o68x{g?{tlAs|X7Nit9`oeX z3%VZjNM1!bAZ5Rr!T1W62U@X25OsDWfj$IH5gvAXETS8K--ET>=%As?3G=MAd&6R` z1fB|!)qr_?FR0{(AgaSu8>@dQIYA2-$2BsKzJ>rGyp&Bt3lq9*tJJPaInD-W6r9S) zMk*&ppwfduRh6twF|_jM_u1my*T&&L<(}LMRL)fvd1nQPh+%|BqF79sQ9km-L!C2| zaUa|S4ACNtI7an0@1zT4;yGqgdV>+j;}*;%$y-OS&X{o~rV^QlO0p{=vtD|05^E_& zD!=EfgJW62vgR8s$dW{?@@bcfaUgy{Kg#?&PhBKPpVXYzegVrP zb@Pp^rh~=mj1gb_D~5UGq;Rq=jrl@HrYUL0bpIlq4WtS7QIHBv2N}U*lyt3UYwYlP z&&|s6H8m>8UrG(Kb@*8qPG|H&ZOBDRJLI)!+RF)0N~{*l#zM%y(3lI^o>U@X|IrS3{8&Ao>ZtjpTxzMMYB5pi4I2n6{{x9x>~>n;ow`?C zBwl7hxM#L9z}#qB5&;1(>YD>HDnda~bjrm38OpAG?Rk=#j^MZa*QsxLxmErWWJ>eX ziQ+fV9)r9mcUvL~L=*2=bxX%$VP+fV6|L|Z@VD$3(xIwR!t{L=r~g;g|sMr(6&n>G9q zJ_X?*Xq7laZ#g7XuOFIhNh7vcpbL&cYPx~=3P^3U7SrlZG>Dc zlCSCKDS1AOpF_`)ICr4#ZxLC*BF&6?+Qtc458rL0Z#Ae!X492)Dx`#Y6eG)tI4+|B zonX*4&=CT3FJA0toc8VA^Yz1GUqK}|((UfFsX@RDdIURTgsTlGl&qoDGuG!&$>F6p zQ)vKZ0zzz3vN6iiM}*I2N5;*Itrx*EE$o1V=(Jj}G=`2eMxew6I%>q$uz@g@ijoxO z>()i)=H3hn-pHqEiFIDTLWWly(n1x4f?><6)>a7&#S^i$nzfa1R;|aam_u?VvIov5 zMD(4QS1^!8dSx!B4W~U9C^~oT_~5T#hi|z*^j)qILd5g^Ve=-rs16Paq#o|!vBiGAKNeef{_cvi7pE)6~2q#cp6#a2>kqKoEe1}12AUOpu zI&b12e2OK*nz=~%t_O(yMIZ}Ifi9`)0ErWG1=*_seoKj)UN?}$56|mZKI;7v-BPy_ zU3;?>sw1#HD}2dDIL{SK8%BSk&Jw(VRC7VL7ro~W%w3$dS+rMmG&nbKM+1m1)P=03Qu8AFUTXp~l=NflS+$Ip(Pdd#IDRx;yj% zYQK6ly}B@|BJ9qmv;I<>%O0UlsCDI&@WzL&oT!W_*mq*bd%ObcI&FWnx|e|sV>4Q$ zFkd7R9HuE6&R~Nr7wg?jTCl^O0H3Dw<4X{$mlFIt8sLRBfu{4Ud$X6*1}+RNS+A%J zF{!$MpFn}KuFU*)RwvU`B==rx97vhhMR83T-A|dK%CK|ug07DRSKxIoLjztu_QG1x zg&1Upf`IOf$^Sz~elsg{G-dmN9t*)QdgzT<8>e}@NuqWmkO%+41*M<^&D>lTP_Up_ z>Y3Y1th}APA17mYOb7e*3q}BoQ?OQa-P=_uNsmj05iB1|O!AQsvG&m0ZRK-wX%T~{ z))OL(UIr=&U#Jfhk|M)Xd{gPZ$)a5VtqM;lfllsJBAUv=3`0h=7pCQcc>n1U1H%wu zElTLnS2>fPez_`3n=^xifXxUoo_A(cNtD(NM-HsHC2S#PL` zj5dhtcGlc&5Z!GsZ(b)Bn=y;hY~|Ek)!+sy-RJPro^tELqfe9|@3OmG%lEenziJ5* zY0%-(_q`hl{9*r)*LmdGeXPP_ImAJDSMWBy!+V$i6mLddbHdxQf%yt}O`T_Ye7-@0_vDO6bD= ziU{{&y}&PsDv2-r4~r`3{-c5$^)P}q<$Qju7d9ltOLCAG+Q_q(AZ9vM(u))H>VZv2 zZ1Of8nxz8O8Vg0jX{=qh$7fz28>&k+m{|Ar9Y+M~b1Tq1^dM2O?fOpXnoomv?9f9A z+0_xR761C>$P95|W=PPf5Vhi8>a?GS;R~_v61irY$e9;K5Uc_c?^kg61FF~uym))E z&G9yEHfTqtG};Q)yrxENrBk_zUz@$T`H2SmK$KN6X$$QU8{hH;mkS=}y{D>MP?NM7 z&KX~FVE^ou&SI$9xxghsjn0}^Wt-u|huamkm>v))a>~v{lLQI$3+a~mqC?H`mzHsG zP<(XAaN^9_?_;mskn_e%ToD0!&Nq>1fn?F0B2ligxVVO_dQ_I#uRZA(>d4XYi}3;w zMkNWO6!l#=5$4ELt0&8CgBQ4TVo*~Jr!nz6o5%*25tDF|EK)B8IDlAE%P>Zz5Y3Pb zycy1e_P;!HlV+|BH6Mx}E>ORtqU(z)dmJvpVSc~S^uB*-e8)tMK3@%Dw)wX9B%1`e zx{H&D9=!YaP5rkCHN|i zMGErazid_eo5RWaMO+49rJjhJ2B^W{X~MIekg;!AVwZob{#x$KGh#a^fd$MVTQ=c= zbd-0~rio?i?~bl}63N)N&pg)5+EJBgH}DXYQ?)C^o;hg_&n4VA@EPr$`a&%%o& ze@n8gpQ^5^jtWkeP{*Sx5cvD=ox^&3I0$L>$4Fx+Y0M_~11A3EUg(fb?S^z=j75(5 zK)l@!-;O@H!Y_50DvJlkv4-ezs|%)uiop|Lh5CY9*HS7IAN|Y2V4YSqvH?uF{0%2X zJyPA2j|qt?`W_>1$`f z>EYWgYWF%6dm0opIr#$xqWD$vrWivs@b?<;@y9Nm{TV<_@zN@3rSR6|QOfV~*r2x0 z!9*K3(`%1}Vaz|j2s8*l`COuTHT6EcVB>wk(KFn6r-Zq=Wlh}2C*N%{%`M7Sc%td- zln(6&6#d`(Luz5qjy*$i-y9Y8L`Zi3eclxxA_T&(-yY4LIi(*R?!|7|C8WW!p`%)7 z9vh|>w&9pt;}w*fa2)GRNiTY#e9rIzM9hq7w8$j(g$i2dMTxF5e?5NuAQMARksVTu zKfF!1QCyYV}-Xk*jtOt&) zwI;FOP4~rHHJqR-=KJA7of>hIXW2y=f(@25rHxaL9ZTc4`!!YUH6B(12lKR* zZ*wR{$)yW&gf0?uDhwtl zZn~+8EUl1lDo?$AcE5re7~mW_5FYYRZr@~h5gzW78geDF)<{`*sPWdk*+y5TdZ~84E270dqG#Z&uNm< zaDCGnd6^S+RM>EFEvBY#BzBBJYX7INk5hqOB1{J#_NaPICd&2AZ`_TW?MEQu7K`r*8zPxj zIcXNcvV8p<64!X>njgPlw2Rcg+jAxD*8IB^qB4Hhl6}L)i(lJ9-Qsyik5(*BZ5Rg^h#Vqi#r9fz_uYny?^`*i2%#H`j==mtcQ-z@fPnyb(UYlH-gz-H#?cz zf%%sDF&bx?KN^JNevG7lQO?~uzN%`0MHkgJXt;{lPpm{-JSSdXoAuV8)#8o2A^qYD zylk|8X!S_$R73BbD(Q@#(R$J|_DtVcz+66J_E;(s+tQS`{k53B0y@rPJX0C^zH|{9 z19R`vE=!1WLE&>Cca(}6N{yomeEUvsrQ*ND} zj1G9Y{ba{w09CeQ(6?WIREoLqGmG%rBNE$>pV`EO>~SRHp*OZ_R4=uq+E+cC%dclR znt&t)%g6iqXO4VF9 zHerLW(kt%l6kjJ(ziljcvG$fh#{lQuoS_m_=uC`u?Hn*f32sFPMv@zsTJ0c z{CTnYLX?n}Y~I9Jtyir^{TR*nQoKRUT%8)FaKLkPq**w5xN85fPI)UyRhe}Ib}W&} z4kdpk{fo8jlUr0_D}`*q-toNG8}jphZPW;{+dfYuzKL%^y&z|#j7*PX1l1IXeo#JV& z95czGk^c|>_{p3pj+k)a&%zE%UR5OH3ZzvG`ChC1qeQFTm_uf8TCyqI{4D~sf=}WT z`q5=uhvV8$lXMAX3f&c&7{W(QF;hT;$hfXCMSZZ8GimQ?{?^aQtNzTP#JQURP%dsg zJfR&UszS}HVn42DaHh)hM@kA{079}RXL`B4e*T)Vd|&7!zdi?EP^z z6^)pS_cJ@fs3<^l3tw%`OlAX$B26nlLhV*Rn*e0(+ygtRab%~>nv5WhryntSEhcts zM##?0C)249v>;wlhW2!QH64yn@;oo{&cVg;C}{LMMN%`fILx0WEjW>)trra~+frR! z!|kA1COMBmMfn%pISAV1>F+;}u|G17cr~S0bPF^fdIV3|ny4bZj>*h57VAPS-pZjx zr}~XQLMIgAdfd3<*Ej9yV1wZ&b@w5XFfz=%+c$2Bb3`vy9;c{+RvIMIKO|uFfvGLk zE=v*;?;sa5V?^c5e$@f+12G-2bb^h;FR!vy4EnjjC3C>-miTP5#}Ee<)Ucvj+O<@a zQYh!J_Nzl1YUxI{SDyJs%2jr8{R?ylA_k?z&7i-xYQI&G9xCcJ>h5;Y+N)`5^Q!L3_$xS@)eS zyz0Q_s2H#BlB%zW@4~OIHOm#5^LZkJhl&zQ@&tqHi==6sq#AJLk)e{olB7n+@oPw= z*rRYp*220-PMR#~r3t*?y2QIzTc zeWf&1r8GQ+?wZ|hOFXp{B-m9Iuce$*buYUqrAf+oOy8UO~zzI?3}5b@9q@WV)E}I>R*%A7mj_SO&vSRjK6uhU%&(lvg;F(dJKoYt6w?&?c~sl?>Zz}wj}3RovFlQvxMyLvS9dsXs+&m z*uzWWmU2>iOddZ~I@Wl~2)U98h>au3DLpo!@sjGa{Y7HC$)g=#-r9x2MEC?`5s`L^ zl4NNhnP+*2Mx!xCPd45)NF5oSWE*!+z~`vFw!sWJgMG9k*0KAYDe#cM<527=J+v_ApiI&lfaz z!Qmm2VyN>R1s5`{DF`SK{XM|J1Tp)!t1@0Ica|4M>N~a23-nwZ(v`gjocI|O!V@H! zCgXvYj=un+XG0d$^k=Rn%5CE!^)OsyPIozd8^AKuF4V;&EnxHQBm!B7X}lQx&K<U@-pvND)^S zcKL*WzrJ>KzjaD?0#m2jz#C9SC?G^7393%usZ=<1GFSs9FamK8amF5eg}2NCi;GU4 zR-VuWK|)Q*mJIgf2c115^Fe0uePsK-U0W^s9#_?tuv+KHaxREa_yGvHnZ`xes5!l2>IpHbkvYYc&Vv)f~CKc#1Puy_LpAY)PZ%CHpro}N~d$bnlq1&vSMSlL~N&gu}R8qn`%EJnv;`4 zhnMGM$|Lik`(5|RvO|-ep&o?HC6x0p^sMT;N3NoAqHzx;%Pj?Hhos)TJS2uP~!=9HR%?U1l@xz-gLow|vEmkhR zOs>IEy<{?pn|*vSNa)VFC>b=16eO@BNy|YX-&E*Cy6xK+%jjWflNf5m|C49x^WTG?IeNT6&GCI^>I?bH4k*1-AEt+tdBVoEp1)2z~51O0%Q|Rj8uqZu@r7s> zTk>w%PN>!B^3I(#xc4aXI=a$f(>F16UNv>%V4(Nh_1-LRn&?{1)Ko}3qwAu;zGPqhwh&G+cfYTm0jMhHiIj?{@h&%{quVN2?@IcHnaH{Y|df3ID3pKMp!<;R_=Un9`QJr+z(hA7}3s z#dp>p7;D0XV}wMKlFX7FjqUFhdKZpPuNs03KYSUgbnV_o;3|R425h^{2g#0*9ofCo z5U@c*5Os_rAk%IN-zEQg9YR+LWjw)s1J@^T2lnnaaJ3-etIYKuE~|*W5rX!9KqHaq zPA!D%6~3CJ0EJD!nEpjmvP#{n0O(3%7XPOaFa?%?z+w`ZXkuDoZNj8};65USi2kPw z6xd1V#)5a9tU{_imTZFn45LmT&TE!!|CejIpnjvwd+n0)aB`po)7SvEE|L_Oy1x;6 zcK+AbC}sBw%jU$l=h>bLj#c(jIOV*jQqZ)N!R1W>4h9b~xLzgg2G4LAp3uy-9J)T8iMuh_Q3P#M5pspZNCXaMrPKGD zgak(f7hj_Xat|VyZF-n12&#L)vm^aQCBD24Nnbv|R0^?7 zAcZ_e;uhc`vOylA>EQ2y1f7}wwlP;B_i&jG&GQTPOFT1x+sjJTD^{`A_#kC}|H0cw zzpKJLaxQ4vxD752ds;yIPZ2+@MCqavQPN@`G4DrBjr*+@hdTbpYBJH%Ze1UGgW1@rDI97i#BrTvH9y{z~Bi$3G^TvN>^I|FmQ`3eE&aW6JxZK_ND>4Xy zA_I1e^GQW}Tr_twDzGSVx@dCg$tj>ywiCxdY5<{6btAruZx?}Z?m<1(^kd!MgD8(c zLEtu$gckaH#|!}lvVJ>)e$dZV8l)B6FwaT&4jRb*_>S>qmWTRnlI{-tw|;&^Q24yp zZ&2>PS#Ec_V0aW_y9%)nj0U5U|9Kk1_dTlTIX4Ld|5$GDR;AkulRD+Y^}jZ#Ur~F# z3${2jH%hDm7=wgugs@*{yiwgz`jU)C_S0CJ*LDG*x`u#>zl>lek+T5<}1 zq#?`v8oHTw#dIj;@UcakYfVKS5~P{c_BX!lS!E!F^|OAK4D{0<;{CWM6fQD9b6WT? zO!HA@88!pqaCDHsiODG#JQ}CYq(sAe^uDfBm$5EvyWv143dzjVawbmtV8~_jc=Dk( zPW!*U*&t)H8Imp#X~tU*#UK8wJpd{GmFMBB6uz)LxoIEUSpq#Et|+9YG(CkLQu>yX zqFTqySfF)q*4&8@*+AETiz%XA6l-=*oSeM;LI!N+h^)!W7R*TR5 zAMtZloZf$}$W1b<_@w0=1l>G!A>2GL^+=N0clP-_OEF86nnva1Ie%)j31H7PX+OOn znW%9w2bx*k-~t^W2sgqxHtNX+C+9Rf=D{ySLG)2)+-(;S4HHhov`JTo{zX#3L}KeX zhIP%}(ERDaPjBJyS8SXm#}}Gd+;8HiLyVrvz+r|yX;qcq#-s{5Z0G_NN1*xEboc##TU|Kbwq>Hr&0V;r^bl5srAjJjvG<TI4?=9@XUJwDf9-g=_aP;GSb3NTFM#oq~$VnGt?_XB+P zxk0P!j)~Ng$Ml2yBjT3vmOHCSaaA`m$(-JZPkU$}!^wef3&94!1L5TmkC4$iEFK;) zekV0mD87a-t3;Sn`e-=6Tk3T)=oYqPQUDPl)>wy1p54Z**l8UKF z;A9dTrA8gB#E_F{rg9S_NufR!@;S`npE1_cP@mMM<#W*7zy&fNBnCI4wR4~fDF{o* zi&c1!2ZH(IRaD7HsG&Y+e^Y4k_HL+;s`*RKm%qscE=&qUMGKHAFz z#Gw$OpWDXC!Xs*xqiLYM*A5;S+PKhg-?;$8nZmo0EpnB7k6c6{(=r0N;%{yH9b+LiT_u0?guoB$ck!{r74x z`QqCnCYx!UcAKMJwx6WP%vOC5)yyxFa&FB`P7Q1fO&0jhv5G|o?W9$%X>He36fH9& z({kI#)181H-9-aJWfcf(iSY07CCYA-+cz|3vQC;RHq6YuxpQ4^bxJByb?D|GU<*V@Mef2R8saF$A0e(I*6d zkj#eci;Yl*qt8RiB%0=FU`A+Sz;r^;2m(|kB6ZGCugrV2ZzAhP$o_ertmFHbYhPbL zWEwZaBA+xQ$b2{Eft*NK>K#R}*r82Aa;M;3KPV5B1_WIaKy9FC1=5>BM%6;53*EE6 zEAQvZez*dN#@ZPmBbI~m5aC!RGDcvG$h@*Dbz?8Jasw9|4tIe1K+D1Ub2?mZtv*Jq zhrK{%OM>CsynbQ!g-#ow=?S2#4VVlavIk21C44{7q|i0~Do_y8zi~2H{flc+wEQHR zg7#1#*8-%pMGI*S&|^rwA@QfQ5q-`##?QXGApzJ~+Fh#*YE^LB}hVp#NZipgmB0XmuUnxC-8+A(u=S|L(81e-m}uoJ!t*bOGL;+BZHR1`DTG6rFcrLuCqhI;K)eE5 zVS)Ifd4Y^@-u}KDNWsCWg!Oxm2d6_IV@5c4P>2Jy-|Hkn;+N;Pv>ih8=|<0=(ITbn zRl8Fsg1#jJ|oY%}Z^_^uMmJ;G+Rl&moR&6RTc6&>%# zJQZ`rE{II$ONuOl5lLI`IXCl5e1u)UhkQ#AF|&Q;^x8g)Z&z}+ z?<&^5eL(+`CoAFvGiC2qtVvxP8zYsuK@)H^cFp3o#7FZ#D3k?0=zo+y!CbgO|9w*{ z3)>dYDC%I)^w=MQbz!TN_Uw_un#?*!=C84Vq6oQk_olh%=8d^7Tv9eRxC^J(g-e}U zO%IFh8F=fyrK+}BGPUY@crKKo=lACZAA@5EJ6Z zPX~HEj0YZ_gWHcG<+b=zx6}-eJw{o-FSh(z>TSEu5f#lMxSY~Nw9IAax2|n>@Lnow zp-Zj6FUM;v?=xJwaBl7Fjo*(6IS6Gmk&0)#&xi}7EtN}F^Z^)Nk@-kW-X~p*Tp=^| zkXz)Z)AeEEsjb1sMek3i?mrib z>+Gl-xFd+heKGBHO`FSZC1NJ}!iqti4+(S*8zT3`3r6yYMyjSRYLES3&+ET*+4X9v zQKEr`Ygb?I;!FQE*0)Z6bbh5ooA(0GpD%Ih6(xDlO0JjI5Oc=(X=Jmcq21c0W^0Fy zm;6D%5UtGd_4;!k=OH zLy&LQxc9gW{_z@;8v%_We@sh4{p=M!l;S=&hk~Sy)sbFOTd(^JwJ5nsCOZWki|qxc z&C#!3>Y-SZR{HHM#eM{u=-@sQ+sG?@D@Q8uJt6)!`>Td$Ym6=l$FMA9NOzrHjz2V0 zm;gfIUsOQYN{Y(-YzTy- zr~zMH+zSVX(jJcV#jO{m;xvwqiv_)vn@$mK&q;3S0Xkyp;O}SJX$)s7Qkpr8gb53DtL%x z(JjAr_5HW8kwCmOpJA`^o~|M$?{WvcxW8Fi=#{!)?XWZAJXvb;>do{!p1DP;Hz=8| zJlneJ!tq)O^l!buS9+kfY-uYkYhF9(s_F@R8_0B$UeXNeSgq-ixYC&ag9Xbphtsp% zZficgQrN_0hp0Hhb+9d$?OZ@kQ@5^9(&l6=c6i27-6g_Ds3FV*y;GLqVQ8@#UDLO* z+WEY~EO-il5%UxWk8)A6d0ygN>lA9Q72n=O5A9>JqM49DPf+z@eKudbs0clLhH5Ml=Bpi9&ymIgmo^CHS2*yhMGm-z_T@n4yAyc>>08~d-U4+Qf&vevN64t zs#fh4ejs#lJJoSE@_R>%uER!%$!(6b22LLmpw8#()`poXRs+(slCmbrDkz$TF0lk6 z@2XlEV(G#c8@i32Q;$0&sA6LTmg6mN&1R&_x}o~9D3j6WiG1R>wwF*apB!0w`Ry`O zbG$DFv_#A>GjysRzg5%6`fQ9zn`qs5EcK=5bIbC;ylgc0?J>#1l< zc0u2DCO?x$wCB9o(A#6a+ZEDYE7KAF6p=v#hqihNtc(h$fWN69@kQ`8@ zx_Y=`DSd!vQLH&^pwOz>XOlqt{S_;9-w{3Z$ea_BiGvSa%tpl$C}6Al^UA2$JFj<*yFXYcT~nx4sQ8 zG-R+C9h!<F%cqDpd_T$ba@pZ0yK^26f`?5RwWUF1oU^DRGMw)U#F$#ps?V{tl~ zLE~C`4~Kdi1+9?dBr@UK^=G)ul8o8n<4F^1ev6{T9S3CbMa@22t*NSFX^w$&J5F+V zr=5ZhVZ8&ROu%_ezGrQJy+pR4PyGR`+oKKZ?FH1%f=P|j!d?Q8Rff;*?yvI1&3(s+ zpO0skoM8fK=1uqNjT>b}?v*aWc^SpQ=C;3Nch;JBDsuEnat^8a$oFS_ZN|mjtj(Ss z?|YYSGU@TSOw}BT9}4E|Z_U$^UHkQ2r)bLkTDL?EkdyYIuJHby%^;K2pl6R;j&w(S z7cS1~+A-=J?_e(K3X5O&E0qz1?fOfzILliCmWEgO7dI=J%I7P-PyEc(%f32<$EEk8 zwue#Mb(K02XreOp^Q+S?yfpYz=ut~_t@x%0<7wIhB(vL;-{fXvjN;SGvYqKGFTE5 z-Cp&te;*H&!PI9pH8%ThFJ_u-- z4Xe@A$E;R0a*rz3MaC@mLz*B5k{3kg#MSVip<2j3EuS5`6xrRLre(MC{l&o1a6#V6 z-Y+bPxxd}AnTuao=_u*h)2}Bw+-x-+bhV^&YYKGY@pB23cryLbfPmJ6C?p@nWv4`| zUq@FH>EDx}u-RZQ!eFHs%lFlgeGkz$&FAA%qP@!`M`G*t1}9uqBjv{vSoEZ!^DPAD zgAk4EDg~8cJ71yyDnnDa#Os;DA~Bs#jVyncYQCQg*Lw>tK(06;Gc9Cqbr?ADq#%3) zsc3*p0lyyPyFonQ55j2R3WIJ5Pr>C5dHShH5G@fB(xk8hFArj{y(Buw3P2HhG91^y z{xjtLv#^770U!zvzewR&5lsB&`fw0U4aFJI)KDYWSND=`s2ddTiDjGdb-%VU&bGC8WHBeG9 z-MF(j7Sz4u-rGNKJcI-H&eigPv-?&j(vtd1yF!^zv(g;pb|Lc{UD+mfo8@t62pBQC z1LDhFmP2{OONszgu5|MnOK`I#=GtUED!nq^n%Y?(gA$uH_{JHzNOHjSxDFYkm2VQXMMv! z47}b3;}&q}1cmRf0-dWdL81dEhiHA)%L*7UE^|y>F==#3CO0oKXM7K58u<`#)%7D!aiS9pF)!0ax2m29Tui@Si~*wafpCU@72Sf%PlXe0`QP6@QW}2nNq3+A zGFrE?&^A*4(0k|2;e&yOVO365{qHxoupHTi#4j6pcg{edP=Dm#01G!eqOh1@ssUj} z(^LJpNmYgOl}?pTsVAh|`UBY)_y3#h8!#^6?rrwnJuU|zgW=M@^*0`XVe93eW2Bsij7NlVz0}pcphU1XCc<1k7s$wFNE#rP!a0OFxG_X#>Kpa*e z@eN-VsyRPR2aQYJQ$FNI3#vZx07(kX8&co`3LqkN?KTkB1&9(@q#Cl7=C>jvxR|geoFlOYpPQeux(acANkm=Sy=9yl%YIPH!!ZyDbggLbkS5C>A%tA}1VLt+Q0xc7b3e~Kc0Wk~ zX+T~YaG|K>fe!`;PQ)PX`}gVKFaI!|pQIjKJGeI;aJyi5fh<`;8-_FBn3V`AVmi(Lte>Dg0)lfU&y}f>CHL!Bx;q`Xdyx!zm znfsaRJ`0}P1y@lOi*hrzB)G!W<&ALN| zp|Hqs&8uQ9vcW7@H&EhoQ|EJn-|Wu~k}TrDTP9?KcE>Y9P%$u!0Gno^2kyfkX(WJi z6X0;jcJATqdq_7Sr$ZPBEXtortG_3MpqvMo9JZ+t#n5U{+K7-0+69NR4DB9& zm?|hy0KwtFm4woVgoIZrD*^4loOut}=Cs275ZDg>`9~UwEc(xJum=idn+ShY zD{bzX+HOeyP6femAQVw5rN?SJmBR9Jbbx7K~< zsWxU}#HkAVHmBy|^nIZRHOBdAHtqj$6&1stgadl|vX5jr@Xj?!?Ji@@zB@_%#n!jl z$AGI2l@UVzG00E;PO7=~MlpPUG&WUUg~b`4PWxb(r=Jd1n!14Fkv9_LXd-cH#p+>Gr4vToMz}Cz>`A>mM6e? z62orLzofLFeA#Ydr{;I2=zo%%{XfV}5wg)x6#IX2(&3CCNRj|Y`+qMXxR)^oQ8dH^ z$n%17;%~0*ht9eK1$5Z|Oo=yC7Z+Ic@*+0AAH>_!%ml@B1JdD}y= z(h_y3=*tblEvluzC|4A5(dk-8wS|N#$?w}K;i@9Z2%h#vORMM9hHF^E-M?k8h(Pi0 z|Ipn6HI1^rsK65o4d5c&l7d|cfFJ`mDu>#75`mPGde1UYm%4B0q6(UtK%v$279hNO{^m)WCq1iM^Zg&(ax6!;4hC zz0g><7VwB6xx}R(ex|YoUR!j1APQ3}Kl2fDI!m%+_YF%I^vGqS3ob(u*X(z?>F&?^ z@8vi_$Z;jp!LvrCIUp4Y9{c$0RqzBM3r`Fzw$1MoGmI9^9d9}NyLQN)8KPC(&oan@ zO#YY=+WvdW%q93Ck6nxCpddI^`OXOJ8DhO*#eoNUXQzfgfI7b2%EOgxP&;&y)c&Xh z=6%#KFMF@-pPz2Ot3fiOq;8`a*Dv&Xb&k|u$^_3rVv^GKiuKy^e`SpamJ6L;chC$2 zby9A#tb4_QF_P|Tib>48i{&(4_TT{&5Q2Qy3qkGc;0YC%8ZK~fey-(FDwZ-<>!? zS!iAWCuQBWIcRNpk;>5Ysd~u8uZnd$gC;z~U1{~&zdM)@w%kB6ozI?hHt3EDniW#7 z&d%0LgS_g6Y!cNlS0fN){RI)-2wYao`O}?Z&I$QXdwo=IuQ)e=1Hr z#^TZguAf(U+lN=w$}=5ji#@Vo+MpLUGkErdmsFbKCf5@+b3AbE%O8d|!s~OG4$(9o zd2Z6~*wo0kEiXm7SNeC?DyUSloT62Jnpx4l3D$ zqRAfmNaDncJyju^>I*p*^k=3O3Quq|D0}7P&|Jz5U9_F0$%3c=@#)VpW}bm|%@3qU zABOT(LZy-J=NW@WqG+1lS&ivH&z2Es7d@Bmf@gyf3PUH@z;D0c<1*{F4&#tf96_wC z)KC|^siFA#C%56_swhh6Eu^0ptsMv2;>1ebF^8|SzR>GKzicasiN2qG$JxOT%-OJm zat_wwD_WikQS;fEm!PUit4hJeg93(*@?E9shI#&^peZLp0p}_@@FC~L+~BLJtEgC* zAe)9B<)PAoRAH2Dwn=_G{5}S{0R0+Zfi}ho~H90GZ%5+p81#u}k012r&jZVL_{3mN389P6tKdjjac?)RD$Q zj?iqV@E%>WufNpJ@Y$j=!vwsqiHAY|ABy->sUOms^#)E~r17J_rEUJ}#=YkH0n%Vu z&B~Ok;QOGvjQ7ttt#652-Wn!y?Nx4(d7^4egtx8nn$-b9mUlGeLV}IJLsW|WN8q=s zN}1S~(|rllLE*u7g5bBVw82#MTu_zk|KESaPj1USPzfHdzHk;|4MI&>Qz=W)^xpph Dksn_w literal 0 HcmV?d00001 diff --git a/docs/assets/t_plus_k.png b/docs/assets/t_plus_k.png new file mode 100644 index 0000000000000000000000000000000000000000..bee9fe56e204436a43cd3ec0a9e3755b691d7d5c GIT binary patch literal 18476 zcmbTdV~{98vnV(=&iIUN+qP}nwr$(CZCht-+qP$RzI)$&yFXs+Mr=oSS9Nu|GNUUh zv%}?NL}8&Ypa1{>V8z9R6n@+H-^&OB?Dt;lM-BJepgRexIVswjIJxRO7z6Md*%}z* ziCgQN8Y>v<8@bz$8*>2w0GpXBsX3`hOK}+5TGQzN2ZqMY+U^${0Dz0v%}(FY(%1>l zz}VE>hMVA~vzGwR+=!b%m06lj+D_2e%v{XF!C28lM#<2_(vaPVfR_i4%Z=lgfVHuc zKAxMkm5n2Z8#lp!(dGE<|7Vz%0Pnv*oGiHs{#z(DX*oPWTL)u2W*SCnLppkTJQfxj zdM0KT7J4c?20D5MTDsqpg_@p;gPDbcffet69R$DJ9E?mj6of?n7q8zZZUQqWCp!*W zT31(B8dpXdTL)8GdUkg9|IlDyp#FuRc67IK(s!e_aU}d73PQ$?h7RU-PUf~Yc>kfO zZ(!@}#7*!k>3^$WZ6_`LzX{to{x3uQnvB*>-;S1^hK|%E9*LMH!jQ#`oD-DONgZb~I=vxWd8ai7W+c=2}aTEN$ zp)oQy;-F(>`?Vq~6CEo99Sa*hJsTYxqY#sj06#N>pfH1w(Erf*KjG4|veFCCv9JgU z3X9Ow3$qBY3)2g*F$uBIF$#zXGO_*-uDFe(lfI3i@&E8O|Kz3b@&3!g z9OnNA3I4w||8HEQ|36lv{S}7xKko7WaF_oX`t`y8bpN;W|33Wh@G-XejTwjE;CN;% zHwFL@(hwKoS8`jw)CTuMUV8i4&P=afVfTpN&cvgL=pNRo%cD=xi=mKdWBZn3``>*{oYUWaXWiUp&ryn|-a?6V`X%f@gntvk zF2yl-q7q7ITe?_+1Up zqjo=%DOCRXf}zL1g7JFi!DF((q)=-vU;2#oKhYT0W~s5$;3x~0hR*#|om!7&jXV800Oat|#!Qquk*M`;BSegf z0~sg6`FAOsp9JI#w04W~W145)rA#Xp9myP3`OVJy-MA-Ut|KvcSJXH=Wx7D+apo2) zFbGbX#(|5};3MYbj!{yyLk%}zrDJ=c0i|c=4sdOe_t6mX9PzFBx@}V8}>Is21BuE5DZQAmI35kG7F~! zD+2x_w#}TJXVcSOP;Mli`i|&vis2Tytv4!t_t~TGgf>VD^e-f;(xNqRwM^)_3`Wx8 zu9_2^N8E^TP}o8zn12wS(m=jjCmK*Dw-A~FDV9#+>DEFj@FT?xuw5yT_vs(C=PidP zSN&MvPM7tMhKI-xEN}XNrrI#jJMI>gv@t#yTJLfwtIzwlYG~YCg?6uZ0&;^+l-Va_ z8WH%ibfmP11%1;yS-^&B=eBGiPd#RBJQli%ts+BJ>UUNzoqUTCHDIPSNO{P_w;Dc|n)>%o*w-`VGN z#p~Ogx^#jJxWU^zLHztS&ndma;q!f7Gi_AcpXe{T_Sr0pA)NVC&gZ$#0?KWbOQdf| zx#3C%goI@o)SiEN%n9A{^mrSC-Y!%7{2bg!<@vFpYGa0fWU2;nKRI}H7YX@*=db?M z{_KK_^<;u?*4}}FhThPXV*Qlkxy-7^7S(aLAHi0KS!G!MYY?~41 zhHJkliYcUhcN=X`#a->#t!FLpk7j^>1T!z=4REILS>v1}^26LjUQNaC3%?!yAA1Gz}J51ypDq#B=J`H^&H-of4+~yVoMU1 zGhiHsh4R;)*npj?8Qca6hA;HcA^U|~sK`Fzozk`YDor3!jj}MGPGt+D40$pu6(HeKnnuaHm|y_rMci*)GzwTEg7!C*8E1===`Vm1QgZ5O!nD`H~~53O+2$(8E4pj3>ePSjl-u zW)ia*+|lZeDAxpKhfi?lPBbgxm9(E&Ce@!wlIDVKIq1BZpsyYCATi#rFQLA$L#RSl_1j%?Ug zeJgRES9x%OGahwCJNCUb@A}7ynrKq)9Cde*Q@l`liItK=33BpC|J<%LOdSB!>d#)8 zqcy`n8F`X3vr*p>c5w@3e8_xy!hElDfix#G%z}ps=|^ZLR{I2j25k1vno3!8M-g~H zUW*#|OLa>Fv(iNk0`d^n@+Nn66gz zZ_i$m;MndmGMI6t*BE}{M=h7V+Vo$*IL0SUAOJ@g5CiVsQZzbPV23Yo%w5`Z#NBxa!aIL9POot#la<9Is#MkMMD4&Nv#N@D>8 z40WcV&9Eouc=$}`*7XImpzlyrx}L}YH2`AL&Q%3|urorgi%PF8knJg6jM-(i$PRR6 z6=|f!5i86pJhg(kNv`U?WXpZL&((Sd?(4@wXH4fYAcz}0C2F55`r4UcUb#U5UaOZO z3>=2dhRX`UX(bSlleOuF<$3p&*EfwJ?Tzc_LN67j+d6lpiO#bq@%fEJo@G|xSioJr zniJiGcq^kQPwm&*yw(f!{OcV)G^*kVw)eHsI}s`0UF$&ZW~H$lheL^-IP0A$r~&6AHAuu!@OR&Taf@y-j(Az z#E76zrglz{DT*>s`G?!MEqw*oCKTU zrg@g9IC_Z@BeP#k)0Y$;b{EyQW8Jp0t0bP|4Q}J@&_|uHTT8I%uL+di1ll0q*FG7= z55p;xxC8NlivBfB=GL9y@W6ioFO_CXN3LpA)faD1(|ee*pu+O{7~WATh|8$Zt{Fwj zzD|cA-|oo-h0yHwR_9nF;Qcj_Jw3+^YK?QFQFv3H{S7=Kx)0nnvZq9k98YT`pv0ih z?@)S5lzb@w%Z=lUBkYV{#!n@oEqhLu8QQEGPxDC#nu`S$C%;3fRfcLb59_t3fpgey z^Ih|SMKx4|vI03KuPA^QBn7;-lMc>GRf|nO0qosrAs6o9K}2$8cc?8MCfCf5?xVY< zm{E^UMT(WZKdlYHLM@1$5~V5M>K;4ZuQ%RNxBC(icGqsLZYe(0z$f=v8b1fLEuDcZ zx_9DN)R+~4#tqckn|rnzt_8V@enHLqRni{mOe?dVon7X?tp(#R-N8zoos4RCs#GQT z;s7i1=+fT!nrnM`!`#~2G{aU-6nt=}y|>g# zZ>MjBozewvmOMMjMnG;-qLm}dk&OoZP^yktC-F;s{=L~sJ+@$WKaPgMS*LQCyLQ<1 zUFCediBnGCl}_>I?}VOUf^bgji2(1-&eW~zdA2k~X;9B5hTU8}zS7N@Yj(=a2)zB^ zci<*nQ!3z*u!ct9M)-%jGHHEJfk>b0h2ibdJ%m>W6VQ%r7ii|M@XXmHt4 zZCM!)J8&p;PB30mBj?{0sn~)^efQ)lz1GV|m6v8 zEl-PhpwXtErG98!a1^$T%m5+-QEJFgn3ul8!mqj;JlE+F*`Nb}B&af49i+h09oW7o zcYNWbO_7H-2t0Ybub+fm6u$h}K}TG$y(8K4+cp@JoUnwP9{A$?X(0>?%vt*!%shUU z>s|p3)p4aW-P^RTLBj~9q|F|HR2K9(gqj)^U}6m5Sg z)3Z|BiIJL}DdCq@9w4@WziSSjlhKP@B|wt2ycO`H&P*3h_dHPqA!I}x2!EcQ%?4i>c7JFnf(=JJ_H1ic`d%=l$!<9U3t5Qvg6wtLTpb|)+5_J_ zm7)%a?gYvIphXO_+QvotmloZ~$*%d$&pz8yZXle3Z~%%B;VU&uq18A1Cso_jH01#SuC_+j>@Q?mie6p$dxN#@{*)|{!AT31*>ib1X9p3A zdhX^xWAe#RTlg!8Ga`Nz159bpBS0ScKs=^jfjI@ZBg-QlF)$*i7zf0e?sC6k9lrIJ z4njo+a#8AI7&B|H!On-@Tj3j8q2|1-64@UpjSow|R+Ai9)_47QT4uI-oytw!qM0Vp z5zxPpM?~?7soV>FT%ikF6nt8$@YxGsAGv8zbncD;y-uA8OLWPsR0II={ICEuqJ{2t zK9P8J(tw%bo(FDB)0j8xeBKn=Z6PJk_GZEXcc4#jr#O916HiyAj=YpUNa)qdaBw|} zR4d4W2CzjjU9Hwb+(S!bhCS00G5C6RtDwI(%q@6hi{_mJUx7Of1GOi!vl<%FCd#<) zpc2}9@Ggy$e6uF1J4zpA;?O6L<)*;Uy8$T>Zg$X>V~2mpPF|c}?Lc*0vC?3w!U2J) zcg%cUxD1br3)O&g0(FRj!|G&V2TgHOwVkF3w4E0{wx74CsNY_G@TY~pKfYhJd_eyt$Iqfoa^ETKl1E~k3Y|E z31YXu?VG6sZ0m)ffVGu+1Ot!C#)cfv8xV|ToBz};wKZ+4Yo`@#KLG)S+} z-T+9r6a(+!!1YyIcI&V(3Ye*n%#RAxB9W+KT16)_UNRQ%yFz%nn=9$(nL<95&oTt% zoRjWsXh6*II4`$???n7i7vG?n0z1u@BQxcDNrC+vme^EFr+M-&!59u80M@FRQ?8`t zEo4hrZSZuKK)3mPV{x>ma%w!4vl?FL30o5uW)}w+zJP{buiL}{mH49r*qWsbLwbE8 znQ2&4BGYG^TcHHjV=|iH@0?(tdVIrR@^ij)!wmhnJf4*l*1zblj_x`!#l!NHe(#z- zytL24$exC5N4rk$(f6z!Y)7x!JTE{DBM|Hbw=E%jU&JTDp0f$V-?Ry8fqm^)zWm~< zQph04_0d#>A8M^`Jo4nx-Hx=u8H+yMmf4zIwBW%Z(S1APYOw&8Gn?dL%dZyxnS;9A z5^m0ldW7ldWQ=cX($)kYeqcKdS#FeYhOq_~wrfOL!D+Y!qFX9YEz)pS-46{&vOlH8 zD{AkCcYczsxH7XQDrMLrtPopBweBOSTqNsh5lkJhX0i!Itn9~CK`-notk(sY?%BN- z^74tO$x>amf*)#9naeu3nkU+x!$QnxzsxyJ?Vs<7ZuirqF)W|WM{@9i>Xs$s+8Jsv zy&0e7&XlF9nyUqoimSV7Wy5}7I-_ibd=&NKyzSX_O}+4M7LKyCpc5fww*>=Boq=!F z44k}I0phKhVDzf*hq|qQk;8Iqk^?5#I zXM|;;3XDtEJK3i<86t87+l9GBLzK^8?-pJ%jGpHD-@@I;v|-R{Use36tuWIsoLaHq zdNc#w0Pjd|ZKb~kVsn*25gmf_9K7ied-mLgO^aYw-X<9^!0_dpQ0DUs%&~hnOcrV~{xTuee|z%M{+;Lbz-{m-wKbZFM@Bi*f$)14 z8QuTg>|*_%r#?XO_~0o^L|jAvz`G1B$6p6D=ls(Yc^4adTQ8^GQY(iFkqHLAwZB78 zC%yKl{_4DS9!#o}y9QyzL+OM%wPSp#R`b!lcFZkl1Jbaa&BUg^ql8-(ix!OXCw6lq zks^LS^P4$h^#fMm`dfk7t8=>QWS_`~lf5b~INMDV=yYh96V0(r-vBbJ$=7286#X=% znsqOV=w)NhTieZ>)vJLh3s_rQLFxxYjm}u(I?A5tT7o-M2E#sA#A5bh6<)+lE7tdk zFHL7SEvtH1&)=j!@O9&Ewe!f$CZ5QSPUnK9XgqiK1jtL@_fF^#4|bKKUoBz^3sR1D zim1?VGW(g(7c~9kI30bTBYKk*?ScXUxl?9+C)I$95a@O`v)S$`37hUKvZHABw$`pJ#Y&e6-6E(sti@rS0jgfbwNYWAP3i!vuE&Yiu0wSQY-pueV15ShGT5>7)_v z`KTz1g?-v>5-iqub1a))MvdgwE(wui&2l@6U`7q8zFFT|+nP#*W()Kv5nT+(lROWj$sm~;6+}KE z*sh~C=$z!2n8~p?&Dr8r`{<3wnG0kH@y!wef|p*ELa%l4 zSeccWzgP0z?JD~xx08j?8d#XaB|F^B8q_cGHnLhD+3xslhpRwJ2pGqZ;gH)r|QYj3|GW$Z)g>NRgXnF>_xgW?P~I)q)D$ zv3@$}CM{L;brEhRNEiqb@Cr?3SE2s9!yltG4?@MwlHWy}d;Ej9t2~vi4obE0fG*|H1SD*RGT?yE-H% z+h+nqUsR&VvsoD_8Zfl3W7fn|Yw))I{+y@v zpM}-Z^nK&>M64mc3LrS3g8Bo=PvfQSmYawY#hcPTXh8RcB{eu`0zsT$XXD<%wO`T zW61z=pdX2$gbgidHVezrp7#96zgposLD`x2COsm8?efC!wHj(5=f`Uvd@B;0F8tAh zUW`y@9KopenOZ+?WjRScBMM+&M0oLSbgG7KJe-lE%?$$C6UzcDE(#Fa`}rOETy*TOOJXN$OgWxUjTJVQ+*5z0UF^MKa7 z4CwoQQsGoSCsRq!D612R|NH#}Usx~8$rU@a!zw$yA#1rhu%zHRrCOA`6E=liqrwU0 zX>;S&^-dHRGk_EP6lPi|`*Y6-ICWu2KY?3{=VUD6LV=;7a*PhLq63M55kaSB&fssW zvx)MiTESzD5|~P_N@|Jqh$2@sCs*`nG`}A%nmr+YvJ_(h!%I0&i*(G3)JCy2U`Hg7x`Za$*u2nXVMLbC#70T6-O>@J!IX(9eqADj_ZAZ8 z?%Bn}0*&=oYyyuB6G|%aRu-aQiKVM-wZAG)wWMXWY6~Q?!Ln*vUAIVCmgAEYbgzQc z$9%Ijk&ggqo70^$#vHu=xAX^acE5HR?auswPA;LqO}ti5`gEsEtQZ6eeVCgwSs!@N zrjGp;Tafs7J~5j^6@KM33y}Sn^eDBZLXqrzXy(a5`!Q1A_yzrvc}TF;xWLV_- z+)4qTBQR4d?vLNfl%_t375yhLJG;#7UVzQ_$9z42?AO5p2jF7S-Q*X-HzA{gclj8v zeylH!gQle@gDpBV#^7PR;1eZxWIEOeHh8wJZxDyJxAfG)E>Z?Z%+b!bWyDHe^@&Qk zbWQg80cw(ozQ`R_f)uCxdwBd4FQdRbQIY@NmLo*Y!qHuu+2evAw}5D6_coi!kQ8Fr za?Z~n=qIOfMYFw@NwF^#-_{c`iSu8-8kM2khQ-9hfXMFcUq2o-hL5t90-$3l{I0Q` z!uGj}wd|A*`^(p%tC6j0Wwb-6r{&We69H0*WEYWRXc`xh78x>w2!i$}-Bv_d8!vkA z_Eisf=#fWD^RRd#>Ax$msZA~pIqr}D@GKgGzv>?VY&6IkaI7iZzIfWVO7^*{$M|{R zIgi|703^X#R2YoD4hg#Dg|0OKr-OsKH_$r^XRboc;lEwkCt9Z_xXb&RUCu88%Q6Q- zr-4v7KhjulYuibyRwi3mfohLp!!%sk)H<=G22x4HFqmxm5bh_dN=>$SB+^I;qmRHk z_4BZ`cDUN5+^pCz%MaOLsEh0~vrl+6QdBJWBx15(VX;gUHuNd9{7rx$yYa-6aT=SD zg(QXebe>bn^VDkJYenAOzu|fZxTaY$E}EBuck0S1T@MI|u8%@%_jXpXmoi+qGsZ~} zu_+gk(+=)ac$Pn{Nn(-k%L{(%Wy z1fO|9^qIe3*Ty>#{~&g-bqN36Y7wS_9KH@ajZ5GY?BgH;*B*=JK1&6n2CLm2C&9;0 z%Mrf@_vq((k*~TjA@6BrzXNG0t{!YJF3Bjh>ihkc^PwKK?C$0YB?pj^V%jS?1fkE3 z%1x?%5VVC}8Iw(KDqt=y$d%_>{8^1A25Ys^99y{f#u|ZlsgyxC9apK-Ge+hSCD%La zNSiPbJj%(+m9-%O=QbT&F^2$cTeQw-q@tcBKBv)64bfi|ZQix82MuxpQt^AAW7q5!Uj5x5PJ<=0?N_Yr^`&v(^NUhFX{o?Seo6g1$ zTI9vCcsAZ4~LjqW3;^08u?lg#+3EcBF$%mO3-yK*vrQUv29fv)FB>oGdnh?gSvE)92^t_ z(Jhpwl$#Jx{`ZAZ&b7 z=|AkZbkoFzQ>5J9Nsw#DUsX28@I6$+gTif*DA$iu^EYZEq_R@f+^})qn>8e1lrJ{q-K887yiAB;qhDCJ!Bdf zn}m3Lh=T|+Oj4RTw8X-)1j7S5tyWsY%64dhUOG5AolbMMo|uw6ntMjD@kNG&kbs$= zLFa{P)eYR?Btc-)U1YG;56IS~eHe};BAFg$6P_quH^QS%ye!IN~k0|@837Fge>4!R3pZoJW8uilN+JJeWtHaUC!+qbYmD zL$Iyx5wCrKW?!y-#N-@Yjhj1Zz6M>ML|O*GGPCEDZFdhpsQ6j?-6b&c(3V)(z?Inb zoXr6tSM1hkX!S`1sFj-OUxQ9Id zKEZp`pef1dqvPPg2fH_yc)SmMej^FEuG?tQOGfO%EZyXd9KS;A`!G)_GMilC}y0PZ(IC-~MsI0OzR1O}C2TogF0z)`a~GYk5&{6j4t@R*CXpf3 z<*PMVjwSK;D$!gp&FUYT)r<4*+X4#fWbZj|d#`N8>)^q>gy+*aR!z8L8+d|N{GtF^f(PYvrKr20UEp_;S}l5X&7ih0l^FOhbY ztlupffK)g;GSjRaV@YWO3!s+OBK-y)N#&2TklvV@Fh3fEiGDFmqn73_vysB}2li(| zNcSK)6irrlqmOTYX7Ar5dc7Dn(!Yh_ygTAcSh z_sOX^1f1IMA^;ArXn=8??O{W;6OOSesnNN*gYbN?6fpG1y~;RGC!|jG@WuUjdqg+n zB@zj5ThoAM6}<%wo@tbn8%3|NNU$=4^naWsglU6GSbF#nTqlhgU?Td_ri$u2=>ku) z59K=Kpz8+Pbs>NnLCxbs8XD=ULd6mW#Gg@MA*}pXJYW~QNBpFqp7v0Hfo|OO7j1TK ziaOX!M^utYoRq}=%xjXWz6IPV+iSFnq=fcab?I$;jI9@32fg-jdA%zVci`J3_9*i)f(9Lje!rM-lD^99R zy>%-}91DtiD0m%T01{e}VwK*H3h~LT?Udg4urm`_a>)RM<;jJUsm{BPt@~W`Rwm)d z9DKfJLU5f!^_B5}VfEIck-#*Dug)_+KbVm#I}ds__T9Ro*LY5jyJ?+3IAP}-oC0MW z`Ut5Yqa1%RxQCCM82+F=oBvcx2HB4!GPntlW1OwLF81&rMp;Z;4Yi{h#*z_?od;_s zvN5iHA3bI9$4-cQD18#wgqMY)4Q*3$&J4{qr15J#(A9V%^2z4c(<}KE8ab%s-oY)SyZHxV$+SW^ zuWop47|PYba{8d|mzHA0vQmYe!*Uy_0XZL}s#k3DPNSe6)NkyS&bM?N!1^BIfyIc( zzuV}+<{7p?`JIO>BhCHP1rAe&@%qXr8#|Sh4 ze~4D>DpBJlfEB}6zZb57s?>k zmm!`~byVf3%+WwwT|LjB{<3>L2*tlIgjUCr#y-E19}+pWX@S3ZCp5}+rYlv0B(`ha zCR-_CSvv)1ryQJ()b!Ir807oISS&k`2m(sg`_oa#cRHc%;4HxGlppC7}` z(*{DtiY{}?MKCBjOsnNAul+itqgwZe6?GL^y0UNu(1A7_SnN>1@KOr7Cxpl!tH9Rx*3E?td%TWa{` zG^c9yXjvNY0}-M!$&B>E)J=m^jRc9hGl`l@`N>7PmB)=yMI`z=l1p5SR1a~OIe|Wz zmDFFUsr}wNu;GvB895E%4?>e!}4TQ8NKi*Apn36CnK zvld$S@9NXAJKau+j+lbJNG%pz2VRO0K7XR|VI(reiz8Cxfv?)73bG~;J+g#qRZ{6> zIu{E?6A-1h!=VUF`)f2FsEW_4@ooDfjv#lbl4HQT1@UeQ~abyg=m5p724u|jeS=d-HY+`VJJwsC`N(JMfiLB;wc$T(qf zuPfsd@sb~WC)z#->azL=1>xqBTeSBDx!$cmeFO-B_{14w>_?mG6;2+=PCRK==02XU zpd-;}r=RPrybrKbts7M5wExs-A*_A&a?;>9?I+%dBT!BlBYL5(oc+98z;gK-x=YAB z>tgGc&5M401nP0GjzhREZtElqv$S`&(p3$}W1SE!)&t}NZxfHEGO})&&uI=5GMnCTiBDX5@#nE|Lax2utc@L1{+FZ zluyLj36Z^%jXT^CH!o`nb4OwEmZ#Rd7%flFaR7p8$;kq3@&|FcX74|S#uJL&=blKNyk~f1&njJ8kq%cXq z`&{0>9vBwZF_O2B%rnyIya*gKZ(-f>-_b0BVTwyBFo$~bN|&v*YJTmFRJoy`g`1e0 zT#%|iBdw`hFA5ueZQsMzZoSj$UTptyK%9o_U_Le}Iw$cFp!n;6vAJ!h-H(%Yz0p=X zW3;#b%wr&tH?Esfj*1%O(XcX1L`9lI?-rqE7N6DY+zix9N z&221Rb95$H+i>Xf{EiAgQ7xTW0G2h-C#Zo@-GluCLKi-6)|bO?5mLb{T&-7Zo{nh4hz? zYzju954U6Z2xK_CYLz$-gh!G4`mQ@D($Y4`k{)%%$TAi(!z+6kc|me%?)roDrGo{4 z*o3>k_91!+!^S6k7Z#N3EE=gBP8?|HvUUbws@0hf5^_Z?S2!{F6$)46_U5GcaZE=K zK%DWqKf~KP%{~4eFs9@@q6Zdf-|mudTjAt1u*~ebNS-zp?LJiFDew@2RRI#;l+mbz zmyh=(F$q|}h*GxFLHDdsqbG9JACBmEbAn+nuMJUJ2>lpzDAkXU05ChJ;W+nqHcjNf zohtRu37^?!#1dr%t7?oVoc|NYmuW5fMCze)x4__>j|<&;6Me^y zEo^?=`D|1`hk|KGAUce3EpkO z<&DgnDCwilaMW)I?D4Oo-l%h_YGmfl{qH|xUCJrVf?}`4NYw&7pG2l2;Nr!O-8hyL zO0FT|GHJ)wTKAvz#&@ibp;846cIN5$)o%3ST1wn4KzJ>Mbt{!Lp~B|()*cx7bz6ua zD{ARV-DShOBdLv3QOcqung|$|-vXj0H=Fr2Tb~ZZ?Q}Gmi>?Kh?kW&|0GI;s|DF;C~H+=Kke5=h}JOPb7CBm4hs$!zi*Lo>FK&?to3nL-zdmNEmI&<2Y zNoB7F5Wo_x>C^_o6#D(sxWiVcwsJ;WudX zH1I>Xx>L|M(Jr_pF#w;AFOU(zKQoYs4xw`5n z3DT3Otd9VgdtLBepOFqN`^Mjgdv~h9P^k|yWqQt(DCS5=k?B8-<1u zes*aT=93M6VL@u_XK9RmHlhS7+gM5F>1v;o2wpddkJ$SpST5nAa|aI}jh_-SR*mTb z#F1EC<>Rz{_N{tQjpwHpGv$kzAEn>w!6ov2KK^YmU0UX4!tLOfq? zWe{b8o1SqIh{#5E{#YD6>S&E%Uya?uhdAe=;K$NU)TNst+7G-*9{O~_(SKv!5C};8 zZ+MncQdMpS5D*7XOi3({Zg0$hHkHeyns z3B29bF1B}oUQu74MUJpc+*Onk9GDNcG7QuCt34JY#);rEUe&m+Z=!!*9jz)ArfE0i zlTNz6tcjeumo%x4xUEmG_}0c}gkM17_;OHInOrqTglIwC0tljq@(#skq+c7%G^>P@ zEGdYSgw-|r7%qVzpHLg!r0q$~apRo;w+F4}%N5IuJ~>z4*1xl;I9A7DGUUz-f=(pI z>SM)?XQM`3B4yJSj2Vg6rgcsE`cQObBMrHtJT$b^0jtuNKnMwXY1R=F8_ouV@NpO* zJaJBCM%t+#T&v!8LmWSFUJgmiY>pP$Qqw5MW#|qGWx?1ex;J|@FTCW*x3jSH#yWP| z;Ig)3#QDZwTimw=RjqYDZ)zJ7Y!v*ah2Xsyv=f9;rw<35)TOStZ0%t)5);)6js7%o zP?JBBfH6d6(U(w`<0}&+aQ;0hb?E&B=zXzko8S(kqwtqfd3V~Ez@18H4?fZ1tddhWuJ94_RPH!qDZ@!&}u<7~>;twy80GXdX5&sEUa zFQ1ALxZ}K{qkLspS${}K4yJudAvv{qL&~`6!^9aTr8c&_mz)qTxPip$iWGZ4bS6b) z9rW-wgSFY|^IR=!T0KH!TgUh8w2Ffw;Ios}W}{h}iC+#uaEuR}j``0I&3*?w4Cvq`?XmPBB43$}5o)BaJG4^66&M#w3qBX!l#oUSnse z`vmxo4$JN$XO)?>%(OU7?IEAzT|F!feryl6T+bPZ7nEH-ovH={p|2a19{vG3`w|hs z3Du-aENoeVjEe*X;?%g9XgEF@_|}$UrT)1zb~5L6d?9y*12jT{7W)3!19g+(TGY~w zmXD-mgpxz(_~rtjo+dZFH=~P~Q?2;6NybF~2*cx^2!NjoB1p9=aodEY{0}qcMD!?k zW>NXnsIR!{&b5@{Z?c%+73PJToWJr<1=>C@E?`jr*LDe{-U+CCJqq0fUS6s|wLBDl zAxPRD7^{Q*nf-!?=W&)0@r0DoT>hr>-Lx{~sxvtvyy4gVV00aSkGJ6Z-4vDVxp3PjDeun)f}(M} z;?M!v-x@MFOVqTA8vB&rGD>1*UWZwkrCsW3&@IgRT7}w-`WbmzQfyD@jK)VHI%&~a zS$Zj3RgiZnu5Um)8p}_tZx@yvMuGzpASfa7lx8_NlrpY^r*a%MOCTBK*>njx{S(1P zPQ)H(@=ho+;CNAw$&q#nHRgu~$tk&Efs*!(k`F@glCg#b4A45S4x%b7qerHFQb1k= za2c#j4+5H;hsTBNlD;ZX*m6`d<;SN|6mm_zjVMoK$-N5ohwa)K7^|$t;w40WiZbL-Yu{H8^qRbr_)$Slk zdNhA$Ac;9-8O}es{{mkno1+DvRZUVkViI1j1OFuPp&++g_Cd2ep-dpv!O{}o$Ls=7 zrGlYGe3$gI))N_$d~ZZ!1wP2xv@cm}Mj?GZ79}IECYT}Pi<%eB8@G=`t_o2fMxjW&SV6*2$wSirI2XPSB13(o0K-Ttn(W#2oRuBUr|0FJ-y6CDc2hyVUpXw?>R z85p+t(Et?ySgNk@)Nw}Je04LRJKhub>Naf*a6C2C-lW=L7<0$^bw?X*S17gG9=k-# z@vGxp`n=Y<@JWkDr%&Er9%6sm0rA@byQn1JVpC+FTt|r5k{rj4 z#bWCL)D}a8U>$h3Wob6FU}q;E1eB&nvlSy4nM0gH^$+K@WV=~HDf8=ygc(fZ3|{rtt}D^!b$l>?;N zk9W%JI@a|T4P;@eZ2_~5)InXu*wu35jC+CYw}y}t^X*F{iO46`@D(pd$D-Fn$Xvqn z$#wIF#-vm>BP1Kf8PkD_M?#lbBQpGe8dhewl`D$*hX?9`Z$@Ji7(J~hM zNy(!Ti&Wa|2m87i&Kp%)QPQsSqnd0b$gUi?FM~8I1odx!e|+FuX9m9op|x7wcgxm~ z>A?5}?YBblcyEw&_enxz)y-)6@f0DCFzbHcu!>@d^s-TJmfSKx-3SPzbobeNB7u2N z$MUn^SWxVUR+bUOyh!9`;cc+={;vQ$2E+M~KXhjA@r8u#Am6gkwW9%8HS<@O4?5$` zfx4%3oX8^Q92wGPFAA&5Eiw+o_I8dVwc!1)!|?8caJ^R^%MRuEzqVt|``4cx+O{}n z^Ccyy1*P4|ICDSGP)zS-8@t#q@OH>ebqy-2-{E;EF~wR0)SCF@Ch#ZTkQ=uXsEcvG zUMODoBApsV>cPxj15XzgMC;`9BxiR?CjWw0n%FJ4h~jGAbSctQ=AcShV!!K4eVZ33+FkR<}^a zw>7J0UG?++CqMr`d^H)BcBjPJQffoYFHAwT~KE@Ms%vl=e7F>wN)&^3Qm>;}~bB2^YJFh2;plPGkQNl-p@xO5mrmI?0`IWUcwW^gfR&4m}?(=&b zbjKSg*6KgS_#oT(_WB<;es=dOxjwf;&bUO5cDa>Fe!BoR8o%Fw+mvR|$xy}V>~gp!F7~F-(JBmp*5v2uM zn(Nx0gF~G!faLNEMDgo!Bjq}X2c+6*@lkK^GAQ$D;L`R?wyx{C$`aIeqAFn1C%4>R z)OGaUp7IgridX=C3ZfFzTp-QhlJ)OjeX)#I<>XlV1Z<@Erm26CBAF@<219qTWAUXB;S@EDwu# zbQx1iDv9hen&4n+vXlcGXE8VgZ}dRCmBEk{Y|@HGrr%X~%`bx{e0bTR1)m+oH=@z!j@NnbPn5tQTaLo{dFW9I?m=k_y(xV(cVZL z2uwXCRW)Vck7svDCQ-DcU4_2|@N*Lx#c*89kmxMLAuhr}QmnYCa=*n7pl~%KD(ijY zbxK#oRZ8XhFw&+#mjv`m9Net1OPj@`7NuJ*BRhg=zyRHppzdx$TF8amcWajM}? z*<}*DH6)8vb)-+Jy|N8I+nV>Uf2d~d?7!=tGWf9=<}_CU<+uNI<=YotMA3RX zqvhq!eEZQg%W4q3s5-^ zw^r2;_4T7#Q<6Fl^nCkCx3aP_7(94zBd{Y#*8cS`KJA-VO8cL1q`RcoC}5hgyP+Mq z_z%@XP Date: Wed, 13 Sep 2017 11:20:52 -0400 Subject: [PATCH 23/92] docs: harmonize headers for tools docs --- docs/abci-cli.rst | 4 +-- docs/mintnet-kubernetes.rst | 64 ++++++++++++++------------------- docs/terraform-digitalocean.rst | 20 ++++------- docs/using-ansible.rst | 30 +++++++--------- docs/using-docker.rst | 16 +++------ 5 files changed, 50 insertions(+), 84 deletions(-) diff --git a/docs/abci-cli.rst b/docs/abci-cli.rst index 371ec0cba..05d56b919 100644 --- a/docs/abci-cli.rst +++ b/docs/abci-cli.rst @@ -1,5 +1,5 @@ -Using the abci-cli -================== +Using ABCI +========== To facilitate testing and debugging of ABCI servers and simple apps, we built a CLI, the ``abci-cli``, for sending ABCI messages from the diff --git a/docs/mintnet-kubernetes.rst b/docs/mintnet-kubernetes.rst index b7e886658..fa4c62661 100644 --- a/docs/mintnet-kubernetes.rst +++ b/docs/mintnet-kubernetes.rst @@ -1,28 +1,24 @@ -Tendermint network powered by Kubernetes -======================================== +Using Kubernetes +================ .. figure:: assets/t_plus_k.png :alt: Tendermint plus Kubernetes Tendermint plus Kubernetes -- `QuickStart (MacOS) <#quickstart-macos>`__ -- `QuickStart (Linux) <#quickstart-linux>`__ -- `Usage <#usage>`__ -- `Security <#security>`__ -- `Fault tolerance <#fault-tolerance>`__ -- `Starting process <#starting-process>`__ - This should primarily be used for testing purposes or for tightly-defined chains operated by a single stakeholder (see `the security precautions <#security>`__). If your desire is to launch an application with many stakeholders, consider using our set of Ansible scripts. -QuickStart (MacOS) ------------------- +Quick Start +----------- + +For either platform, see the `requirements `__ -`Requirements `__ +MacOS +^^^^^ :: @@ -32,10 +28,8 @@ QuickStart (MacOS) git clone https://github.com/tendermint/tools.git && cd tools/mintnet-kubernetes/examples/basecoin && make create -QuickStart (Linux) ------------------- - -`Requirements `__ +Linux +^^^^^ :: @@ -45,23 +39,22 @@ QuickStart (Linux) git clone https://github.com/tendermint/tools.git && cd tools/mintnet-kubernetes/examples/basecoin && make create -Verify everything works -~~~~~~~~~~~~~~~~~~~~~~~ +Verify it worked +~~~~~~~~~~~~~~~~ **Using a shell:** -1. wait until all the pods are ``Running``. +First wait until all the pods are ``Running``: ``kubectl get pods -w -o wide -L tm`` -2. query the Tendermint app logs from the first pod. +then query the Tendermint app logs from the first pod: ``kubectl logs -c tm -f tm-0`` -3. use `Rest API `__ to fetch - the status of the second pod's Tendermint app. Note we are using - ``kubectl exec`` because pods are not exposed (and should not be) to - the outer network. +finally, use `Rest API `__ to fetch the status of the second pod's Tendermint app. +Note we are using ``kubectl exec`` because pods are not exposed (and should not be) to the +outer network: ``kubectl exec -c tm tm-0 -- curl -s http://tm-1.basecoin:46657/status | json_pp`` @@ -81,8 +74,8 @@ Clean up Usage ----- -(1/4) Setup a Kubernetes cluster -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Setup a Kubernetes cluster +^^^^^^^^^^^^^^^^^^^^^^^^^^ - locally using `Minikube `__ - on GCE with a single click in the web UI @@ -102,8 +95,8 @@ Ocean `__. **Make sure you have Kubernetes >= 1.5, because you will be using StatefulSets, which is a beta feature in 1.5.** -(2/4) Create a configuration file -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Create a configuration file +^^^^^^^^^^^^^^^^^^^^^^^^^^^ Download a template: @@ -123,17 +116,17 @@ update ``validators`` set in ConfigMap. You will be able to scale the cluster up or down later, but new pods (nodes) won't become validators automatically. -(3/4) Deploy your application -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Deploy your application +^^^^^^^^^^^^^^^^^^^^^^^ :: kubectl create -f ./app.yaml -(4/4) Observe your cluster -~~~~~~~~~~~~~~~~~~~~~~~~~~ +Observe your cluster +^^^^^^^^^^^^^^^^^^^^ -**web UI** <-> https://github.com/kubernetes/dashboard +`web UI `__ The easiest way to access Dashboard is to use kubectl. Run the following command in your desktop environment: @@ -246,8 +239,3 @@ containers, creating public-private key pair for each pod. Every ``tm`` container then asks other pods for their public keys, which are served with nginx (``pub-key`` container). When ``tm`` container have all the keys, it forms a genesis file and starts Tendermint process. - -TODO ----- - -- [ ] run tendermint from tmuser ``securityContext: fsGroup: 999`` diff --git a/docs/terraform-digitalocean.rst b/docs/terraform-digitalocean.rst index 399a0a76d..95af507f0 100644 --- a/docs/terraform-digitalocean.rst +++ b/docs/terraform-digitalocean.rst @@ -1,11 +1,13 @@ -Terraform for Digital Ocean -=========================== +Using Terraform +=============== This is a generic `Terraform `__ -configuration that sets up DigitalOcean droplets. +configuration that sets up DigitalOcean droplets. See the +`terraform-digitalocean `__ +for the required files. Prerequisites -============= +------------- - Install `HashiCorp Terraform `__ on a linux machine. @@ -27,9 +29,6 @@ Prerequisites DO_API_TOKEN="" curl -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $DO_API_TOKEN" "https://api.digitalocean.com/v2/account/keys" -How to run -========== - Initialization -------------- @@ -110,10 +109,3 @@ define the testnet name in the firewall rule. It is not necessary to add the nodes one-by-one. Also, the firewall rule "remembers" the testnet name tag so if you change the servers but keep the name, the firewall rules will still apply. - -What's next -=========== - -After setting up the nodes, head over to the `ansible -folder `__ to -set up tendermint and basecoin. diff --git a/docs/using-ansible.rst b/docs/using-ansible.rst index b10f8cb19..33e31c3ba 100644 --- a/docs/using-ansible.rst +++ b/docs/using-ansible.rst @@ -1,17 +1,13 @@ -Ansible playbook for Tendermint applications -============================================ +Using Ansible +============= .. figure:: assets/a_plus_t.png :alt: Ansible plus Tendermint Ansible plus Tendermint -- `Prerequisites <#Prerequisites>`__ -- `Ansible setup <#Ansible%20setup>`__ -- `Running the playbook <#Running%20the%20playbook>`__ - -The playbooks in this folder run `ansible `__ -roles which: +The playbooks in `our ansible directory `__ +run ansible `roles `__ which: - install and configure basecoin or ethermint - start/stop basecoin or ethermint and reset their configuration @@ -25,19 +21,18 @@ Prerequisites Optional for DigitalOcean droplets: \* DigitalOcean API Token \* python dopy package -Head over to the `Terraform -folder `__ -for a description on how to get a DigitalOcean API Token. +For a description on how to get a DigitalOcean API Token, see the explanation +in the `using terraform tutorial `__. Optional for Amazon AWS instances: \* Amazon AWS API access key ID and secret access key. The cloud inventory scripts come from the ansible team at their `GitHub `__ page. You can get the -latest version from the contrib/inventory folder. +latest version from the ``contrib/inventory`` folder. -Ansible setup -------------- +Setup +----- Ansible requires a "command machine" or "local machine" or "orchestrator machine" to run on. This can be your laptop or any machine that can run @@ -61,8 +56,7 @@ CentOS/RedHat: sudo yum install epel-release sudo yum install ansible -Mac OSX: If you have (Homebrew)[https://brew.sh] installed, then it's -simply +Mac OSX: If you have `Homebrew `__ installed, then it's: :: @@ -143,7 +137,7 @@ Mac OSX: sudo pip install boto Refreshing the DigitalOcean inventory -------------------------------------- +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If you just finished creating droplets, the local DigitalOcean inventory cache is not up-to-date. To refresh it, run: @@ -154,7 +148,7 @@ cache is not up-to-date. To refresh it, run: python -u inventory/digital_ocean.py --refresh-cache 1> /dev/null Refreshing the Amazon AWS inventory ------------------------------------ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If you just finished creating Amazon AWS EC2 instances, the local AWS inventory cache is not up-to-date. To refresh it, run: diff --git a/docs/using-docker.rst b/docs/using-docker.rst index 85a302611..b135c0072 100644 --- a/docs/using-docker.rst +++ b/docs/using-docker.rst @@ -1,18 +1,10 @@ -Docker container description for Tendermint applications -======================================================== +Using Docker +============ -- Overview (#Overview) -- Tendermint (#Tendermint) -- Basecoin (#Basecoin) -- Ethermint (#Ethermint) - -Overview --------- - -This folder contains Docker container descriptions. Using this folder +`This folder `__ contains Docker container descriptions. Using this folder you can build your own Docker images with the tendermint application. -It is assumed that you set up docker already. +It is assumed that you have already setup docker. If you don't want to build the images yourself, you should be able to download them from Docker Hub. From 921a2b41f0886312b0a1f2729c8ce469d8c1ccc0 Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Wed, 13 Sep 2017 14:08:21 -0400 Subject: [PATCH 24/92] docs: add kubes docs to mintnet doc, from tools --- docs/assets/gce1.png | Bin 0 -> 13143 bytes docs/assets/gce2.png | Bin 0 -> 31246 bytes docs/mintnet-kubernetes.rst | 60 ++++++++++++++++++++++++++++++++---- 3 files changed, 54 insertions(+), 6 deletions(-) create mode 100644 docs/assets/gce1.png create mode 100644 docs/assets/gce2.png diff --git a/docs/assets/gce1.png b/docs/assets/gce1.png new file mode 100644 index 0000000000000000000000000000000000000000..3bf3ad005bfb3b13f80cdee45c6d633f8d08c480 GIT binary patch literal 13143 zcmZvj2QXY=`}UC~t0b0<7QOdaM2Q}qY(fwvI?;(1tj=1!3$c2XL`3gI%dVb62%`5O zdhg{sdFTDl$UEQ6&e|Dg&w1*7U)S$`Hd6D6$}OV1L^wD&x71V>wQz85z=6*uU;^MB z;*k6a2ZsqqP4S@)!ek?hFf(@es=Muw-J$2My!t!U18r_L_D4@^oE3|}-27U}c}eV6 zgt`7ut(xcZJSp!S$Gpbs*R3zhj|POV*7{n6kNPIl8oid>O#}oc&nEXWm;LLumi^k6 zrN-tL78V|>ffVKbeKC;*JCeigDOi~B{&_*rL7>;q!@vQ1?ga_Q= z#y?*N?gnAN^+sbC-W4nT`^YK?F5!DVoiQIBG6?RqSMWAi6&Bk4NH6-|N4{VNZg23R z>fpowJYkM3!Hg(AwT$xmU2w^I~VRa63U>W_^~7nWbxOC)$nNTF=1TsP z48K5Bftk@STjRA~1CHlza=x(tRp#WeHuAV3V7?<#x7PmIM5BA*o>pQZr)TTcr7vf* z%=xy$a_6U|bcYhZlU>jAW7o6Q;&R8{BzDg+tJJaIsirT-s%^Zy-AqvvkHw}IFLQJ* zO_)c@jMsL7zGY0gwBOQw52rbzK)L{<_(nD`AVbhHOnRpSimPz@>hjDvzvSJcR8Kr1 zjclpSqgG^OLZ{b^|F7pYw&tBt6z(&?C0lh?Unw013nI^C`$Zf^_^7(P4us9E+os(6r3snCjff|--l-cD>z494Pd5|x8y9J@cdyI@Ue%72 z8aDMe&QO{9nJYz8)~=N`^%33SO1R{I>~-h*LJX#6gS;`rHV9}HM6{gmem(r5K$^AR zAbWLQ4HX5)AFTZ{%#!fPTxpm^5<8wB?~LRs+~f>WO*Qc;qL+Bp<8KNqzq8%10#ir* zvRBQq)PmT^$~0k{=NY0-CB{vjx~0!WnhwVuoVw%aJ%^v^jRBor?!XH?i%wqLT|O() zE|f?_A>`mpbr2Ps9RE7$lf~DE)4p4ceq;g><&%x>Ir}9!0jEV%uHE#TM-eO_68W_l z;XZLS81vmn$z0~{%Y7cmB$Qa3jly;UBuOQXu98@w`3J}2^cUaLMcn5?!L{XWm*cMU zW2XKG&p5yM9Bn$?ARuvJi4oFfthVX*v_h@Df1HsS#F7(mar!_Rf!COA^II(N@}v-Z zZc$PFvFG|m`lG?bhpBn>N2|WvcDmV{Fgrm`Nm4PJD!hKS#6}~nYhj; zZK=QEvx9QLRk=L#m1F5x%b3YlRq1{Dvo9Y%`JeSM$_%Uzmi^lcwB{ig2ClhK`5~s4 zQV%g6&pzt7V)J~DHv1}owb!_pmjKZMhlkuG{ZvtH=8Fbq-spuP3T7 zRKDRil!(tx`X?ns$k~Fho8!5+4hw0cNG^LaFsoCpt6sIg zGo%fNaI^1YZ>IcDx=Z5tJLbGn@bjI+lkKtH-(QWqR8a8hveCK7>C;W8xJbdl9L*|-7o+YvK#Wj)m9z&V zBZ!bo1Hu!wayg#)=>dp2pXp#hC zIv8Mw@ysRXN6!-L+aSMH-K5-r&{7Hggus>o6F}9qY#6x8c73m?LC`W&B_`6$U-Cd$ zN;pCsB5m0fLEUj11%+F>Y=by)-S8OtJ98n3An{wIxM82?XbRzJ z^R3JHnCLK34<}2#chaALU$_|<2qJ9=@#wEGZQN|ZR>x!PxuHnmuAr7Q(Nma zF|v{MB~>6tvUls^Xr}(BhO9KO?oP-e!6LWY%=#&ik;}_(iGi82LsVF7-MJ|QArIFh z#DFAG#E#np%+s@BOzf^w;_`_XT5BVv?%|J_7$)kSw>qHxNF3~qZbF}65L94K3FT^b zv!6&$o`&qjKiyS>$Z=g9rBix6+gq~ESIOv&`=a%oLkZc-+98vT#3#;vr5P+B-7Ffx z^d#t!8cX1zlJ}P3ypA{uU!XTuQNDp`MJs&YIA51ZCh*e0?Zl6{*XNNoN0OK+1hhS0%9EHF@Pgl!+04Df4O zl_?KIaQpt+AiL`u#9Q#dPjd)aCp~|f8cgZY;}3M#7tnx)85RXKM_itzi-Yh6I&vZE zX;DES?e`y_NXJ|YU2^HrSr|Bu+O+9#(QwY3MJj$PS?CrZPT|@qK9K93e#QE^l>;>p zLlfz&wqjEFrX9tY<9X2W%da+af`McG=~UI$grU; z>Q;4Ju3z4wK>~1{_A{Oj|6E4~3?J(C zRV&A#@J^DUiD&Vh0}u-j`to$Gyn^Ald9ROn&BvJU+VC*p2`UApPL32)9mBwi&>3ze zk>Cj8_nJzOd7dGM{Ped(oc~;{MrLLT#r9mmzyS(~?Asv`tB0(ZJHx~I*hslS5G9|! z89vdSiZ3a=#Lq~`(SR&86z9<`(I9tt{r0(ZapFH>t>l_CRR8!PJ zT90rB6cHvBsgo?3fT5Cu6TVN$xNUfpNW0O>(OGIb(9F{?JCwGOEsThmj3sIXqnbdUS1raTv7#XbrB0)&J8 zNQvFh(*`?0x7OJXurw_uJ>mm|)ZC?1?;7_EAWxj7eGg-7TWJyvfXJ}{$WM(|Yu${2 zlbfH`%7T=Tyz+JOp+stwH0>^cxc0m__+`_Pue2fEDP>>G@f2gbRzKyLA-(ra*mLi@ zSko2|a4i7kUr_lLYCkU)ODk5FuM+>fFY|%Ig+WI-?S0#X$7mQPwqBhj!TSI9(&2yi zl3ke0JDTofv**o+2~$%xwG{m5_O64E7NNT~_2tW(wKmF5H8Ww)9&z8exJQE*cy&=< zI@f}C^{y{V(h!5hcRE}jqmihz3XPHsIByWK{qY&l)5|1Dk26FK3ZG=H$ID*2AlXEa zcBlJ)HjluZtHMcy3PrX9dAhBBC#u#N%--L#B!59l5gaByf7s7}1vEu;Y+#c`p#vE% z*6X(){%BR+a+nU%CbED0O&fQD^vxTYE{Lc$``*3q=BJz312$15OWY4!lj1hC8eTEcNR5pNSt_S3eK46J9 zfCX41<-H3kNtN+G6;}W!YBqT~In1>B%Y>W?YHf_mpZa2T$ zY+N1(vG zu}6oxB+DA0u3YA806TSfzSrO67vhZugr+s1cJdLe=CvK)^fJlbZ;^KQdyEn*K~kLx&_S|QT}O_XIZeLDJb^}P_7bXO7AuQ@~kM4*V|9u@u>N5gx0ab&v8%_!=qj|}IA*v!}j&IaF{ zM3O9w0b!}s>gyZPfO7<|LDdV@1?p^{E7!zIhvV(JE(>z^=ju!Kw#AGooL>lCs6i?* z<;K2l4|iZ>+1+01PaiUI;A?M*T5U86(mL6u64iMeRf0%==81%toPD@;I5}nII5on0 zxY}*#s6O-2a~;tv;&<-c^huZ5!Q1NT+Cd+OqzhNyX$`MK8{f&VuKr1@nFGnMBhHc* z^=%bjB
  • *rS)KUbg?5=%DJa(NYQtzsYA9*u${WHgeZUe>C9OKmOL%b4;Ivw6l)R z%L3v`&yDBH!)D|*LYyrJBZe;r=bSl88Kiy01$yS%!?VEh5OYVSOqi(e!K#(hc#UhW z8TGA;(}U{y&gdpJU2Z|#2BricMnHdLEU%e=)Or^ME!n=D4Iw27!Mj1Tuvy(NQ~eiO z4AXlE+YfP_?f=x+0G5_0N%mb4d+~O5&6WhtgXp(&Gbnr?Zq4bfwyR4u`8a>VWJYtv z55qyYH;iP!o4n=CgOTit6a+jL0?%K!?J$&VFaq9=W&o^k%w(%y`d$XovOe&+!tx|^Vi8q`L9Jt9E@W!AGni)>?=;ctbVI^ znpl!zgDIo1sjUiGfk#tbPbwUzJ`G3uof0jK06T9}@wRdx;9eC@FMw_v8@^@K{ixGh z8|gLe-5r`YV6y$PB0|ozmPR@<$RLQI|G7c_Rto;*hnHC3eRtPR2jSio~2DLk39-bjDr^6Uz8Bw6y^qqEd%Di1dU$@%8fw9%Jz@TU5oZPXL;u7g=fAA_mK^kZNBroby#XJg$_uCNhTM}(UHjcyogF*s6&=bK)x8V%c26R!XD5YT1BE&2uJWBc9A~via(1Jp&t{!+u_KEDXk@Lw6dd3Nj<(@q7jql+ zGoHlY^(`Vc^fpW7VC;YzBNohdfA2@G!o7VvAc#E?H@CGka;MK5uaAQ*l6|0_0>7XX zS#dz{+^7vL>6?VYBOZ0JqonJSjDFYIn27NrH;?+>SPqHXID5oY@xPz~`g#41ESINN)& z!SxA*WxjDljLmuFM=tJzpU%ywpip9Bq5TJn?9umf<18^<_e?R#Nhkd2tIAf9f_SK9 zd`S|E&o~rCuWX(Tvq2r3xt=Bz8WQb&)*bh5Y50_;$Ka!ds#Dic)WcJ z7axijttkcinI0IY|8-e+yul>JX({K-X;E)tedM%|L+9&B%H9OvmdPv9(`8z)47R%1 zNOl3{^e~NaYtvR8_K;%Hs`_r)j*nbnFB{!{lM8W{L_d=y@g;vPqx^uk5P)-6mZM~w zNOR0$7-pSa5trFqofG<56-klJ*k2uj$pY;2awV}6ERmJj3x%Gw$_s!w`r$+8EHSI0 z6;)3mMo53VzV%d9?n=@N(ysF%c=+O5RCro02i?IBre@5$y z0Q!R-_N%xiZuxh>f9AVA?bU;&@EeZAYS$QMC)UO56YYPKf^ML9VglxP(c1<1^Ke9- z^Ctk}O|f8$%Uf4ByOGA?3;J(cXL-@stQlKIK&kQI(Nzq8@|fov?eBQVCPBjT457WZ zfTn;`JX)$_q}r@k2`};Lk2kw3>rj6ytznqsu*`y3m#!u* zJ4B)X5j=|8sic9tPQGV)SkH8W!;2(F4jD=KJ2#xct$2}ooB9Wsn}pV@lp%)OOjqA)H9*rCYqx;19 zoxp73!ILTy4?1E|&l95eX8uw|C;U9TYN1j5cs~nszyo`-BzTpmzrKz}D@TwY%{1b4 zOpqbroCAfU#yos1v-k~sd&;ERg7&D%?&-BDV>*z3TVN0eC2Zplh ze7>3&n{c-sIO&OgF#D2l_iIpwMT6_m$H4i~XECxd=DJS=InbkoPnkP_amLDV7T_zS zLu94&;BM-{gpi~V*BEnoF3N1Y7@MSW81tPle`V^P5`?xanB(@!XW9b*K8uHB4eOKK zEmMe?Ym|#HZvcHvEdXg(H2Z&5GVmkye@lY0vG~?@JRe<_iQsBTXFYWLFt_3=)Q8zIMM2DSeRK}R+NBS}EYgn$_aF)Jz#(+GF%k0#xQB=z;V{07 zwFsEhD!D7Xhl-e9A&f+jQWmc^)HXER{?c%;?jglIZmucI3vNrGsIz`F zL_8lJ!T=emA-=s2W#K3R5Q z>g0RhcgjYv&^=J*bTZ|io=U{mEMX6UR>@av)4NMgj2Y~990$@lp~SQ~f_Uy_XkfSsxBGsOe4g>1N&)m>_@Ce|bINsqY;>e5YiiJrN(b z{2oMbE-Z_ncu&)t6=_FGz1YeRlvTdne#wo4K>iAt}3wd51wsp;Ytjcxhdw3#*Ui7Mz(Wm_ z?{+)Tfa{+DWT+<(1X01w^UJ&$mET=k&TV`*45a?K&>EcMHmnC8vDQ{Q?&%p5q1yRvPe0n zh-H1&*7{^PTGK-n2EeN-vTKYB68wr8=JM7c^g!hXQ;*fnp+E(cHZ9=a+U-;M z&yr;E(ETKJk5%;dOd9skxKayMSf{RkQO!TMb3x*;5cZSjaVOumKp9Nb%?|Xct$Pypey=VAlzrWh1-s&~y#6Xo*d&}df z^8PHz`e!Bj1ILoOuR+=+Bbi^W7hDs#!RLTA1e7s1ug(X})$DLD0oO_KK$=E>u9G@n&r_{Ns*@E z(l#m`+allZWZU%-bV##U1z?K zV11?_W52!!%LNnJc@di_MCPPF0oTG>75-kB--Y*KwDP1Ml4dfMkCi z5ICSjWdPJ}Z!FOaBs9AyYKs5zVGl%n!!@kt?<1H_nd#$8k->fd(xPsjBBAf$n2oj( zcjEb6ozuj6JE?3!!fN<_h?4{Xc{~^$Mx6cZRl+bVTECrB7KRu1_pLeO_f4vt&&_mzVp`y*1 zp*0$tOmh+Yp$Y(EH1u14PqzXP(YRV6@jx8$9WF>>FXshUSYRRW)N!B;I^|kp^(C1Q zjk|LM?+85K34d0k)p@UZ8w>Khl+|zbYrFvJ$D5X?TED-fKu-q&KwxK*tOy{lsU?Ta z;#SJ%EumU^K<%p@Ff3WgbIgEcyqbj0`n#zZNSgsg{ZUn6tV*wa5Rt|L3REkGBs2)NrgrVFUQ?ks@u* zcFplm9>wOLfrbarX+1l>f9%>8aIp@cMR(=L;(#{Y7_DsYilMQ5)e2z9QGni1d9Wl0 zFaIrab{PY%C#%vGK#r0jhTQ&O#k||-NrKySwt7acp((t2eSfR}#l}r_f9vyIY`Xxl zn*fd@tcsqMnY4_T`aIoA;dxetdxJo~)_%C;X}&Tcsnm-eh}Zc#P=GUy}6AKyiJCq);8Wt;UzBQG4!4mUi1r|3Oj?#r5L=sZRd8wA^Ej?jok-s{rbg4XmRyoLIFq6WFaxx1LrL#5WF^=bv{A zf}e;36!ZenL3Q@SJYqZ9KC4A>ZRKX^58U4id+OtI_yg^Ji3O--&6%t+8E+3T%e98L zlyEH*;(_A!Uj*9p=Y7Vc2leXT(xGp#UbXf2$r1 z1E(eh&Pliz`>}Bdfii7A(*cJE0EgpT^2Ndov^TgdNdTJ6x{na36pGBnyP-ScTAj2K z^cUYP_0=@=>iNxm z4XKCOJKe-!yYiZ^HyIG0$7}2s#`z*I+tERq7MY*r6g9o=uqeVVe@35`+yv0iGUFyA zS@ZMRfHY=i+t~vEhD^|VO_J%r=J;O@cyUuw= zrs-(fcLP68fHX;q72OPzVnIrklSfaDF7PG2Q;DJ>)&6xvI^G(xbErFUZifwxypxON z*#xF)i3ELmu$IGy*1u(K@&0LoKJ)wBZCd$5h)#p*j!yGV7cEfq5d5`*Wbh zn}*a5%J(G*%**sj`3kiY92`-IB8cxn-V7tzv{KZ%JU^ii2bStEP&T*O7_W7ohq#75 zjTHRyW#{i{7$6h2NH%xgj@+dM)LZ6F_^g%GH@- zz4D-wcWRa-N!-OHC5tQsTaC-v`^~kR2aLf5t3j?Im!I|Ju?aSbD0~Xt?iru8($^$1 z9Cup{D=m~*FdZG;EFhEjKt+3>X1o(PK(whS4W)(JTC2fN)d)Gdo>MlwY}ybN6RlJY zg&aKQi>c@dI(|rpuMY4BeDW@$)w_q{HD2KB=$eQ+i(@a>%$v_l!OqE#%`_7_bV?sEiSW z#NQEpSc(>Y-H+i^ zv#0l(!wsE!z0=OycY0*4K|d|Q?-h1L&s%$lIdoef($wec#HULc&Ln8~r&fmd1t4gc zL~NMLZTSbl!ngTb(teyK)OqXBPkCNgSDuXKKA**=oeP}kzGDm*Ng#+U581(!K@~vd zgd`M1YVIF`1_mI0$y5tTD2O~rP|YVc;mUGo&cI>|7s?;`#Am0IN>bx4OON50*%sw5 zvsYRwB_!2Iq)JprZHc-YDF_1RzImgcGskZp!6GD22B&_+;G2EpDHUIvi=>@;IisX! z$sV6M^I-r?YK^JQ-V*b>=&I1H6YpuUegE`3H$14Tzri~X%06a$FA z3K?-W8W-zxRWv;_FF}~A1t}y+f%n_ezn>ZI6saCuC}<+kmN8W{2MJxR{=YK^BUGMLSPISPuIZ%2cZjkSz4nkui*aVrY zKFA*na6cf0nm9DK0hJmL6K6`371lI%b{mv%7|d1FX?)0AKwK__aQaA-PS=!7w&dgP zgIo?iEK>pXf&pSj3htN(#O(^hw`P_20~zU&52}Y0xcU=~j8B$>e{{0aItuV)Yp>!d z0EbkCB!mB+S1A&@M>e+*Ne&{9{zhliIWOyuYZ0~+;~M9%!)7jQp5sco#14Rn{;k%t zmp1tpKfRC4&%pBE-RX{_Sk~XwY{TBA=;zT}*<+GjR@!U5OFESjr#d=7(Up%^G6Q!h z;3eZ0Qq@$!i?m7IR}U_TLN#JIb8V<4E|HNb8h7t0$EJ?%1*v?|gW=Os3#r2fo(j^b zwlaI0JlN?QNF=nJJGl_s!KU6?y`-&=eAI0yZ?jKc7(!N-sDI7N zDBzdtufGVC-gN0;$QzP;ddoGEx)VoFoe^(t2=^kVMFkNFX3cN=X(M=s1#BbVIoqWgL`J9~>}vuZ zJ|H)->~f-X@{4u9jQ@N<0iR~z2>43fad9E7n-=O+ki+@_(;<&Ht*P;&8m0=?0%jOW zfv*tX$=l^dIHLM1c{f@auh$!`e{6ns!~e$F>&B4IS9#Ri;JHX8@i{@2{h^P~zK%U? z&1R$AKE55VS<(N%Xf?}g+7(HB7ULpy%H^ASEbO+Zy3p~c?t|F^OW$aB&$3$92vuA- zH-9I&M@|twl(a_@t?L|J{m^T)>s**ZJ69gxM&I&>@QK^o8@Jl^VCDs|Stw-~ zC-^Cf3fvyroA%?r6irDKGo3h1yfz#*WYEhdm{`3Fw=8(N@a2Wo9ai4w_)um$YBI#9 zZBS+2!*&}|C~pNK6~+h7nPrpJWj>cyd&M(asRv#NRZCdgPKUUkO}h6tHXAr`vJvFI zPBycsMfdf+oud)W4v&;(Mzlam*e;b(-i0KmjJ(;CK-}GZwDypSnTS>j;$nV9$-B_- zEoCrVPTXU5;}0vzuox&_R5h^hqFSi?_%nmDRC&z_5UGhD(9wk)MTLS_1fP;1|L$?7jmTC*KEX^_)-wP z`0h-!gTrFfXr#B}i+r*{9Kp#nF2PfOeCTR+1?;vcQyuh(GZvbw@PQgC5=-3<9#9Ig zVy7|hWD5@NuovFRM_4wHGMl1SE0QQnk&#_WoK`8p!w<8`g2Qdi2isi~Kvm=zb^e%h zXfYWz1M1HJZe&QOp$bxjg@BZ^h@L z7V%ftF}siAneQ&#&8K0FTgJF!%qi9Xs4!(p$Z-jJA_^_E)W>Cs`6HZTJVZ;XB)W4+ zdkHx}l*Q&HbQX(0ueN)td{pJ-yek(_89T-e-6iiCd$+e1ol(`xiLG1oG3^bffLm&+ zq^OI@Z)Nb?igS4p9Oa7pgc={Qf?jNaQ0nUL1TZD&gDjtN45e0vU%g1%q5?F>1rBT6 z4qC#@l-xbo>l1UfJ7pM})&FQyo+uvT2=>49giLg9AoZ)~A- zu&##`mwAMhiUrA3yBiY$7P_!bEly2?e|{{$L=DJ-~lj!gO8Dn;QDg{W))j*uIN%y#ZK6R`hK`)cRjz5g`@THwu2T0Rsp%treE18Xjm zm+2x7lf1e@hVv<0zJC@xt9HPHTuTpi_qfFKiIS}br-&bs)i zbo@WyHbSlg;DSmd?}Yw=NB}B!{CoB{(9;5PJnS5OKK675%7Z<95tmUie>U<$o~gc=Np{> literal 0 HcmV?d00001 diff --git a/docs/assets/gce2.png b/docs/assets/gce2.png new file mode 100644 index 0000000000000000000000000000000000000000..358dcc04b36585b1a891e47973b3dcc7ef5dbc1b GIT binary patch literal 31246 zcmdqJ^;eW_*#4`4bPOROIYTRw0z-F6$$(0?Aky7k1JWQNr7(&pBArrF0}>Ka(jZ+* z_xGCTd7pRfKVYxD*WN$4Toc^))%SUP&f}bzM_MW*gm(yU+_*uarmBdzapUG8_~(R= z3*O0G4^6&tgXM;rqMV+O=~nKoy4ydGuV`jqZ&^sN!U()9y}i9@xCm=3S@kDoZhGO> zz^Khb9^$J;dx&1IR<>LR2DkGIal&f{0+08nS7-eP`Vzc9H_cX$%~lpoZ~Xl9G&>do z!IGE7$G-`Ev_a0riQqdF9K8_{^6$HF9K{U;4wMDwpErENLR6)+af&U#m;ZgMSRL=b zuPzf{!44(KQBlx;zbr3{;${5r(n*~7$kBbpq&rdn-7UGn&WJ@upkC0)h1_gV>g=%*D{jhQG8_{+o zjcM8Kdrzq1nr!-CdtDxm8;zHmd=5DM%YHjG=g5A2_?^hJUMeH!ro$4anFj97JPGg8 z{yeE)BGWDkpG=$ldN+mYSxTLf8z!slMjZYuc8--8R0S52A zuu7D&!D9|6%MCo5tQk_KpX_E1b_hH_s!nEAnYv#)Z8>i0@3=1eoY7y*y!FN}@b#@> zb-wEJokqTAjFt*O$lnKZ|ua zDfK-+|I^V9mLzvX@X12-XY~wTuP!>p;#VZm9_z&sp3-?;mxf(~K_7A&F$Fuf&e@3)r>XFw~BU+WQx9?5e`|{ON zA)2(hE1tgb4Abf`Fmz*ciiGLX0icm0_G3JoKi-BLJw(RNUR@rWZO*l{yxhC1_rY9Z zr>-zJ`0DgYz1y>lf$ho4qVg~E{i1Uo0~$qIxei~Njw%$$q&)5$H(y`*)8tt~Y+U$PaPteTdZyS)Fje^? z!B+vSuM#Z|$Xhty6efuGl0S^UBTO6}((eNy+}>%}EG?bYDE)oLuAUB8AdxE*ASxu~ zPSQD9D@-)b4{WoTu+0yQyTpdv(jvAWgYZXuZ#YR-VX}KkeZQjIs=MLPBZond7A(F~ z!-=m)Q})V!3>7)jfepfrQ=^~T!+V?>w^vHL#o&KBC>hrs>VKz;1YP)yx6F9Y&72)> z(lloAJf30VEO;zPq?S3HC}7n!0lKhw7P}kpa5W&5OhRK=nW6F)va z{1a$3IC{$8eczAZdgS2YTslRU8COSS#Dm*7;ue*b6xhsLS(R$vOg~)qo_#_8(T` z6ia!-yFpAL9HE{T1QxY~mAbXZ;m@bPpwSGA1*IBae*59?gHx*(kICl^%LdzvNH%2O zY~ZoUQ$Fj4c2Wub&xX|uleJDWQ;roIGmW+1<&%_rRDRex zXZ=3SfR2(%q)Vhc{3v|r!QW{R`op!r)Nh@Ab)qjzs9R|Ooq(SapXAC;xi$7`bn^R+ zt0a--guEXr>Cq<%adCXE>&J58xb<6vbuwUn5=K=LnYJtBM>U^3`{h{c`BQcyF0qEi zpMg&#BK?)-+w}o&6Hf3vSb!$!A~*vBz7cT33lm(mYm){#<|(#u3nh`^s!3cKYy7IC z4psZ>TGE^kIpkp4MRY}~sxGm^d^u@il))2+UBPcaWm9seVZ{So}BI0G& zm1ErUSrsTTP7&umdTJKC^d250yu%h57`oa!K2$B)V@@!_zAB zkvYq}wiKmhrY~5PBeTs|L1O<8d9PMQelNDHn zd7o5s4+PX94*bM4>6|PEU9A<0HUD&@4k}QArTD&09w$!W7Ot!P=0E+It#`NeUrttW zm`nZi|?WRN6r8=!(G<9UwBv67)s<^TiK&LE}X1 zFW^)<{$Ohlk^dw3!VLQYCfMW3!8 z{`GI~y&0!JE*AH5XLcOoEcf*-lEu674<3BSJ&^fW7MfV?aCW|3JJ&#>?_j8Yx1Pn5 zbPxJ10)lyhRljC^VG|wXvMZ;Tk9apX5;J-!E-bOP)HQI&ecJn<(c2NYA+y|hg(t}4 zu-CX*+Bl|vo(S_4zUIXow%8O|@sNPS;KMO|#QrAAWO7Nh5dfDW^=UvGYCX&}*S z!}2XFeMp$^;R{CdZ1OSL6Gd(&eyGU*k=16nUMKh9#1{m*^L4 zQ`actdzMB{?pODf64yBxQb zHr8K+mQsRnq6~r-E3P!kWn51>071!bro_LIRXmC17IA83=kD{q9NFky&^BHlhIs#p zn1O_Yd8e~)Fwe0CS9#>uAP^#it!A#{Eo=7D$rt>1dKPfDUL+B=AK4!Ohw{knZHeDn zjS|@B`UM_}uMn8+^TvO98$?l_gZbz3*1j{Iug{fOiO*y{h{J5DeD$dV~v<%r0^q{Zvi4tfTJM#In+OI z^1ZdZY?iV5jC6dzAAfc;|Cr46MJ21#*-hwa{F5MTW^0!Ti|G2?bH3)2f|qM9HpmOE zl5Tlk2-)TlMAClAHS*BFWvT2=4!$Ze1(qnxDsRqK7m9j#6xMyyH-|LNC*p2#$Io1D ztspfVM5n=m4jlhB!j|sFDzq5&(uAtG@=rX83*?K;SNxg3-ziW%Lo%_uoIa^y(BY`M zWUC42ezJ6+RfdE=CmOkTAtk8L49|qSEAEW8Q&4JSmqjb0Ows8Ne}e6mM2!_GzmdlR zWr%}zD5E+!?hXIaiFSoNleeUOI3`Q~;Q4$(U;#=w{?2V;4oa-vWy84}o25^UT_eF$ zzc;W0S6Jk+>sg%yMVqF+Y4S5=Qyf?Kd9Y-D>TYGY_OL zSWbIDXn!JUAd4Hmiw<+aV#r=fDOfT$(($y7rPSGwI-Xt4bjf!)ziCv9s58#vW+mOnXEw!JKe1gIDZV#d zglf=@7ru~6Amnxkrqz0XToi7imz#6d-DE8{?3>MaPL6)=66n&yVl0|P-F)1Q#%;m# z7IAQ6l6NuS9ue5|!ze3R3+SUYl&L=B)29CVBSInXkf1$iK}8XVLIAdi(g zBY_%WTx68=DZ?!3sVY0GUk;V=kBbq{a=U5+PTl)?; z5*KPZ&F@j)E;#QpxSn+!X+F$XQ{P^cUfLQlU7t(!`fYP`I`4aQCR(}ur?g+F{`07b z1PE0d_Xi}lhg6tN3U~Kc`YZPbW#&u+4<9S+&iX8oNGwE9mdHmEzH^G3^njCQIBeXF z^CvXo$PYRzt|Z7X^|KqB$rp2fKLet9OdQ?_0%XT_64AKNpNwjqynv)II6vCHf8sqB ze|=34_tGo-jr1z{f^(fE91TZ4KIXQ)_cSX| zz&dZc>CjKatM?jqXxmU*l^@Ml6~FPgg7Mcl*9Wh|7nL&I_iK_V(+#7?ewO@HzRnoR z6mX8b6byJPTxvF1VfDP_-R(CO!5op0U~@lyyUAN z&epH)B2=!dkzUiTDj?QKemVbI)Op8xsz8b8xR%ctGnP}+_3kDfdW~M&XoGfrTFSub z=Y{cvrE1@(o0g<^2rQFEIhNz{3R}<}8IzT^5)NxJe)PO~qcN6A?YQ&~^BXrL3S^d7 zE3M7d@?U@j_(;=-X-{c5C+2njG$m)ZhIJTE(D}A#5nh}(AR<~cfpV)+#B`R4l7EpV z9z#l6PS|*)9&XZGW$g}I6XjYN;V182kD%9oXP9}pO?i%+FnXocohB=8=y*Byrg3Z0 zG?B=!Cb0_YS6IiSZhQT5Ts_lD8U`A|HhaFxKKB2iXLEteqr zTb|!W@vZwXS}OWNWXhb|_xnzN_rxB8G0-kbJD%~*7kY!L`KlAN*M^siNPeR9)@$+^ zjW-|pdF7z==2WZwsx${)EWcG3vH!)N?(KFm8IPDP0_5_Sh8C-$K&ln=G6(i^Pjasq#sr{ns?*vM65?)zZx_>&kt68 z$M(N=jlK2sq!)87Hu1QI|FZgIDB|BnXjJ9#YR1pT%71%v@~E#{XsZN!^Vd-B#>5YS zmzHX|dcs{;ODgZf9|$M6YRg<&YfBxb30czKChmEbVX|`OJ|MPQcP&+}fDf7A$aX;R zXQ?_bu^g@qwf7JfTwR^-=DOoR&67^^W6*3}5D;%Y_xf;Wbu?ar)r}#m z9JRoTD*5`=JpUZytWnw!KtcEfE}1ksS-sDq1>VWTog!Hi%9@U*zKaE;@4#$HZWH?* z?=+=bZ2hYJa!!hk6=a=vG~;bHUJG|yHa{V@%X+t$Z%nb;QUs@8|%N}R1?Mq$vztKM~ z3V!Pw5|FZ@F?wOG)!8Fpk|pZ`Xo9Cmf!)aa0Wk$HQ@-`h95MH}Hw!?ycumqW1fYKd z1?kyu0b8iemN%eLN;-g65i9U z(e725;VR65RTy$PKn@#m0nxUYQPfhex2qQ$D?FF&BB{@yta3@w_qFe=f&+D{t4H!R05eNB1Y7%_ zu56HqPMbge^yK!JZWf?>yIW;bXj49fEzTD+x%Ua|*pKz-!0>UQLT@j+`$oDAG7k3^N?QK z-nzEwK_}DZ)U@G)ll<{4#SPozt8wY(Pw|hxAZDWy47PT*y{+!iOZgW6nc-}jYS25A zz+Rd7pr&5(>krB2?+TO-jhY{Sai3o$Uhw(*gn5d641Iavy)8M7EBRyKb~Fbag+a@1i8IH`nyEe19QO_vGZ)m#(Bht^?lqkr&^!Z|DuwmES(S-UUM zIU17y6wl{yS;vRJ>~pCKW0zb-h<$V27t%^l91j}A9+1z`V1W~hb-l)N9hqY(-WNLy zCzUy#b{}ApjfGUUkX^n>jAn2{vL@=ezZM@NXTV3MpC&z!SyYu8D7C<&lL>5V>snCI zIyeTcc>uH8dX_o5jy`<$^G{*B`D3)TOD7e(tNRf8QDl_v)qye& zp@+WiudAIdWD?!~R6MRR|8v*>JU!mA#ff!0R6X0E(Z=4dP(A&#*2~eh2tAI?#5Vbk zt_UdNxcC9`H5XZ7mvv-`TD4FXKMO4bF?nk0)~#zR7bqMVEpPOt<*FguhYFwE>Lodw zABHiTEXO8CKj$Z{A~(KgMHfjtC^RU!Bzg9w`K(t-Ak9OhH&RV1v$VV1T!=A$$hHCt zE6@L4GJ?{2dj8J&0oK1}XQduWK*bnH>y%9P!M?ozH81Y?oybBkZ0qm^11*+1l z;zUIT?R5~6JP;ctD5A^wza;TZ@)sQKDVOZ!s4H#!vnG<{l17)T<(0$wV!YQ}zoJb~ z;u12{sfzJLd*xnN#W1}SDr!gNy}%qeDJBZkaJw}b1LXsVTf2cxIJ4qyjzg{-S4kI- zw%%P%EF%ZAq|wv^{^I;3@iRG*m-0>4ijvTgrxo1rlJzNq4XD53qtHv1MGP}Ep?5uE zQX#fYaP^~qCwzE-o;G`>i-_hy=EXDz$bF#oLNa~pq-5*YgnBB+#%@+ZJ&AcuCAz{u zP-gByq1A%NNtzn@fSaj|PcZ}KABGC*-L~3#KSDn_bNPNUkVe$g&hudrFYDH;O(TO; zk}>cHYJmgaUU~g^T)6epU!;66y07wSbEaL>ptIo*TvBZA3wOV3`&Fij&~1o*@M?jF zTJGWFbRK@DwlE!QOG@raQ{|fy96R9Alz(L1A2F&{NN$%>Q`G-P4#|q`vH0q#DYpxS zo7#q=JFU|2Xfx!n9EXx;?Jkg;U61@YIY^C$Qx!!aJ|}jGdX9@f=6z1|1P9Xik|fBv zX=Ml6D7qr+@v@S)-;lu#cy2;YDf*-H36Zcvk5!Ey>!wc;oRtc5mlYYA0satpGsIBpfg~l_<*6vtE>~XIri60SCbPw&ji^l#t50P>ka2snjd|u>UxMRnp z{}EwQsH|DjGLfi6&uA3h_nu+<0GW3f&YR1h8&`s4ocC^Mfr-tPTr$UQ>jh(5R7Q;xmo@TKC`*r z7vmqyX=Y?*NhgbI7^@T<=}%VWH!tQ-!h{cXcW9SHdaIqE{4~9?OuyV)#qzIH7s`HM zRnPd#s_MMaDtkl3bIgri#H)O}vfuBhm!{I8BZA#)hMRse_SL8DI;O(U$G0?R8ZJt6 zuWJ(hYN{Km?{@42aay+5RVMVhb#vwiERH>IBapYvr9)Ve#HVq|pzoQGNm3%Bk*yR( z2~uQ=Tmj1duGXy$e3aQ9VuA{#AVc>=dHuNVLsI`^MUNzkA@Qy;=YhYvcqvL>S|g>f zlkTru9NZ-^5`;K#SvF+|tDS|o!tBX^a`@xBJhzMPyQL$s*+VNg{Lz1_JldTH$+FJp zc}vtqe#2U*E+0Qu_~{CbThW_t>v|Ure)OIBWvt6SW9HCr_ajF^V{+c zB)HvS=X*QH=BT`Z9MV+rYtH^eI6}L^enJ*zoZg7eDO3FR7vDg(D}}c>wcJ`NtBD3j zIY{P~IJK#6;)nu69Q@pf7B!H13r&spQZs)K%PYExTkVJ`0G*e7cD&S)kouElQQL2e zU+C=n;3xMx#n|N8vHHOsCuQzg{iKSaR4}jnSnJO;ozlP4bf>*1lf;h&-tOV*IE)pj z-b!`r;F>28ieV8U#){&L9aSSb4;r#%z`^HyF5BKX=YN>HfjpUg`FbkkEm4VgOYxIR z!&{pZ4Go8vFVc;U^G@aj8xPw4I6alHU%U6Dq{fuyfh+wImqXp=;fCI)$Aw|8IRl49 zPRq^NjV|v@_LtetZHFoDnU&_`mOY&b>-Vhv%HN4gYGmHT_uO*ChA(l3tNT5)f;KLi znS(mj0)+DeE`K6!+L^+hA3b^FRYWIbeS zRHon3ZNmjxdgs=;{tuECh7>K-KtNl%T=N=%cu$mG*lmVEOdN*_qQ)&g4`km8|0dSG z0(q2_tX6Rxl(Tk&w!E)*n0i2(l&_&6-t7;m+JM@OXe3srRJK;yKu3Qk6slpG#;_D! z{!B`wt}RNq#v|XZ$n1do0&XOJ#gJHZ->iP>?$bEwZKt-B$4?3$`qpwV^X}7pt}y&L z9#r$%WcTsiSG9z1zv%ma(GhMaHJ_nXUxpYf@6Axk3vKR;=adno)-P7pXS?SKb;%O2ho<4J#QcQl=`$@`HmETP7tZ^&*~{|}V$wcptL>}aO@^yKXsf*a@WtHA3Ny(&55@B}NVD$cP0g?4 z--bdI-{_@^V_R6^b9;Z)7P+OQq)Jf_;j+iy>$vG2+8zB+!Mfe`bIEZ9v0Se(jltJ| zR56{q{#FxKHgX@yF>xspjfeUc_+2ldgk5#9S>>aBn9=WH zC}HJ)p(ffNLqPc~Ngn-skzPXJPN{&|#RI#+P4Jb6x3F3+kFqh4Bqs3w(^BK*W7r`A zYUzpi-|v5V`bA4jFWK)Q(j)Z0p0812);RsOu)QJRG$r1<-lk1SrN{omidge*7*C4t)ZXb zQc|BjrgkO8Bvd>88mpPKp~uk8t-W@#!H;}uqOUj@F$mVgZ~dLu&&N+EZL%HH5s5&w zsz?CQvAtC}IPgCW*F`n=aAaeojV)l5HGv@Gls@3608A&2lNHaNGzXkkF@yr#^&KeV zv7D4MAb71Yrir>j-h6-~9j4Q0wlG~GGc3$Ds%XzK60Ctax zZ+A1SCIFV1k>nvp9EaZ{5%1dX?avmi-I;0pRo=~%oia-IYuqf@5o62H%$JtBC${l^ zQRz)zlKAX75FL)58>8hI^ailiGmYLlkn)aDzB5CARG26|_3mA1{mLDQgC7beuVc{+ z6dJGByk~uj7~RlN5$_%2M&P^H#_%EuQfJDZjcJ3>X34msJL>H0t|?bqyFZ4K^;Z%lO*}ltwaH9cv+A z1Jl4W@UCMrIfjAv>z;hJh@)X>p9}~X$3=VDGmICfz8F@2Y`>1ZS6UV?GKG%MM>_A# zw~>;;G99|g+HmRXcyQ0K{+Rf0SKm5OBH>p%M%ICp`;BZN+k_dw=}Mhl-PEzZXl9Fy z6=^dUI-~Q--{!lzAybWJ8a#8aQfnfAxZ*-R4>mA4QgI=hK3Y*5RV>R%Ae$WlD&ar7 zF$RFT-6Cy)M_l>g*5nAdrfAVV-`Bmc^FQpdqw6#VT+3x1>j+V8^c4H6;R37T=Mm44 zx>{oF_Ys`!ayLBo`$daBKAFpq#pxw3w!)98`(q4!G~B6n3lWM^Eer-9d;O5Zn*>IY zZNLx8OuT1)(#fII=4iX&h-lUN<5690VdDNzUz#`K=#=CA2gNseyzk{(p;iX-cafP_ zmR!C|(-zT;>8cw^9LbW(p8?w&uxQ2jq4tk3EE^E|&2*gS*9Rlo-s5r~4`QlpUfq4X z1%d+i%Lzczwg5Kwj@TtqhIEH$Qqist%3M!h?h-Av*3AJ9x>NY%vthlLTKn)daBGN7 z8~hDHw^@&V&>2`;da!q?qr&(MlcsII)lJa7I2U{!e_Q9hTPs}mi|9pC6*G3=!Yx_F zCCD-`t2|A5)%G?#B8+`RAcRZ{OZ^TBy@CJ%~o(t0pZ{)YoceO}et(2ek1Ox(Z>BPihi{8gr1_t;A-?x+~C=49n!J^~Z zu%gFRMsylzimU>f566stB|#N(gpM#t{k7pu4XWZLROtP2jMrvhvQ;hq-aEDKGJ!TR zF}G#3@R5kM{6^m~bMVE`BvuadtKpZj!$OeHW7g`^#3q_OTm( z3t&@Tm^tM%7^78Ub3KBhWnd{24NtngpyPP~Qq*`Qs7Z0YihlF_+_z<4} zu4Mh?>)(f=?a?Dq7vawLVZ#F+#BfPCA%rV|%sNGH(Ie5{_2cvWFa{Sm|5qKiPbQ5u zF9*VIt&&IIV3c0Tv>Mw`PGd;v>T@r3SYi7VK^a{tKaqqrz??vF84}Xd@n&~T)IBLo ztDsP=f##i6z!Q)H5OvC7+vqCZ+vkZLg}PbyF?2#dGYbpHz^rESeVN5k z)UpGK_HRce2yn`?-gyTZGXHTSdsKqk=UOa^x`1`6c_E5^deuP^Hk`nE2=Y&22b5m-i@o-Np_li-8ZWM7po;rDv;=ZZb>DpUMB5K{a4Jb0lZ?t=UpM8bNW=okW>@2PN79SybEXV=&$39Sf96yWS-N7y#S z>OBaiY?JK+r=p8;*0Y;PJU8flEj@4GGpql=Y4@$dCf?SxB`9#UD(T4uK(jMvqQ?}^ z6fzY>6hFD5wC0>zF1=L#Zsk1N0$Zo$`xAZVZ#6LY~D4s|!3gY~}RYm-Lew6QkCJlEnoJWRJ2ELb+1VugVu$9?5K` zyRvH?yNFD@+8RF+_ZHujpP75QS7iAlK&AuK~8m$QR+JIR`KH|C{X_MW9<6D!n z_=$Xw{Rp`m*tG{)=)Qe6RO@U;bHBWYaCq6Tz+;NT5Ej{+DNv<`(xOtIN?uE3C)}{i zr7x+=aOGD%)0{*0@s}Pj;~^asj8O>-k;YKrE)BEE0M#*8UBsX=DJ|F3EwZlGJ8bq% z^PyHZ*JV=_%bQt;PU!!B_aKH)oKxvHs*u7plLFp1M5}0aMQjm9XfaAxlC#&M zx-u*I(NUvqiNx>bYLh>b z1e%?Zb3I0Br4fgkd+eF=^V1f52>ZkKd7XI^c8NA*%hm~i#&$k`fBu2euH}Qhz+Y)4 zx-FtZ_dt%pk<^rk%x}jX)w!rJ#dAm_pSk>HrS0HA%!*~&Bo!@BRkBtCSDY?~8}dcQ zS!?#6iY!6nZ~7;@LX6laas_Jaj5CEY5K@21;ca{4Y~Yfu1ZMB|_oLw#DF?Jg@Sk0y z=6H*H33sp)+&IM5X8NK(h|xsrymxPcd-8Cm3Oyodd*tFMJ%0G9IVDoI80JaFm@M8n zshf-!odC%_8I>3kJ1)i#{iZD2@N@$IQxK3E1rV-RVDWPa#oQ76Y&1?3%7%mTA0`8z z#8QL`%55kSe3-h+grjftOe*7W95wT!bENT`+*bO>p)2#A*z(^CKkM#PEQfDE+zr04868jq*hy#Edbqp?-5T1>0z&R8(=EU3G>rmV~d6t(qzs{Iv z9y$uO7wLMtU)5PyCW*_yBbn8XD^5Cy=-~sRPl-o6Pjr=>hfs$kapq3|zlyut8-m6I z-?VsE9d1|Kuid%d8AWq61~n6>3 z>mdsE6F+)c-%37={hB0}n7){KnzDX+sOGlx+ytLoEJP}?vcO7b*Y{0cU9+1Lj+>G; ze7Z-*Hvge*SoD$0{Uz)55bSDXNxOFI8&S!tZi1960%DkBp3>?%jvyjQoV;b1Qhe~H z6vV&G<>&?{pQXRvU!t4IXX0p~9b+dBt5M&Xw(bN~8g~}r!8`l3)PpisCvfN7)fKV3 zsn~Lh13rlo84^zLti0?5cXH9FI`@sG0cvgWqCDG0F~9T=?E0;twwY?WmT2ga-s-rm z{06NA`Ug$k%@qN);kT0Ri5l?fXkou)@~4Xfn+fL(-&hJ@!m(HLaF;WdL}}&ttSY#j z{`KM=Q%i-k-(+KN&EU8Y!eYaRISh--T~Z*9za!dBw=XHSG+!{7^ZU+I6E# z(wmgvPv#vvi|fq0+j~-T|BRWd+k61_UdWaYPih9Az#L2Khhs_pqQbLS$`B1?>}ht9 zy$ft8AI`XMMUW~!^kDU#%M0egHTM$X@5;?Cr8p`lf9v^jR6;^5@mh?!kCKJjD;q44{p8pw!4>CG#uV+uy^YMH-)K2B_Kddb(y99fZBMHNS= zGx4XJnc2U~f4Y9!wY}$H`k=0yTueDvH{YYiXISizA-mU)rQ!- zHOEK9^7NaXzXomr6hAGa-Gzm4Q2xAjPsxJb+thn{HA%eF){+G$oe8N0J>1KTc_Jn~ zaU2@1cwuxM5FCbL;>3#!nm)hvffGd1XE;!>R*pk+`E*vtIpc2@EGzQBrd$!>5Ne;_p-7<{OHs`ayZAt_vFO2*FxnIFgEJ@C0Xqw4WAD}solMZz zmwnt5(T=qq$-CJC{_tn^PsEa=Mt_jc6aZyn7lQqS{1j{SfEj+4q*6 z+h1~0*Du62HS}P5TA{HLV!S)XJ9P`z?oNJsCa$ZaoVoY&JkA*HtOjh3lk=P@!&oDLnSg;k4GZONtwnf4%Acv&t+{9MY87{Z>Qsed&J5>XBVUlKQEioHx( z)w`hji?jQ(2Mj--B2zW(zq(D*cx^_LVaIN$z!7Ao?Vy9iD#6^qp%k>*{&s^eHbT6!aeXJ49y%e{5dhJ*r*xl%jZF zpF-!hwK5fz3et7et(%1Nm-|TJ2J2%+Pp!b|$wly_kEKuvMfT+DYG>=JH&Lp%og0I6`fKESUo#v!3BqI%b-k+y4QVS*NmAy>K7vcDK~j2c-(QT; z9z%REUqO7TYZ#lLPL6SLN-#L|Up{&dJacY|Ovz41-Tf8Hs(J%#E28U;_>8;xdd3XU zF3jo6fE3Ec48hBP4_`KftqGacEncp|;8as%6q9F=R98$D_wt%3WwPOcN|{ZC4(Sro zl|$ZEh=TlSsakHl)>r)K17U96sW^A*lN`OKxwVL)Bklk`dIlwttkdTLVD-Bd=E21j zd-Bg>wxBGkx$!~Li}4cYQclDmX!XlfDQ973pbhr&1;fIr>72t_VBit;VWyV+I4`=c zS@7wQC<9}<$AneVfT3sRDAa-BDT5S~fB5KSg5!&GFKaK2e%@JnJUffj-N;X=kY#r}4^z)qd-N(b}- z?m!wde!q=_pY_$>(qw9+i5(l5M|+aO{dANed#HiDSWj-0)ugeSNN9ihx@2iB@AkSW z29n|dXgUn&BLZI$*4hvtUToTisn3{{l|`9~ESKmIVo+*?EFI=GZbS+6e_sFJ|M?rm zR6n{_GX<)D2sbb(>*Dxi$nid`|EaU8v{GR6^M8vD@u|2$qhXOo@dW=gE(F3Q38I3? z=+7^x|L2b%p)d=T%1M|5=U*E8W)C=GX_#t08vGA&mQ99%;0jUqGxI<2`%M9K(P!eL z4W`2wv#e4qm}Aw2CszM)UP3Z3rVAFIUPu0qm-hd?hJ@xMpMeU&El?X5zqL*(v1MMU z9AAMc0eqKXYUop_(=X2wjsIqFv!D_{?waPn^C=+7oj^6Hw-h}%=q~{+@&YwphJYbi z`eWeU6u|Ee7&Ghwve)bJ|HY^Jvnmw5l2>(aJW~D*1cIh3F1!fGBl9x30F>47Xlt?t zfZ+P;U)A=u!N95}PKS2rzS?QrZ7>oj$x8RE7age~+&p{sJ$({z5j$X_s(oB)TwnH@ z`>!8BTiWLs!8!4b{g*)W?)O{8`5wb+M?VzE44V879lGM^yhb!d>GHRM7K_Qe%~@P{ z4^#2v2*|P$@3_aEzM9}kh=<5AN(k446^D=rjEY}bc+x^g4CVIXOFrI{33@arhwAG} zV6-=KtbPUR8jI%7Pxn_P7Qg717dV3Yt)1fnrI$IZKfe09ADe)JwCOpWXFVzGz&YAt z0IH};KxHMs&A#*kmJvg*3-Gn zapx4!`OmIPU1RSx>F+%)oi;Q9minagfka2(YN`fIquwiujM<6asqSsNRqc$I5UdeX zwAONcWe+NnsxYMP>?8;<|57Hx2uw|$Sa-~oGoYk9?p;QeHk#wigiCaS)<_I6_8L;6 zF^DE*WRU)z!RLL+(HORHbQ~?w0#N2TfV>U>neUidFMbF9(P?iq#Ne!4$kc@q{TllH8q4(Wgfe1|9r_deRKdcGx0v7CfytuZNGYr~&ubsaGD*#5se*RuK zC1e0tBbJ&u8Dq&UVV7V<|5Hs1(ClWZKX4Qq2lLyYl-rMSZ$|Ne2K%aX(ekJkfNEO% zo*PTkWflr5st3&2(@p-~qBUSqv{mE%$qo41Qz~7-02iKjAJtKr1ZI<6h9o1W`cA*2 zt@Sv9;5m>RIRaZ+yaGV>N|VM6W4D;|xA%L2ix30ej=%!`1!^qU`9q$!;TF6xqZK+^ zV#JjR>3m5eu_f?2iS)7{aL1MbW5#T!#~I@grGAQS0?< zaZm)g%^&$SnGUXSmh_9-$YH@cTsL%!Hk%I!A+ei6JMDI|-L&dS6NNkHz<0}|n;Q>1 z1Sd*26EMly&oor$0_WAO4E`>_Z)d&c0=0BbOMKkrhv1>plNa%tNH%PRn(W%8{a?216WG zNBFQ%jH;QSFQ)w07N6y+=HASW%JD1y;E@3S8i_-*HP7oRHZe zn4@<{MQ4A=K;9W{eI&KlihV1uv4z^owO@F=?$7BaQ^1}a&4TRfI1(5=z^^kPZ$Uua zPxGG_;AlBjyMgB>`5j6o1x*VLV{DG3b&T;1CRhrkSd~DBH-yt+Pf0Gt32gxntUIVo zu}8<&aDl~E|D7}kq@L`}V?)E>z%aZb-MtV>%ks1|vO}N`O*9WAIP=OtJ$Gj>MK?BQPwE)(9%Bsix#wb=C6&fzZFJZ~Oc)}{dL=sR)4r}&7ttL6 zDz(Mi{rd#E>(rAewHV>J>L<-*TByQGX}3aC=3<2k^SHdz53hH1lh1BHHC|wN02Al8 z-7*iEWpO`P200No(h)F+JI%IQr>-`?g7@B$t`LwuGxk3S?dt)B6Wa$VzQ_?qp0zbb z@9BkuJGP8}RmRq=Mf?=m7`F1gsYkL@idCgvVV`9NZQv&7p{H0Tx15N7dye z-PabD91qXIMl$*M+Jo-cJf~nxJ5@Ee=K_pY!&-8WRuyUSVjG^ik0TWY3=8&}Bq6?m zpVsH9*<)vVl7CClkk%`XG+B^sH70u(DqZkok67iQMZvcdIiA^*l%H>AF-bEz6EhZR zK{myJHmsl|zT^h&Zpz4El`)I(CH9vg0r|cwbSitU6*$2j*hL}VDsE`qbo5u@{94t9 zP;WpD42LJ$6f>VLO@x@-MGWfMhTH$n8|~o*X&Kf@?iM-T+ZgL!D5GD0!}WK*7UQH1 zn@Iwb4%g7p+G#iS)jgeW;<*Dkh1YV!5g(SO!pmG?Ju6#vO9{N~w@t@d?G&FZG?7hi zkRL{?NzM2w$^ffYXTRApQNFWV5*FNVL!LVzmgN_ z@JFPQ7W)gN_;E6-rK$P7LJMm2(LRnF1pe5q0*ito2h~eV-F@m=1f`aXqCds~{z2e& zvDNf|20PW+deHgxLxbPNw+f44j$ljBbLWpmqzn@g#b;g0h8|@2JNnxfc%nlSW-p zNpuw)zbLjO#6u~w35q!nWmI2V*DOx$Psr^g-O&-+=hcrQp+9q_9{lpt!cfq2Rf1{WruER_&gv+3i01ahR)hSZg zfIyBFxEy++cQ>c1KdDcqh^$B;)6mHTvE|m-kS@k>rL;R>Vs}Zf6aEx{N;gV&g{lB* zjW2Fwfd!5^`1omml498y85!ttT2BHM)eLWAMM1dW#3e1f`em)}rs$Y&Rd5b>nJ07O z)8t?yauwG=1;6|vtM0hI0Z%wf;Mswz4#AL6M1oFu1gA#k{N8eO zxvjzm+)^=li#YlM9W>AX1J}w@fnt)RfR(?{%v=pOq#SieXb5J&q#Dt8Yx&`WO;*F; zVA@ZpwR>8&ZM_-u%3@tg$^-_&*nX%aP01I#SC8 zL&QG3**XT9CImf1U7b5ncn-JJ9qGGGg5R^qN!3`6zev(_QDDz3Fgl=FrGi;u#vDMhZbO?(yi=*THtJDvAqPWq~p2iH= z#Db)_Ik)x5-U1lx1AkslSA^w zj7K2J664uJSSmeW&In2znPhnzsg{fDs_b&5|Kx^yjtDx+(;Jfy}LK|=h(@?13>2G%RSPd%Gm93{Okqrt7685 z%5%hvDc~oSYtK|OnI?ncjz`rqV}(!!5N5P+SH1ceu}fk$C@$jIbz9rbf$Q5%)rPV0 zwmWuIJ<8QLUCUgP&@(c5G@gvb3L6LZBj&9WDWtM7-+1*!J#k0+i_0_)r2@lHgq(S8L$^_D<*%})-${jV@!$};=TSe&fBXf41V4~tiuP;s+X4AWD!ON-_9G=ym5Op3 z5;N!^lV78k@+{2cTz$T;6MgJ7*l93={Z+i*f0C7j6^L58ws$Z8Q=4hV;0^YtA{$x% zN!BFx~zQo<&cZjkP71f-+{Hnj;s zN;W0Zedm_=2~m6IsfrIzx^6Lf%8AoUMHk%%wVSA^}i`A z4;Vr^p+S@GziDs33>cG>MNay^xpN;DIIgV2#;bqR-v5UN;t!-fz`&-sBR|3WAHyU0 zD->f99r*uDCYCXk+qTs8R&%(?M4LsC2Ws@ePnc)dN@d%*o#J zb~smCUmi9HR{z#2JR`)D7!R3MlBZYt<7|P1E9})Jc%n;zMf3;+Yo8xnfhOKKaPGTr z&eq$%oXT%10Qm<9k;?yy%{ajlSi1By)uHo_8?Z0s7fB*944@7oPA-+H9CHSYS+9U~ zGabplIJ!Lkp+Dv&cJg`NYqON$kcVG`&*1OJg|R-8YqETs z2f~q3;5s@2aR?GlvQ=8Y@S&Sk9x-L$3{~n91JbgiIp>j?+9{*W>`RaU>j6*9b^s11 zHv>`wFawgwjj`VX^Dc1^VErh&LZ)Z`vpk63iY$6y(gA-i{~)6#=)^PdaeGGq6n)cCA9Y`}F&l5GSB9Xl|>v$zu4Y zcoZd`MDHr^3gl{$sWcvow}6Dsf&rcmQ*gF#SPFInDy-&BvC9K3xvFT!7{8Hp(d|CQ zC>L|Aj&0xoe9`WEiTcWt`U1rLb>pfcL%?@whZN{gyG@(E$dbg6gZmsK-IqwuS?GMb zX{0=487g2@9F-J(c313TQT`MOeKLy_>QMz|oOJ^jNWt?|Qr7xX_G)Zq#%dv;|MnMb zl%MWZG&+owB~jh|_@2)vG{eyigs(#AK;7`8S1wf_R1|T74b4pGyeVu3C7u2tsD)2n|7kl0$-^? z9cq<5yk>(!<#=;yBpsyE=Kz|6-SdtS#XoN21W{Z*cWnO1hhCV4@*S@e;GYO#%Khb= zfJQ8>e+#qgqJ_TLd+GNIBsdbi_khCYJJJ#8b%DMM;#&H}7!FAWQT-Ub40CMO0hXHu;2cHA9S z&NMYMGkJaV3|+f6LhU;YHD)jQ*DUJ1-O z1S&1@s*asKx>>r_FR(s_tpjg~REu;)v<6XO^*N*?0E3h00d#oTiU1Bg%T%5q^sHua z&Gwqi=*Nzr5*j010@=#$WX5HaV4;0q)SJ|_L%_Px-=PNXz?hyqU?wRfHr2XV2LpZR z($l;gDFO!`LbK+rs&J)txfo&V>n*+p;4YpQ7Mr# z#1+qP8U5!NbA>e`BWywa30wGOQM%S^jwII=q>9is+6YZCYBR&P{ zBO)h@fz*{+c5SDYQ#6!+PEtld!f7-M<{c8s6zQ-b_V%nnQr6_`VB-a{KBkrm#(TmT z4|Ak5@>s%<+lufnNP~%PUL72kR1a58mg+alZ9Cv!z=f2spHBG!)D3C-{bJ0UD<6Fi z6)*(Cp3+mxlkTp9`7D2KLjJwbps^}k_aT8RSd1BR#sF8Zxd1LVNu#HE6>SM)t^$*2 z2Jn)mh)|D9bSoLymK=z4hm$Xn#l>X;~0#@T&M zT}>J{oAU=ee@z(#%>%$$X&%?JX5}|`hE=c}f?RyHpog!R-iZaR3Q!E{osIsP(~4ZE zpJQQ4Tyqbq*Uf1=^}=FjvAB zk|az{|EvN4g<>%%k|vQ?-%b|8T8V%GO|1n|hswkwvQ-g?thUQWQoTh&9?c0fSmcs=3`$#V&I3IxfH8s1=NB?8!7Vz1c1S&XLe7KdTC$;t^?6PsfHToS<2g9 zP;-Aa3k1!!$Y*1D64ZWX9_f_XfLd=~&9|EJ{}V$(`Fuf{lO5k+`Ah(;uz*;Xh;$7n z;nMHHv!k{7)1c$~$_Ov8oNoiSg1Ox57mAM{$yU3?art00*1_ptX4i-^)Xt6jATGZLr<%T)S<< zXMkKYCLz}s?4crojX)Q!#~C5C0@wvo07~&G3x;Ic83K*vWQ*s1iT39w(Za%Lg@I<8 zp5O!XQz5jz9z7l{V3Fl_MLY*_&OT3@Kpdr07G_dSg3C5CAUgTIgyH-+#j7hZpRL$s z_S=iRjz&5CO+|$CFH$3qETkY90SrCUmz@lU27PSPd!0;&z-m7J?CQj<83YD~ao|+S z4%rPM?AueT5wJjUZUq+A)HguG=h)FJy(f(TBI86Twa8>0+=>mb8B{xtrPT=hKn>p( zFfJV1NjYAw?D&5_DU%7mU%Arr{sOcJpb@ve{*^vAJ^rZ_&;Gcrce)P9=97VR`a5Sv zPAZVHxoW)HhS!eUk&Zz?$Pu9|4V3E!-y-ENkt-?)XXa(#5eP$M+L9I)`}kpREdHl3%nQBCd5GxN$JY7q(X!lnE$K^SbPm7c`ZlxQwg8kBWGt!|cpJ|K zRBAb*Mc%r(greT(YQLyzwwy9_=OBAc6n~J^wBgRJPc&hq9_iSSP5WyQRPDEWq`)3) zoJxg+l&H(hom^sQKrAoJX`YfV;W8{!@L4qoEv7vsNuboI?yku>(TAvF$AzaC^>JBP|YCy0joi$b2;eMM8a|oQoXf?L~XtTNvp-> z+2LPK%f=+I_wVk6C!>MtvHM+!r4fU70Nq;9l&K3IwHdyL=jSRvg{lLOAW-MgZ<8wz ztV#rm^=h=i7=BwgiFo9~tC2*tgN##AK>s`kU$vc|;af$l_H=YcoX1qcP^qdv0%~JU zLV6O+J6c^!l8fv2w=uyh@pu|2>?;79Yl0jSu>wlxJ0EdBe<3pjaEC}2uY+Hve>)T6 ztlt5}v+@P367+kTx9et88O1+}y!lSIxzO6IrJNcDuE#9-J$q?|Ry)vUvFjY;lpdyF zW147ow+A!uddw=JFX}nkjy{4|@sX!<(P4rqStFpngd9pHrEC@=Wz ztE|hvdWXy$m`SS+T2=&&IG9L;WAlps}7Mbn8&iJMb%s`Cb;jbkv>bU%R z&hd{Qh@Vq)TtQsiRF)Z%Ye3JI!MwIxX4ot&N6;HTNa4JR6od-XKF|tfjqnt%$}R`1 zTuc|$g~HI!U!%7pQ@DP;ZSY+QQ(YP~Wr-9PUq*AB2^*FPzti8gD(U$WA0)gGBm@D5LR@!$;I^d3uP?Yf{AI$(IKg<%lkpB?s9j@y4-6AWp2%yyEJ{G z1@?1+G!n0TAwu{{eVAyN_sI+nFh%{jm*F^(-J235Wn{eBJU#o!wwjq2Xsnj{%aC;~ zo-CgYPCuTmQ8<7a@ffWIvD~2|T0yMS4RdafCom-iKfW91m0I$hHxzpvcYPNmtqu^} zSYfoglWKm6y(UpKaitxjx8oe{42!mfk2@a~N)PS-o41TIhn}&c zim470lj`;IM=jM#KoJ%q7ym8c`CulFNv~(97$neBwX5rEAlqDA;~swOOp||4E){ic z@Nr*iB~jvH0kYpQ?8~4-<&vsk!}lOm;ZX`zFUHLD`&yEm2;{0lGsi0T)n79MaA`QL zzWzQhxQ7<_(~LcUazwtP{R~mOQ*F}WVagi%^C74{u!g?le$(Nm@L6+5i zF?A9eb`QZ%#z*q`hC4uGHV1LEH4O~_?ioj?L}_Jl+!1i2H#JA4D?R35P~NW%p>3fK zi3QSJ3)81iCt&n-NOa~vr`_UuTL;pjSs^_^9LGdS7mulE>J?RVaqP%2+662*^H67* z_;*D08l1``9YZeW4;@a(bfQt-P`>V-Qn_3VsO%IB7O)If5XDzYmBfuVW;^(!U{wwb zYZ$L^%L_3bEy7lS$;jb$Dr;DQXp0bcgoeX~BP7qUk3dQpp1egWO#GyV9TMHRCrd3C zclyp(HYAIN8)ug-jNy<~RIWaI&C_EC z&=~s5#1%CtS7UcN!kCw((Ib|_7?t_4@k)Cn(0#G%vPvdP+|&aGJ;^ zvX~4Qp$4E<`7Z@Te&;&)AdZ7ck&*_WSJNwWq)Y(dqQbgN)i(KC3B539Fn9XmjHuNd zO6a5-VxW)`!zm2PBl9R>g7mi88tw7B?(;dRWW2a^GpM-u8!wG7ymW-OEhQ0-wo7+d zVuWMDqe?p7Ke@M|%;26)7J-(#gR^NwM)mYfgz%`m0~*LACj2@!vfguy5m z4_B^G*vGIEy=<&_pudk5V+vHwEa<&!5uHbRv~mth(Kx#>*fHuBVHxM5PS*de(} zhy%0`q6k(&X<%@R9AG81xDC2kS75D{83s=TP8`5WgoT4=A!~Pj$YpURF7Q9c8y#qXv10s>Qdn&yo$$}d1G>??CP z%y~2ZBbFx1yv*1B9^nYyhqBq4n%W!DY&6_V!%Qqh9U>3rLw-siO3)-YrK>%nb92Kk zen)DsGSG*I!H%I+o7wN{k4XaWI2LJ5K1Gi5@Alvp@Y8b)H)@ zWM6ZchPsp4Z7T_J&9FMe?~`F?`!i{>SYD*hjIT65@Ax)N|*J7aybMHW$FO!{;Y znWYU!0V6F^2*(Bn>w|Lu9CwH$Pl|B}V|0!1OT8#W>$K#;K9`y%SG^O9&chh)kL%Dw zXV5K6QDOzQTDm2-%)S2p^ZhTsqypIF=PEOI@fHajC4y( zR?4BoOlj(kST<`>GGon?+-(ivtD{`}JCbmZb+Tu>pNWGAmc$V?LlClFm)Y)LCfq)G z^c!sjbpdK9nW_ZrG55*Q`RfTsB@!1hC0=-J2y)q61wK*uTz!S5LGYOuzo$!>T2+&% zkhk)2m5uo;ROhIX#IL5`WGR@*9~x;o+VFIW6Wt1Cp+(2-h;GVj--@<)OE;11Wl)6I zC6y7z%B>oW5lKOaF-$gkS8NFvZ#sB<*HJfHfXs;DX~haAW-wl`<2HuMttqCZZ^=xX z1Lh_HGY^gBVs9xSc#i}5@RR*Gh2)6{1mF5SW5fvG5Ao%$L+%x739$aem~MacXi!G= z+f6lDdsXp%ljT7~&Ir|){@|4^4dG+Vbs8m0N3Y1I<4NA8n)%QMA)P?Xb`%uyO5|?@ zMAw{)?$992Y-w9N8?&P|Itd?qz$Z4-JQG!jcl;W<;3+a-V9HBpeMYjz4e7);u|^h3 z*IBi`lWT6HliJuUQ*2R1K91zN^*rMlWYWt>tRFlrI3u2{B$^|2%^p#?41tA%%7}oE z>QAm42##+1703p9?)t@U9F)E^6uvbp77Qz?5?+>w(~ zIP_2Q`^ce9*~ibC@O+8|5*4wbQ_*Il7N*om)OCkbx@{@e&Re4))}J0=EiIjk3^g;f zmK58Qj7@rD2~zF`iVQ59Qv`Vm+xk;?5g}QHd}Dk!8wu2{-H5pDD~S~`g0H2 z4U<^*WrP{0lkrEYFL6TKv>Au471#Z{&`?B(@vA$2IwXSTAN1RdjkD!jxxYkyn8wbb zo_L5pCh<-gd*%};P?@FSzgNgF{mo~tG`*_)zQ^|-zi{l8;S@2>@>yrxU7HdH=P!SV z)#6FI`z>cowBS1rt@PsgeZD#Q?Sm4oMfHw;VsBKu1;*GkuhhHwd1*cylR;+1k&@cl za?8^|SM5prB;pP8geu!OWlV2EdF$~Prqlx|w%3a15SfPqW0Bf=2)~6mFFm5Xo<}zY zV+%^qhnb0Qm*x7Wq7R6kqEYV)vu^O7;xn?KaZ|TiKKf5#o`zoC3OFo`s}tpTVCw(Y zQ2y6na}XbvPwVXwB>$GdHh9PQ>2N1yBkL74EH$&oG2zNFeNiOXa|XlSbeRTUf((>m zy>WvX2`|sPMR%kTseRpr&Q!Ty`mL!V#haJTe=dv+4u}bI#vO+`FHIl+-SjkHixNNy$q{ukfQh_o$=%!`Y5X`x|1w|inre}GIey2u!L%<&!0He zA5X)1i?KIQ@_chCp-%BkG@gnCoWhEhs~kse!q z$e8zH7sMo@MP^v!PB*JFAICoz_yf2&)INfxA6Qe(SUe?ejq8B`+~vd&-}!Lj*!!hwkV{^W_^k z<~??GqPTzAZMu|dH(b4DYuFOOxFPym0WzXxI973UmogKP41VIIeJ2tIOQX6D98?b> z>}%q}bKZGR36xQ8t_693q#DrZ6bush=C}>&Iy+S;9p|y&`YSQ~V$nE2ieQYYIw(gx z!tdEX{HgMqojBU{-GS#QuD#iE$jCV4uFvBE-JyRwI1&IkKO9c!(Y*%j3cHcnD}0}F zVWJ)9hvxDLp^2gC&-t(SWvd*{M2R(f9qD97W(NX}KO@hLgQ0t*3gw}*Z)m5&@_w?C zvBwg+*-zK}7W6>(PxOvvh=V8 zzDM~1$WV8UmizLA_!kU)Pgl#@kyc}kb0n<4D|W|wvE!X{@{F~5;%i&dYR*E>n+_qX zCAyO;Lb|0zDv6}ok8dqkA&#nYf7objk|ot4fYA{k(uNbKXR zm`n$w*pLE0^)auB*1WF$?^8%}8&!zKmEi}yRB*%Lngcxo3O|s2qT(g818G%2V_A*L zi5_m0a_-65?n|LqV1RkbuiC`jwmzwO`u ze#ifIJ-{WLr^$FDQde#&=U$}si!u4Jdk`L3)0fC;SIE?)eGcjDpVi`0*X2ib_VqFR zzbcP}cDuGt*>Dh7ug5f-8%6}rYjkE)>xs3OiY6=r<=CrsgNN&f^j}n~>Q7b`eHv&~ zBIVSxxNJ+d(6Z#L=Xj}IE#?05$G6K}HcBx_QtSC8m41uMr|k(x(OI`Rg!S{0#Al}# zWz_X@hQ;c>+wmEv9>Hmj;t;m$>&cT;l9!+U)amrTJo@>X*YYNc2YJAde9Z^*2EH@3Cn((!2ccB9(s zX$UEw3d!AJ?TK%zjOS^E2Cl6)tBHA(cviI6^4TZb;ebzd=Pzo(ft`+K#lp8vuXqi) z)MoP2!?6Od?Z@kyAeXZ~v)S6N=S$@nFB)G7&E`mP9H;yl_$@P8rl(!3ed(9Af|#)E zU3dylZ^;gtDAewdDVV28dBbHmHAeWHTXP-W-osHzNF3h)WG6e}wNdO> zX?275*^S<&)EBHFC;-TnK8;_cX!`e_Qi*1_Q=MZDKu8}wDipcVoY3hb*HT6n_R8(q ziHg&3of+C3(wPwVx=O&7Z9L%qDsR;$dFo_+;rAWC*0#Kiv1BG#yF_Z!>ZrSXzjQ)Y zyJY0SbvS{DnBc;&@Ef$vioI(ri{E`RlgGt+ljj%AJJ&4F5=}QJSnXN1^xQc#cjxVk zl*Z%Au1@kB-G=E8=UqRhq^E3QyM<^-H*O#-NKCgQ%}^zVM`Z;5WCKF5;~)wq*Uo zN2}Xs`7ATRu8$gRzPd}1>Xx@%Gw#q{8@)Mhe>QZsF#Qc0W)X%~PGceVbZ&vhZ&wVk zaX-6?v}K*|#2K%FUok3VOCkM*gO55L!stpB6=z3nWO-GhI? zxvyVY&>*c|L01G2THa1i?r`Wld zx{SB-3)g96Q=|Dhmz=ksP2FdY!hf?fT&v!)cw~_*kXMS(As3YkV~^l^%FzWavt_bl z)TV^O<(m<2;IeV8y(Ocaj@xsrZrfAMd=6t?LyhgwXZrd@zROc@JG!>#jkxI zYN!<6er*fytnU?N({9TjYc);gY4{AUFJROkshbMF?W*VbS@_7fNT8>7d?mr@lcTcj zVJp)Zt($SwAiQKfL}meH+A&kTp708x0?x49YF^pM(W_>g7!uJ5B8+ z+Ph^*2p>hbJNDu0+x5@Gmtzet`Ny_L8~r&2IhHApwVuDsGGf%(>?X7|AZy>3skG#p zDSIpwbGTZRW%ncF`U+o##^co%1?gSG!b(UAB98BY*Zz3+Wf|$L^`Ksh$}Y!jVP~56 z+#ASR=%k`rDdY>(>3$;d>vcq97RP)+lf{bzt=3FTUOuu(dm$ z+#Wo$N+wTA7Sk>GaXBdafMsHA!jNKPNbSQD?>+uj?v~nkpO?R6Pql8#WoV8Yd@WD!3RgC528tcfn z>lBQwI;76szWII9$@6R1U*xYFBAZmygT~^MA^6Xpd4JIyqMUz`L>^WK8wf6Sv0moy z{$YRNCBRd^ikCuI0d)>%t{I2v)FX1VMa=9C*b~KcD|fA%hp%K2b-vGDA!J3H-8mM5 z&iww=+CU0-+S50V8Ta$Q6(i3wL`+p=(Tgy*0i zK~$dtSpxGs(q^_WO^<7|bQ>OAd0uA!bFzpd+Q#s_o3PX@%0QdCaMsMEou;Z2`cuu; zIbv4Ol2IE+?zp;RfX~y!`Iu}D2WOvgbNn)SV=`>h^{9X2Y`YQf;ObABU9xY_d?}2M zV^0JMo3eqI&N7!ynFS^nOL%4LUvthIX2l4kX7+R0YFtG+;WY~@)oJXGWwAXuTC=n( zYCqe0k8yp+TW(M2Zt3{FG|Ga34f3kp*#h640)9pLg3sZ>B@Wi}udkR?;?`HJCh`n9 z`)tEH;wf#47V?g5-68pL<7R5ORID)Cq)(eZIZ-%p&!fIYG{4zjmY7?=MC{UL#ru-a zvab6C(Qcs=2pXV)|Jj9_2F=sXyB}o)Tk+^=+ig!D>snz-7Dt(`_1Fhf!AA zJ=;Diq+%HA+wdx%Cb3Py`_ZGrecJrSv{QGZPpgt8h-9ycby%@#`)n;o%QjSU9)^DG z^{6}(jIjPVi zj*5DckIimJzx3-P4-G_`d(ZQ1pDVr!7}F$vaujb_X@T3EZmX%Rpy(hnCAXToSVk&Z z6a^b#%cbekK{T3{KCd5c+uu@&=AfLW z<=vyRI#oN{V%oaV;b$$~ofT)MQJ?kR`Tpoat!ioEackEuH>3Q$1(iL#>r?sG+xu$4 z@b)OW`)KoVwwyHrk1^r>JJx?79B)HV zP_gJ7-uQP>ATZfjC=jDirx4CGUIKBj3KEEC7Kw9b*+oHIg<8{R6MOuTZw8Ob9u7J& zVlP1z2fPLOJ}jqP)q3O$;MI2-D2%RRVlNs0{z(vll0~DeqmFg+L0{DOO&PSMgg2j( zeW!jmkBt=_c_fruVz(scN1`VG9smoSQM>?i{2ocbzu%d)vo2?SbGQIPI#}QaHnzmy zA%0PNr+&Pj!*_Fl4lDvhhYLo~&0rQ;#0$<(M=Aauz~2NN>(HbrK=NkHzlrH!mltoK ze-9wejr!h;+fai3=IAGP2oS!$HL`yXKu2~<5>T8o32u&lfsXaxr-!`f|9$uVuaonH z9&3v;A(l95FpU5q*OT@)wjen1zjFams9f`VUSvTz;PT2cdsz6P45TTq*iCXpHvhU_ zSL;5>+c`WuSEqfZ6b!^Rgu{2F(S+Q2D@Aoo4e65-O2Z!xo zstpROQy3bYB1T+1LvEi~bc^)AC&vt%%n9(p?HT|0Su5D%X-;Et4wEjzDLWMKnf9pPqshk@1q$h>yH>mXatr9#SmU#Nyxhi_IOf&!s;jgP)~zXFtmIinIjIJ?$U!f@A+B(B!a4f&((~ZTeW&Pog15jX zsaRjkdX4S3jo3Wv zky(LDJIW<5=z!J_b;$TGX<9mBcvF;qB`__ -for overview and comparison of different options. See our guides for -`Google Cloud Engine `__ or `Digital -Ocean `__. +for overview and comparison of different options. + +Kubernetes on Digital Ocean +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Available options: + +- `kubeadm (alpha) `__ +- `kargo `__ +- `rancher `__ +- `terraform `__ + +As you can see, there is no single tool for creating a cluster on DO. +Therefore, choose the one you know and comfortable working with. If you know +and used `terraform `__ before, then choose it. If you +know Ansible, then pick kargo. If none of these seem familiar to you, go with +``kubeadm``. Rancher is a beautiful UI for deploying and managing containers in +production. + +Kubernetes on Google Cloud Engine +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Review the `Official Documentation `__ for Kubernetes on Google Compute +Engine. + +**Create a cluster** + +The recommended way is to use `Google Container +Engine `__. You should be able +to create a fully fledged cluster with just a few clicks. + +**Connect to it** + +Install ``gcloud`` as a part of `Google Cloud SDK `__. + +Make sure you have credentials for GCloud by running ``gcloud auth login``. + +In order to make API calls against GCE, you must also run ``gcloud auth +application-default login``. + +Press ``Connect``: + +.. figure:: assets/gce1.png + +and execute the first command in your shell. Then start a proxy by +executing ``kubectl` proxy``. + +.. figure:: assets/gce2.png + +Now you should be able to run ``kubectl`` command to create resources, get +resource info, logs, etc. **Make sure you have Kubernetes >= 1.5, because you will be using StatefulSets, which is a beta feature in 1.5.** @@ -128,14 +176,14 @@ Observe your cluster `web UI `__ -The easiest way to access Dashboard is to use kubectl. Run the following +The easiest way to access Dashboard is to use ``kubectl``. Run the following command in your desktop environment: :: kubectl proxy -kubectl will handle authentication with apiserver and make Dashboard +``kubectl`` will handle authentication with apiserver and make Dashboard available at http://localhost:8001/ui **shell** @@ -238,4 +286,4 @@ Init containers (``tm-gen-validator``) are run before all other containers, creating public-private key pair for each pod. Every ``tm`` container then asks other pods for their public keys, which are served with nginx (``pub-key`` container). When ``tm`` container have all the -keys, it forms a genesis file and starts Tendermint process. +keys, it forms a genesis file and starts the Tendermint process. From 5c96e0c81262b262968beffe66edcbec723161f0 Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Wed, 13 Sep 2017 15:48:45 -0400 Subject: [PATCH 25/92] docs: add original tm-bench/monitor files --- docs/tm-bench.md | 64 +++++++++++++++++++++++ docs/tm-monitor.md | 123 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 187 insertions(+) create mode 100644 docs/tm-bench.md create mode 100644 docs/tm-monitor.md diff --git a/docs/tm-bench.md b/docs/tm-bench.md new file mode 100644 index 000000000..2476ca11d --- /dev/null +++ b/docs/tm-bench.md @@ -0,0 +1,64 @@ +# Tendermint blockchain benchmarking tool (tm-bench) + +`tm-bench` is a simple benchmarking tool for [Tendermint +core](https://github.com/tendermint/tendermint) nodes. + +``` +λ tm-bench -T 10 -r 1000 localhost:46657 +Stats Avg Stdev Max +Block latency 6.18ms 3.19ms 14ms +Blocks/sec 0.828 0.378 1 +Txs/sec 963 493 1811 +``` + +* [QuickStart using Docker](#quickstart-using-docker) +* [QuickStart using binaries](#quickstart-using-binaries) +* [Usage](#usage) + +## QuickStart using Docker + +``` +docker run -it --rm -v "/tmp:/tendermint" tendermint/tendermint init +docker run -it --rm -v "/tmp:/tendermint" -p "46657:46657" --name=tm tendermint/tendermint + +docker run -it --rm --link=tm tendermint/bench tm:46657 +``` + +## QuickStart using binaries + +Linux: + +``` +curl -L https://s3-us-west-2.amazonaws.com/tendermint/0.8.0/tendermint_linux_amd64.zip && sudo unzip -d /usr/local/bin tendermint_linux_amd64.zip && sudo chmod +x tendermint +tendermint init +tendermint node --app_proxy=dummy + +tm-bench localhost:46657 +``` + +Max OS: + +``` +curl -L https://s3-us-west-2.amazonaws.com/tendermint/0.8.0/tendermint_darwin_amd64.zip && sudo unzip -d /usr/local/bin tendermint_darwin_amd64.zip && sudo chmod +x tendermint +tendermint init +tendermint node --app_proxy=dummy + +tm-bench localhost:46657 +``` + +## Usage + +``` +tm-bench [-c 1] [-T 10] [-r 1000] [endpoints] + +Examples: + tm-bench localhost:46657 +Flags: + -T int + Exit after the specified amount of time in seconds (default 10) + -c int + Connections to keep open per endpoint (default 1) + -r int + Txs per second to send in a connection (default 1000) + -v Verbose output +``` diff --git a/docs/tm-monitor.md b/docs/tm-monitor.md new file mode 100644 index 000000000..85aacdeed --- /dev/null +++ b/docs/tm-monitor.md @@ -0,0 +1,123 @@ +# Tendermint monitor (tm-monitor) + +Tendermint monitor watches over one or more [Tendermint +core](https://github.com/tendermint/tendermint) applications (nodes), +collecting and providing various statistics to the user. + +* [QuickStart using Docker](#quickstart-using-docker) +* [QuickStart using binaries](#quickstart-using-binaries) +* [Usage](#usage) +* [RPC UI](#rpc-ui) + +## QuickStart using Docker + +``` +docker run -it --rm -v "/tmp:/tendermint" tendermint/tendermint init +docker run -it --rm -v "/tmp:/tendermint" -p "46657:46657" --name=tm tendermint/tendermint + +docker run -it --rm --link=tm tendermint/monitor tm:46657 +``` + +## QuickStart using binaries + +Linux: + +``` +curl -L https://s3-us-west-2.amazonaws.com/tendermint/0.8.0/tendermint_linux_amd64.zip && sudo unzip -d /usr/local/bin tendermint_linux_amd64.zip && sudo chmod +x tendermint +tendermint init +tendermint node --app_proxy=dummy + +tm-monitor localhost:46657 +``` + +Max OS: + +``` +curl -L https://s3-us-west-2.amazonaws.com/tendermint/0.8.0/tendermint_darwin_amd64.zip && sudo unzip -d /usr/local/bin tendermint_darwin_amd64.zip && sudo chmod +x tendermint +tendermint init +tendermint node --app_proxy=dummy + +tm-monitor localhost:46657 +``` + +## Usage + +``` +tm-monitor [-v] [-no-ton] [-listen-addr="tcp://0.0.0.0:46670"] [endpoints] + +Examples: + # monitor single instance + tm-monitor localhost:46657 + + # monitor a few instances by providing comma-separated list of RPC endpoints + tm-monitor host1:46657,host2:46657 +Flags: + -listen-addr string + HTTP and Websocket server listen address (default "tcp://0.0.0.0:46670") + -no-ton + Do not show ton (table of nodes) + -v verbose logging +``` + +[![asciicast](https://asciinema.org/a/105974.png)](https://asciinema.org/a/105974) + +### RPC UI + +Run `tm-monitor` and visit [http://localhost:46670](http://localhost:46670). +You should see the list of the available RPC endpoints: + +``` +http://localhost:46670/status +http://localhost:46670/status/network +http://localhost:46670/monitor?endpoint=_ +http://localhost:46670/status/node?name=_ +http://localhost:46670/unmonitor?endpoint=_ +``` + +The API is available as GET requests with URI encoded parameters, or as JSONRPC +POST requests. The JSONRPC methods are also exposed over websocket. + +### Ideas + +- currently we get IPs and dial, but should reverse so the nodes dial the + netmon, both for node privacy and easier reconfig (validators changing + ip/port). It would be good to have both. For testnets with others we def need + them to dial the monitor. But I want to be able to run the monitor from my + laptop without openning ports. + If we don't want to open all the ports, maybe something like this would be a + good fit for us: tm-monitor agent running on each node, collecting all the + metrics. Each tm-monitor agent monitors local TM node and sends stats to a + single master tm-monitor master. That way we'll only need to open a single + port for UI on the node with tm-monitor master. And I believe it could be + done with a single package with a few subcommands. + ``` + # agent collecting metrics from localhost (default) + tm-monitor agent --master="192.168.1.17:8888" + + # agent collecting metrics from another TM node (useful for testing, development) + tm-monitor agent --master="192.168.1.17:8888" --node="192.168.1.18:46657" + + # master accepting stats from agents + tm-monitor master [--ton] OR [--ui] (`--ui` mode by default) + + # display table of nodes in the terminal (useful for testing, development, playing with TM) + # --nodes="localhost:46657" by default + tm-monitor + + # display table of nodes in the terminal (useful for testing, development, playing with TM) + tm-monitor --nodes="192.168.1.18:46657,192.168.1.19:46657" + ``` +- uptime over last day, month, year. There are different meanings for uptime. + One is to constantly ping the nodes and make sure they respond to eg. + /status. A more fine-grained one is to check for votes in the block commits. +- show network size + auto discovery. You can get a list of connected peers at + /net_info. But no single one will be connected to the whole network, so need + to tease out all the unique peers from calling /net_info on all of them. + Unless you have some prior information about how many peers in the net ... + More: we could add `-auto-discovery` option and try to connect to every node. +- input plugin for https://github.com/influxdata/telegraf, so the user is able + to get the metrics and send them whenever he wants to (grafana, prometheus, + etc.). + +Feel free to vote on the ideas or add your own by saying hello on +[Slack](http://forum.tendermint.com:3000/) or by opening an issue. From 34e6474ad9808a9c00b349dd3d9568f0a4ea9116 Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Wed, 13 Sep 2017 16:36:47 -0400 Subject: [PATCH 26/92] docs: organize tm-bench/monitor description --- docs/benchmarking-and-monitoring.rst | 144 +++++++++++++++++++++++++++ docs/index.rst | 1 + docs/tm-bench.md | 64 ------------ docs/tm-monitor.md | 123 ----------------------- 4 files changed, 145 insertions(+), 187 deletions(-) create mode 100644 docs/benchmarking-and-monitoring.rst delete mode 100644 docs/tm-bench.md delete mode 100644 docs/tm-monitor.md diff --git a/docs/benchmarking-and-monitoring.rst b/docs/benchmarking-and-monitoring.rst new file mode 100644 index 000000000..676f2bbfc --- /dev/null +++ b/docs/benchmarking-and-monitoring.rst @@ -0,0 +1,144 @@ +Benchmarking and Monitoring +=========================== + +tm-bench +-------- + +Tendermint blockchain benchmarking tool: https://github.com/tendermint/tools/tree/master/tm-bench + +For example, the following: + +:: + + tm-bench -T 10 -r 1000 localhost:46657 + +will output: + +:: + + Stats Avg Stdev Max + Block latency 6.18ms 3.19ms 14ms + Blocks/sec 0.828 0.378 1 + Txs/sec 963 493 1811 + +Quick Start +^^^^^^^^^^^ + +Docker +~~~~~~ + +:: + + docker run -it --rm -v "/tmp:/tendermint" tendermint/tendermint init + docker run -it --rm -v "/tmp:/tendermint" -p "46657:46657" --name=tm tendermint/tendermint + + docker run -it --rm --link=tm tendermint/bench tm:46657 + +Binaries +~~~~~~~~ + +If **Linux**, start with: + +:: + + curl -L https://s3-us-west-2.amazonaws.com/tendermint/0.10.4/tendermint_linux_amd64.zip && sudo unzip -d /usr/local/bin tendermint_linux_amd64.zip && sudo chmod +x tendermint + +if **Mac OS**, start with: + +:: + + curl -L https://s3-us-west-2.amazonaws.com/tendermint/0.10.4/tendermint_darwin_amd64.zip && sudo unzip -d /usr/local/bin tendermint_darwin_amd64.zip && sudo chmod +x tendermint + +then run: + +:: + + tendermint init + tendermint node --app_proxy=dummy + + tm-bench localhost:46657 + +with the last command being in a seperate window. + +Usage +^^^^^ + +:: + + tm-bench [-c 1] [-T 10] [-r 1000] [endpoints] + + Examples: + tm-bench localhost:46657 + Flags: + -T int + Exit after the specified amount of time in seconds (default 10) + -c int + Connections to keep open per endpoint (default 1) + -r int + Txs per second to send in a connection (default 1000) + -v Verbose output + +tm-monitor +---------- + +Tendermint blockchain monitoring tool; watches over one or more nodes, collecting and providing various statistics to the user: https://github.com/tendermint/tools/tree/master/tm-monitor + +Quick Start +^^^^^^^^^^^ + +Docker +~~~~~~ + +:: + + docker run -it --rm -v "/tmp:/tendermint" tendermint/tendermint init + docker run -it --rm -v "/tmp:/tendermint" -p "46657:46657" --name=tm tendermint/tendermint + + docker run -it --rm --link=tm tendermint/monitor tm:46657 + +Binaries +~~~~~~~~ + +This will be the same as you did for ``tm-bench`` above, except for the last line which should be: + +:: + + tm-monitor localhost:46657 + +Usage +^^^^^ + +:: + + tm-monitor [-v] [-no-ton] [-listen-addr="tcp://0.0.0.0:46670"] [endpoints] + + Examples: + # monitor single instance + tm-monitor localhost:46657 + + # monitor a few instances by providing comma-separated list of RPC endpoints + tm-monitor host1:46657,host2:46657 + Flags: + -listen-addr string + HTTP and Websocket server listen address (default "tcp://0.0.0.0:46670") + -no-ton + Do not show ton (table of nodes) + -v verbose logging + +RPC UI +^^^^^^ + +Run ``tm-monitor`` and visit http://localhost:46670 +You should see the list of the available RPC endpoints: + +:: + + http://localhost:46670/status + http://localhost:46670/status/network + http://localhost:46670/monitor?endpoint=_ + http://localhost:46670/status/node?name=_ + http://localhost:46670/unmonitor?endpoint=_ + +The API is available as GET requests with URI encoded parameters, or as JSONRPC +POST requests. The JSONRPC methods are also exposed over websocket. + diff --git a/docs/index.rst b/docs/index.rst index afeec1d94..1820729bd 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -36,6 +36,7 @@ Tendermint Tools using-docker.rst mintnet-kubernetes.rst terraform-digitalocean.rst + benchmarking-and-monitoring.rst Tendermint 102 diff --git a/docs/tm-bench.md b/docs/tm-bench.md deleted file mode 100644 index 2476ca11d..000000000 --- a/docs/tm-bench.md +++ /dev/null @@ -1,64 +0,0 @@ -# Tendermint blockchain benchmarking tool (tm-bench) - -`tm-bench` is a simple benchmarking tool for [Tendermint -core](https://github.com/tendermint/tendermint) nodes. - -``` -λ tm-bench -T 10 -r 1000 localhost:46657 -Stats Avg Stdev Max -Block latency 6.18ms 3.19ms 14ms -Blocks/sec 0.828 0.378 1 -Txs/sec 963 493 1811 -``` - -* [QuickStart using Docker](#quickstart-using-docker) -* [QuickStart using binaries](#quickstart-using-binaries) -* [Usage](#usage) - -## QuickStart using Docker - -``` -docker run -it --rm -v "/tmp:/tendermint" tendermint/tendermint init -docker run -it --rm -v "/tmp:/tendermint" -p "46657:46657" --name=tm tendermint/tendermint - -docker run -it --rm --link=tm tendermint/bench tm:46657 -``` - -## QuickStart using binaries - -Linux: - -``` -curl -L https://s3-us-west-2.amazonaws.com/tendermint/0.8.0/tendermint_linux_amd64.zip && sudo unzip -d /usr/local/bin tendermint_linux_amd64.zip && sudo chmod +x tendermint -tendermint init -tendermint node --app_proxy=dummy - -tm-bench localhost:46657 -``` - -Max OS: - -``` -curl -L https://s3-us-west-2.amazonaws.com/tendermint/0.8.0/tendermint_darwin_amd64.zip && sudo unzip -d /usr/local/bin tendermint_darwin_amd64.zip && sudo chmod +x tendermint -tendermint init -tendermint node --app_proxy=dummy - -tm-bench localhost:46657 -``` - -## Usage - -``` -tm-bench [-c 1] [-T 10] [-r 1000] [endpoints] - -Examples: - tm-bench localhost:46657 -Flags: - -T int - Exit after the specified amount of time in seconds (default 10) - -c int - Connections to keep open per endpoint (default 1) - -r int - Txs per second to send in a connection (default 1000) - -v Verbose output -``` diff --git a/docs/tm-monitor.md b/docs/tm-monitor.md deleted file mode 100644 index 85aacdeed..000000000 --- a/docs/tm-monitor.md +++ /dev/null @@ -1,123 +0,0 @@ -# Tendermint monitor (tm-monitor) - -Tendermint monitor watches over one or more [Tendermint -core](https://github.com/tendermint/tendermint) applications (nodes), -collecting and providing various statistics to the user. - -* [QuickStart using Docker](#quickstart-using-docker) -* [QuickStart using binaries](#quickstart-using-binaries) -* [Usage](#usage) -* [RPC UI](#rpc-ui) - -## QuickStart using Docker - -``` -docker run -it --rm -v "/tmp:/tendermint" tendermint/tendermint init -docker run -it --rm -v "/tmp:/tendermint" -p "46657:46657" --name=tm tendermint/tendermint - -docker run -it --rm --link=tm tendermint/monitor tm:46657 -``` - -## QuickStart using binaries - -Linux: - -``` -curl -L https://s3-us-west-2.amazonaws.com/tendermint/0.8.0/tendermint_linux_amd64.zip && sudo unzip -d /usr/local/bin tendermint_linux_amd64.zip && sudo chmod +x tendermint -tendermint init -tendermint node --app_proxy=dummy - -tm-monitor localhost:46657 -``` - -Max OS: - -``` -curl -L https://s3-us-west-2.amazonaws.com/tendermint/0.8.0/tendermint_darwin_amd64.zip && sudo unzip -d /usr/local/bin tendermint_darwin_amd64.zip && sudo chmod +x tendermint -tendermint init -tendermint node --app_proxy=dummy - -tm-monitor localhost:46657 -``` - -## Usage - -``` -tm-monitor [-v] [-no-ton] [-listen-addr="tcp://0.0.0.0:46670"] [endpoints] - -Examples: - # monitor single instance - tm-monitor localhost:46657 - - # monitor a few instances by providing comma-separated list of RPC endpoints - tm-monitor host1:46657,host2:46657 -Flags: - -listen-addr string - HTTP and Websocket server listen address (default "tcp://0.0.0.0:46670") - -no-ton - Do not show ton (table of nodes) - -v verbose logging -``` - -[![asciicast](https://asciinema.org/a/105974.png)](https://asciinema.org/a/105974) - -### RPC UI - -Run `tm-monitor` and visit [http://localhost:46670](http://localhost:46670). -You should see the list of the available RPC endpoints: - -``` -http://localhost:46670/status -http://localhost:46670/status/network -http://localhost:46670/monitor?endpoint=_ -http://localhost:46670/status/node?name=_ -http://localhost:46670/unmonitor?endpoint=_ -``` - -The API is available as GET requests with URI encoded parameters, or as JSONRPC -POST requests. The JSONRPC methods are also exposed over websocket. - -### Ideas - -- currently we get IPs and dial, but should reverse so the nodes dial the - netmon, both for node privacy and easier reconfig (validators changing - ip/port). It would be good to have both. For testnets with others we def need - them to dial the monitor. But I want to be able to run the monitor from my - laptop without openning ports. - If we don't want to open all the ports, maybe something like this would be a - good fit for us: tm-monitor agent running on each node, collecting all the - metrics. Each tm-monitor agent monitors local TM node and sends stats to a - single master tm-monitor master. That way we'll only need to open a single - port for UI on the node with tm-monitor master. And I believe it could be - done with a single package with a few subcommands. - ``` - # agent collecting metrics from localhost (default) - tm-monitor agent --master="192.168.1.17:8888" - - # agent collecting metrics from another TM node (useful for testing, development) - tm-monitor agent --master="192.168.1.17:8888" --node="192.168.1.18:46657" - - # master accepting stats from agents - tm-monitor master [--ton] OR [--ui] (`--ui` mode by default) - - # display table of nodes in the terminal (useful for testing, development, playing with TM) - # --nodes="localhost:46657" by default - tm-monitor - - # display table of nodes in the terminal (useful for testing, development, playing with TM) - tm-monitor --nodes="192.168.1.18:46657,192.168.1.19:46657" - ``` -- uptime over last day, month, year. There are different meanings for uptime. - One is to constantly ping the nodes and make sure they respond to eg. - /status. A more fine-grained one is to check for votes in the block commits. -- show network size + auto discovery. You can get a list of connected peers at - /net_info. But no single one will be connected to the whole network, so need - to tease out all the unique peers from calling /net_info on all of them. - Unless you have some prior information about how many peers in the net ... - More: we could add `-auto-discovery` option and try to connect to every node. -- input plugin for https://github.com/influxdata/telegraf, so the user is able - to get the metrics and send them whenever he wants to (grafana, prometheus, - etc.). - -Feel free to vote on the ideas or add your own by saying hello on -[Slack](http://forum.tendermint.com:3000/) or by opening an issue. From f79e13af3857d1a3fa0606494843fe288b0a00a3 Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Sat, 16 Sep 2017 14:11:58 -0400 Subject: [PATCH 27/92] docs: pull from tools on build --- docs/benchmarking-and-monitoring.rst | 144 ------------- docs/conf.py | 13 +- docs/index.rst | 10 +- docs/mintnet-kubernetes.rst | 289 --------------------------- docs/terraform-digitalocean.rst | 111 ---------- docs/using-ansible.rst | 288 -------------------------- docs/using-docker.rst | 120 ----------- 7 files changed, 17 insertions(+), 958 deletions(-) delete mode 100644 docs/benchmarking-and-monitoring.rst delete mode 100644 docs/mintnet-kubernetes.rst delete mode 100644 docs/terraform-digitalocean.rst delete mode 100644 docs/using-ansible.rst delete mode 100644 docs/using-docker.rst diff --git a/docs/benchmarking-and-monitoring.rst b/docs/benchmarking-and-monitoring.rst deleted file mode 100644 index 676f2bbfc..000000000 --- a/docs/benchmarking-and-monitoring.rst +++ /dev/null @@ -1,144 +0,0 @@ -Benchmarking and Monitoring -=========================== - -tm-bench --------- - -Tendermint blockchain benchmarking tool: https://github.com/tendermint/tools/tree/master/tm-bench - -For example, the following: - -:: - - tm-bench -T 10 -r 1000 localhost:46657 - -will output: - -:: - - Stats Avg Stdev Max - Block latency 6.18ms 3.19ms 14ms - Blocks/sec 0.828 0.378 1 - Txs/sec 963 493 1811 - -Quick Start -^^^^^^^^^^^ - -Docker -~~~~~~ - -:: - - docker run -it --rm -v "/tmp:/tendermint" tendermint/tendermint init - docker run -it --rm -v "/tmp:/tendermint" -p "46657:46657" --name=tm tendermint/tendermint - - docker run -it --rm --link=tm tendermint/bench tm:46657 - -Binaries -~~~~~~~~ - -If **Linux**, start with: - -:: - - curl -L https://s3-us-west-2.amazonaws.com/tendermint/0.10.4/tendermint_linux_amd64.zip && sudo unzip -d /usr/local/bin tendermint_linux_amd64.zip && sudo chmod +x tendermint - -if **Mac OS**, start with: - -:: - - curl -L https://s3-us-west-2.amazonaws.com/tendermint/0.10.4/tendermint_darwin_amd64.zip && sudo unzip -d /usr/local/bin tendermint_darwin_amd64.zip && sudo chmod +x tendermint - -then run: - -:: - - tendermint init - tendermint node --app_proxy=dummy - - tm-bench localhost:46657 - -with the last command being in a seperate window. - -Usage -^^^^^ - -:: - - tm-bench [-c 1] [-T 10] [-r 1000] [endpoints] - - Examples: - tm-bench localhost:46657 - Flags: - -T int - Exit after the specified amount of time in seconds (default 10) - -c int - Connections to keep open per endpoint (default 1) - -r int - Txs per second to send in a connection (default 1000) - -v Verbose output - -tm-monitor ----------- - -Tendermint blockchain monitoring tool; watches over one or more nodes, collecting and providing various statistics to the user: https://github.com/tendermint/tools/tree/master/tm-monitor - -Quick Start -^^^^^^^^^^^ - -Docker -~~~~~~ - -:: - - docker run -it --rm -v "/tmp:/tendermint" tendermint/tendermint init - docker run -it --rm -v "/tmp:/tendermint" -p "46657:46657" --name=tm tendermint/tendermint - - docker run -it --rm --link=tm tendermint/monitor tm:46657 - -Binaries -~~~~~~~~ - -This will be the same as you did for ``tm-bench`` above, except for the last line which should be: - -:: - - tm-monitor localhost:46657 - -Usage -^^^^^ - -:: - - tm-monitor [-v] [-no-ton] [-listen-addr="tcp://0.0.0.0:46670"] [endpoints] - - Examples: - # monitor single instance - tm-monitor localhost:46657 - - # monitor a few instances by providing comma-separated list of RPC endpoints - tm-monitor host1:46657,host2:46657 - Flags: - -listen-addr string - HTTP and Websocket server listen address (default "tcp://0.0.0.0:46670") - -no-ton - Do not show ton (table of nodes) - -v verbose logging - -RPC UI -^^^^^^ - -Run ``tm-monitor`` and visit http://localhost:46670 -You should see the list of the available RPC endpoints: - -:: - - http://localhost:46670/status - http://localhost:46670/status/network - http://localhost:46670/monitor?endpoint=_ - http://localhost:46670/status/node?name=_ - http://localhost:46670/unmonitor?endpoint=_ - -The API is available as GET requests with URI encoded parameters, or as JSONRPC -POST requests. The JSONRPC methods are also exposed over websocket. - diff --git a/docs/conf.py b/docs/conf.py index 25a6bbf87..dfb6e635a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,9 +16,10 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # -# import os +import os # import sys # sys.path.insert(0, os.path.abspath('.')) +import urllib import sphinx_rtd_theme @@ -169,3 +170,13 @@ texinfo_documents = [ author, 'Tendermint', 'Byzantine Fault Tolerant Consensus.', 'Database'), ] + +os.mkdir('./tools') +urllib.urlretrieve('https://raw.githubusercontent.com/tendermint/tools/master/ansible/README.rst', filename='tools/ansible.rst') +urllib.urlretrieve('https://raw.githubusercontent.com/tendermint/tools/master/docker/README.rst', filename='tools/docker.rst') +urllib.urlretrieve('https://raw.githubusercontent.com/tendermint/tools/master/mintnet-kubernetes/README.rst', filename='tools/mintnet-kubernetes.rst') +urllib.urlretrieve('https://raw.githubusercontent.com/tendermint/tools/master/terraform-digitalocean/README.rst', filename='tools/terraform-digitalocean.rst') +urllib.urlretrieve('https://raw.githubusercontent.com/tendermint/tools/master/tm-bench/README.rst', filename='tools/benchmarking-and-monitoring.rst') +# the readme for below is included in tm-bench +# urllib.urlretrieve('https://raw.githubusercontent.com/tendermint/tools/master/tm-monitor/README.rst', filename='tools/tm-monitor.rst') + diff --git a/docs/index.rst b/docs/index.rst index 1820729bd..df1b6c339 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -32,11 +32,11 @@ Tendermint Tools :maxdepth: 2 deploy-testnets.rst - using-ansible.rst - using-docker.rst - mintnet-kubernetes.rst - terraform-digitalocean.rst - benchmarking-and-monitoring.rst + tools/ansible.rst + tools/docker.rst + tools/mintnet-kubernetes.rst + tools/terraform-digitalocean.rst + tools/benchmarking-and-monitoring.rst Tendermint 102 diff --git a/docs/mintnet-kubernetes.rst b/docs/mintnet-kubernetes.rst deleted file mode 100644 index 627cadd77..000000000 --- a/docs/mintnet-kubernetes.rst +++ /dev/null @@ -1,289 +0,0 @@ -Using Kubernetes -================ - -.. figure:: assets/t_plus_k.png - :alt: Tendermint plus Kubernetes - - Tendermint plus Kubernetes - -This should primarily be used for testing purposes or for -tightly-defined chains operated by a single stakeholder (see `the -security precautions <#security>`__). If your desire is to launch an -application with many stakeholders, consider using our set of Ansible -scripts. - -Quick Start ------------ - -For either platform, see the `requirements `__ - -MacOS -^^^^^ - -:: - - curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/darwin/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/kubectl - curl -Lo minikube https://storage.googleapis.com/minikube/releases/v0.18.0/minikube-darwin-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/ - minikube start - - git clone https://github.com/tendermint/tools.git && cd tools/mintnet-kubernetes/examples/basecoin && make create - -Linux -^^^^^ - -:: - - curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/kubectl - curl -Lo minikube https://storage.googleapis.com/minikube/releases/v0.18.0/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/ - minikube start - - git clone https://github.com/tendermint/tools.git && cd tools/mintnet-kubernetes/examples/basecoin && make create - -Verify it worked -~~~~~~~~~~~~~~~~ - -**Using a shell:** - -First wait until all the pods are ``Running``: - -``kubectl get pods -w -o wide -L tm`` - -then query the Tendermint app logs from the first pod: - -``kubectl logs -c tm -f tm-0`` - -finally, use `Rest API `__ to fetch the status of the second pod's Tendermint app. -Note we are using ``kubectl exec`` because pods are not exposed (and should not be) to the -outer network: - -``kubectl exec -c tm tm-0 -- curl -s http://tm-1.basecoin:46657/status | json_pp`` - -**Using the dashboard:** - -:: - - minikube dashboard - -Clean up -~~~~~~~~ - -:: - - make destroy - -Usage ------ - -Setup a Kubernetes cluster -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -- locally using `Minikube `__ -- on GCE with a single click in the web UI -- on AWS using `Kubernetes - Operations `__ -- on Linux machines (Digital Ocean) using - `kubeadm `__ -- on AWS, Azure, GCE or bare metal using `Kargo - (Ansible) `__ - -Please refer to `the official -documentation `__ -for overview and comparison of different options. - -Kubernetes on Digital Ocean -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Available options: - -- `kubeadm (alpha) `__ -- `kargo `__ -- `rancher `__ -- `terraform `__ - -As you can see, there is no single tool for creating a cluster on DO. -Therefore, choose the one you know and comfortable working with. If you know -and used `terraform `__ before, then choose it. If you -know Ansible, then pick kargo. If none of these seem familiar to you, go with -``kubeadm``. Rancher is a beautiful UI for deploying and managing containers in -production. - -Kubernetes on Google Cloud Engine -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Review the `Official Documentation `__ for Kubernetes on Google Compute -Engine. - -**Create a cluster** - -The recommended way is to use `Google Container -Engine `__. You should be able -to create a fully fledged cluster with just a few clicks. - -**Connect to it** - -Install ``gcloud`` as a part of `Google Cloud SDK `__. - -Make sure you have credentials for GCloud by running ``gcloud auth login``. - -In order to make API calls against GCE, you must also run ``gcloud auth -application-default login``. - -Press ``Connect``: - -.. figure:: assets/gce1.png - -and execute the first command in your shell. Then start a proxy by -executing ``kubectl` proxy``. - -.. figure:: assets/gce2.png - -Now you should be able to run ``kubectl`` command to create resources, get -resource info, logs, etc. - -**Make sure you have Kubernetes >= 1.5, because you will be using -StatefulSets, which is a beta feature in 1.5.** - -Create a configuration file -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Download a template: - -:: - - curl -Lo app.yaml https://github.com/tendermint/tools/raw/master/mintnet-kubernetes/app.template.yaml - -Open ``app.yaml`` in your favorite editor and configure your app -container (navigate to ``- name: app``). Kubernetes DSL (Domain Specific -Language) is very simple, so it should be easy. You will need to set -Docker image, command and/or run arguments. Replace variables prefixed -with ``YOUR_APP`` with corresponding values. Set genesis time to now and -preferable chain ID in ConfigMap. - -Please note if you are changing ``replicas`` number, do not forget to -update ``validators`` set in ConfigMap. You will be able to scale the -cluster up or down later, but new pods (nodes) won't become validators -automatically. - -Deploy your application -^^^^^^^^^^^^^^^^^^^^^^^ - -:: - - kubectl create -f ./app.yaml - -Observe your cluster -^^^^^^^^^^^^^^^^^^^^ - -`web UI `__ - -The easiest way to access Dashboard is to use ``kubectl``. Run the following -command in your desktop environment: - -:: - - kubectl proxy - -``kubectl`` will handle authentication with apiserver and make Dashboard -available at http://localhost:8001/ui - -**shell** - -List all the pods: - -:: - - kubectl get pods -o wide -L tm - -StatefulSet details: - -:: - - kubectl describe statefulsets tm - -First pod details: - -:: - - kubectl describe pod tm-0 - -Tendermint app logs from the first pod: - -:: - - kubectl logs tm-0 -c tm -f - -App logs from the first pod: - -:: - - kubectl logs tm-0 -c app -f - -Status of the second pod's Tendermint app: - -:: - - kubectl exec -c tm tm-0 -- curl -s http://tm-1.:46657/status | json_pp - -Security --------- - -Due to the nature of Kubernetes, where you typically have a single -master, the master could be a SPOF (Single Point Of Failure). Therefore, -you need to make sure only authorized people can access it. And these -people themselves had taken basic measures in order not to get hacked. - -These are the best practices: - -- all access to the master is over TLS -- access to the API Server is X.509 certificate or token based -- etcd is not exposed directly to the cluster -- ensure that images are free of vulnerabilities - (`1 `__) -- ensure that only authorized images are used in your environment -- disable direct access to Kubernetes nodes (no SSH) -- define resource quota - -Resources: - -- https://kubernetes.io/docs/admin/accessing-the-api/ -- http://blog.kubernetes.io/2016/08/security-best-practices-kubernetes-deployment.html -- https://blog.openshift.com/securing-kubernetes/ - -Fault tolerance ---------------- - -Having a single master (API server) is a bad thing also because if -something happens to it, you risk being left without an access to the -application. - -To avoid that you can `run Kubernetes in multiple -zones `__, each zone -running an `API -server `__ and load -balance requests between them. Do not forget to make sure only one -instance of scheduler and controller-manager are running at once. - -Running in multiple zones is a lightweight version of a broader `Cluster -Federation feature `__. -Federated deployments could span across multiple regions (not zones). We -haven't tried this feature yet, so any feedback is highly appreciated! -Especially, related to additional latency and cost of exchanging data -between the regions. - -Resources: - -- https://kubernetes.io/docs/admin/high-availability/ - -Starting process ----------------- - -.. figure:: assets/statefulset.png - :alt: StatefulSet - - StatefulSet - -Init containers (``tm-gen-validator``) are run before all other -containers, creating public-private key pair for each pod. Every ``tm`` -container then asks other pods for their public keys, which are served -with nginx (``pub-key`` container). When ``tm`` container have all the -keys, it forms a genesis file and starts the Tendermint process. diff --git a/docs/terraform-digitalocean.rst b/docs/terraform-digitalocean.rst deleted file mode 100644 index 95af507f0..000000000 --- a/docs/terraform-digitalocean.rst +++ /dev/null @@ -1,111 +0,0 @@ -Using Terraform -=============== - -This is a generic `Terraform `__ -configuration that sets up DigitalOcean droplets. See the -`terraform-digitalocean `__ -for the required files. - -Prerequisites -------------- - -- Install `HashiCorp Terraform `__ on a linux - machine. -- Create a `DigitalOcean API - token `__ with - read and write capability. -- Create a private/public key pair for SSH. This is needed to log onto - your droplets as well as by Ansible to connect for configuration - changes. -- Set up the public SSH key at the `DigitalOcean security - page `__. - `Here `__'s - a tutorial. -- Find out your SSH key ID at DigitalOcean by querying the below - command on your linux box: - -:: - - DO_API_TOKEN="" - curl -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $DO_API_TOKEN" "https://api.digitalocean.com/v2/account/keys" - -Initialization --------------- - -If this is your first time using terraform, you have to initialize it by -running the below command. (Note: initialization can be run multiple -times) - -:: - - terraform init - -After initialization it's good measure to create a new Terraform -environment for the droplets so they are always managed together. - -:: - - TESTNET_NAME="testnet-servers" - terraform env new "$TESTNET_NAME" - -Note this ``terraform env`` command is only available in terraform -``v0.9`` and up. - -Execution ---------- - -The below command will create 4 nodes in DigitalOcean. They will be -named ``testnet-servers-node0`` to ``testnet-servers-node3`` and they -will be tagged as ``testnet-servers``. - -:: - - DO_API_TOKEN="" - SSH_IDS="[ \"\" ]" - terraform apply -var TESTNET_NAME="testnet-servers" -var servers=4 -var DO_API_TOKEN="$DO_API_TOKEN" -var ssh_keys="$SSH_IDS" - -Note: ``ssh_keys`` is a list of strings. You can add multiple keys. For -example: ``["1234567","9876543"]``. - -Alternatively you can use the default settings. The number of default -servers is 4 and the testnet name is ``tf-testnet1``. Variables can also -be defined as environment variables instead of the command-line. -Environment variables that start with ``TF_VAR_`` will be translated -into the Terraform configuration. For example the number of servers can -be overriden by setting the ``TF_VAR_servers`` variable. - -:: - - TF_VAR_DO_API_TOKEN="" - TF_VAR_TESTNET_NAME="testnet-servers" - terraform-apply - -Security --------- - -DigitalOcean uses the root user by default on its droplets. This is fine -as long as SSH keys are used. However some people still would like to -disable root and use an alternative user to connect to the droplets - -then ``sudo`` from there. Terraform can do this but it requires SSH -agent running on the machine where terraform is run, with one of the SSH -keys of the droplets added to the agent. (This will be neede for ansible -too, so it's worth setting it up here. Check out the -`ansible `__ -page for more information.) After setting up the SSH key, run -``terraform apply`` with ``-var noroot=true`` to create your droplets. -Terraform will create a user called ``ec2-user`` and move the SSH keys -over, this way disabling SSH login for root. It also adds the -``ec2-user`` to the sudoers file, so after logging in as ec2-user you -can ``sudo`` to ``root``. - -DigitalOcean announced firewalls but the current version of Terraform -(0.9.8 as of this writing) does not support it yet. Fortunately it is -quite easy to set it up through the web interface (and not that bad -through the `RESTful -API `__ -either). When adding droplets to a firewall rule, you can add tags. All -droplets in a testnet are tagged with the testnet name so it's enough to -define the testnet name in the firewall rule. It is not necessary to add -the nodes one-by-one. Also, the firewall rule "remembers" the testnet -name tag so if you change the servers but keep the name, the firewall -rules will still apply. diff --git a/docs/using-ansible.rst b/docs/using-ansible.rst deleted file mode 100644 index 33e31c3ba..000000000 --- a/docs/using-ansible.rst +++ /dev/null @@ -1,288 +0,0 @@ -Using Ansible -============= - -.. figure:: assets/a_plus_t.png - :alt: Ansible plus Tendermint - - Ansible plus Tendermint - -The playbooks in `our ansible directory `__ -run ansible `roles `__ which: - -- install and configure basecoin or ethermint -- start/stop basecoin or ethermint and reset their configuration - -Prerequisites -------------- - -- Ansible 2.0 or higher -- SSH key to the servers - -Optional for DigitalOcean droplets: \* DigitalOcean API Token \* python -dopy package - -For a description on how to get a DigitalOcean API Token, see the explanation -in the `using terraform tutorial `__. - -Optional for Amazon AWS instances: \* Amazon AWS API access key ID and -secret access key. - -The cloud inventory scripts come from the ansible team at their -`GitHub `__ page. You can get the -latest version from the ``contrib/inventory`` folder. - -Setup ------ - -Ansible requires a "command machine" or "local machine" or "orchestrator -machine" to run on. This can be your laptop or any machine that can run -ansible. (It does not have to be part of the cloud network that hosts -your servers.) - -Use the official `Ansible installation -guide `__ to -install Ansible. Here are a few examples on basic installation commands: - -Ubuntu/Debian: - -:: - - sudo apt-get install ansible - -CentOS/RedHat: - -:: - - sudo yum install epel-release - sudo yum install ansible - -Mac OSX: If you have `Homebrew `__ installed, then it's: - -:: - - brew install ansible - -If not, you can install it using ``pip``: - -:: - - sudo easy_install pip - sudo pip install ansible - -To make life easier, you can start an SSH Agent and load your SSH -key(s). This way ansible will have an uninterrupted way of connecting to -your servers. - -:: - - ssh-agent > ~/.ssh/ssh.env - source ~/.ssh/ssh.env - - ssh-add private.key - -Subsequently, as long as the agent is running, you can use -``source ~/.ssh/ssh.env`` to load the keys to the current session. Note: -On Mac OSX, you can add the ``-K`` option to ssh-add to store the -passphrase in your keychain. The security of this feature is debated but -it is convenient. - -Optional cloud dependencies -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -If you are using a cloud provider to host your servers, you need the -below dependencies installed on your local machine. - -DigitalOcean inventory dependencies: -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Ubuntu/Debian: - -:: - - sudo apt-get install python-pip - sudo pip install dopy - -CentOS/RedHat: - -:: - - sudo yum install python-pip - sudo pip install dopy - -Mac OSX: - -:: - - sudo pip install dopy - -Amazon AWS inventory dependencies: -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Ubuntu/Debian: - -:: - - sudo apt-get install python-boto - -CentOS/RedHat: - -:: - - sudo yum install python-boto - -Mac OSX: - -:: - - sudo pip install boto - -Refreshing the DigitalOcean inventory -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If you just finished creating droplets, the local DigitalOcean inventory -cache is not up-to-date. To refresh it, run: - -:: - - DO_API_TOKEN="" - python -u inventory/digital_ocean.py --refresh-cache 1> /dev/null - -Refreshing the Amazon AWS inventory -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If you just finished creating Amazon AWS EC2 instances, the local AWS -inventory cache is not up-to-date. To refresh it, run: - -:: - - AWS_ACCESS_KEY_ID='' - AWS_SECRET_ACCESS_KEY='' - python -u inventory/ec2.py --refresh-cache 1> /dev/null - -Note: you don't need the access key and secret key set, if you are -running ansible on an Amazon AMI instance with the proper IAM -permissions set. - -Running the playbooks ---------------------- - -The playbooks are locked down to only run if the environment variable -``TF_VAR_TESTNET_NAME`` is populated. This is a precaution so you don't -accidentally run the playbook on all your servers. - -The variable ``TF_VAR_TESTNET_NAME`` contains the testnet name which -ansible translates into an ansible group. If you used Terraform to -create the servers, it was the testnet name used there. - -If the playbook cannot connect to the servers because of public key -denial, your SSH Agent is not set up properly. Alternatively you can add -the SSH key to ansible using the ``--private-key`` option. - -If you need to connect to the nodes as root but your local username is -different, use the ansible option ``-u root`` to tell ansible to connect -to the servers and authenticate as the root user. - -If you secured your server and you need to ``sudo`` for root access, use -the the ``-b`` or ``--become`` option to tell ansible to sudo to root -after connecting to the server. In the Terraform-DigitalOcean example, -if you created the ec2-user by adding the ``noroot=true`` option (or if -you are simply on Amazon AWS), you need to add the options -``-u ec2-user -b`` to ansible to tell it to connect as the ec2-user and -then sudo to root to run the playbook. - -DigitalOcean -~~~~~~~~~~~~ - -:: - - DO_API_TOKEN="" - TF_VAR_TESTNET_NAME="testnet-servers" - ansible-playbook -i inventory/digital_ocean.py install.yml -e service=basecoin - -Amazon AWS -~~~~~~~~~~ - -:: - - AWS_ACCESS_KEY_ID='' - AWS_SECRET_ACCESS_KEY='' - TF_VAR_TESTNET_NAME="testnet-servers" - ansible-playbook -i inventory/ec2.py install.yml -e service=basecoin - -Installing custom versions -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -By default ansible installs the tendermint, basecoin or ethermint binary -versions from the latest release in the repository. If you build your -own version of the binaries, you can tell ansible to install that -instead. - -:: - - GOPATH="" - go get -u github.com/tendermint/basecoin/cmd/basecoin - - DO_API_TOKEN="" - TF_VAR_TESTNET_NAME="testnet-servers" - ansible-playbook -i inventory/digital_ocean.py install.yml -e service=basecoin -e release_install=false - -Alternatively you can change the variable settings in -``group_vars/all``. - -Other commands and roles ------------------------- - -There are few extra playbooks to make life easier managing your servers. - -- install.yml - Install basecoin or ethermint applications. (Tendermint - gets installed automatically.) Use the ``service`` parameter to - define which application to install. Defaults to ``basecoin``. -- reset.yml - Stop the application, reset the configuration and data, - then start the application again. You need to pass - ``-e service=``, like ``-e service=basecoin``. It will - restart the underlying tendermint application too. -- restart.yml - Restart a service on all nodes. You need to pass - ``-e service=``, like ``-e service=basecoin``. It will - restart the underlying tendermint application too. -- stop.yml - Stop the application. You need to pass - ``-e service=``. -- status.yml - Check the service status and print it. You need to pass - ``-e service=``. -- start.yml - Start the application. You need to pass - ``-e service=``. -- ubuntu16-patch.yml - Ubuntu 16.04 does not have the minimum required - python package installed to be able to run ansible. If you are using - ubuntu, run this playbook first on the target machines. This will - install the python pacakge that is required for ansible to work - correctly on the remote nodes. -- upgrade.yml - Upgrade the ``service`` on your testnet. It will stop - the service and restart it at the end. It will only work if the - upgraded version is backward compatible with the installed version. -- upgrade-reset.yml - Upgrade the ``service`` on your testnet and reset - the database. It will stop the service and restart it at the end. It - will work for upgrades where the new version is not - backward-compatible with the installed version - however it will - reset the testnet to its default. - -The roles are self-sufficient under the ``roles/`` folder. - -- install - install the application defined in the ``service`` - parameter. It can install release packages and update them with - custom-compiled binaries. -- unsafe\_reset - delete the database for a service, including the - tendermint database. -- config - configure the application defined in ``service``. It also - configures the underlying tendermint service. Check - ``group_vars/all`` for options. -- stop - stop an application. Requires the ``service`` parameter set. -- status - check the status of an application. Requires the ``service`` - parameter set. -- start - start an application. Requires the ``service`` parameter set. - -Default variables ------------------ - -Default variables are documented under ``group_vars/all``. You can the -parameters there to deploy a previously created genesis.json file -(instead of dynamically creating it) or if you want to deploy custom -built binaries instead of deploying a released version. diff --git a/docs/using-docker.rst b/docs/using-docker.rst deleted file mode 100644 index b135c0072..000000000 --- a/docs/using-docker.rst +++ /dev/null @@ -1,120 +0,0 @@ -Using Docker -============ - -`This folder `__ contains Docker container descriptions. Using this folder -you can build your own Docker images with the tendermint application. - -It is assumed that you have already setup docker. - -If you don't want to build the images yourself, you should be able to -download them from Docker Hub. - -Tendermint ----------- - -Build the container: Copy the ``tendermint`` binary to the -``tendermint`` folder. - -:: - - docker build -t tendermint tendermint - -The application configuration will be stored at ``/tendermint`` in the -container. The ports 46656 and 46657 will be open for ABCI applications -to connect. - -Initialize tendermint configuration and keep it after the container is -finished in a docker volume called ``data``: - -:: - - docker run --rm -v data:/tendermint tendermint init - -If you want the docker volume to be a physical directory on your -filesystem, you have to give an absolute path to docker and make sure -the permissions allow the application to write it. - -Get the public key of tendermint: - -:: - - docker run --rm -v data:/tendermint tendermint show_validator - -Run the docker tendermint application with: - -:: - - docker run --rm -d -v data:/tendermint tendermint node - -Basecoin --------- - -Build the container: Copy the ``basecoin`` binary to the ``basecoin`` -folder. - -:: - - docker build -t basecoin basecoin - -The application configuration will be stored at ``/basecoin``. - -Initialize basecoin configuration and keep it after the container is -finished: - -:: - - docker run --rm -v basecoindata:/basecoin basecoin init deadbeef - -Use your own basecoin account instead of ``deadbeef`` in the ``init`` -command. - -Get the public key of basecoin: We use a trick here: since the basecoin -and the tendermint configuration folders are similar, the ``tendermint`` -command can extract the public key for us if we feed the basecoin -configuration folder to tendermint. - -:: - - docker run --rm -v basecoindata:/tendermint tendermint show_validator - -Run the docker tendermint application with: This is a two-step process: -\* Run the basecoin container. \* Run the tendermint container and -expose the ports that allow clients to connect. The --proxy\_app should -contain the basecoin application's IP address and port. - -:: - - docker run --rm -d -v basecoindata:/basecoin basecoin start --without-tendermint - docker run --rm -d -v data:/tendermint -p 46656-46657:46656-46657 tendermint node --proxy_app tcp://172.17.0.2:46658 - -Ethermint ---------- - -Build the container: Copy the ``ethermint`` binary and the setup folder -to the ``ethermint`` folder. - -:: - - docker build -t ethermint ethermint - -The application configuration will be stored at ``/ethermint``. The -files required for initializing ethermint (the files in the source -``setup`` folder) are under ``/setup``. - -Initialize ethermint configuration: - -:: - - docker run --rm -v ethermintdata:/ethermint ethermint init /setup/genesis.json - -Start ethermint as a validator node: This is a two-step process: \* Run -the ethermint container. You will have to define where tendermint runs -as the ethermint binary connects to it explicitly. \* Run the tendermint -container and expose the ports that allow clients to connect. The ---proxy\_app should contain the ethermint application's IP address and -port. - -:: - - docker run --rm -d -v ethermintdata:/ethermint ethermint --tendermint_addr tcp://172.17.0.3:46657 - docker run --rm -d -v data:/tendermint -p 46656-46657:46656-46657 tendermint node --proxy_app tcp://172.17.0.2:46658 From d635783d075ffe4ec73cfbc2c53eca9335eff6ad Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Sat, 16 Sep 2017 14:37:01 -0400 Subject: [PATCH 28/92] docs: finish pull from tools --- docs/assets/a_plus_t.png | Bin 13830 -> 0 bytes docs/{images => assets}/abci.png | Bin docs/{images => assets}/consensus_logic.png | Bin docs/assets/gce1.png | Bin 13143 -> 0 bytes docs/assets/gce2.png | Bin 31246 -> 0 bytes docs/assets/statefulset.png | Bin 17367 -> 0 bytes docs/assets/t_plus_k.png | Bin 18476 -> 0 bytes .../tm-transaction-flow.png | Bin docs/conf.py | 31 ++++++++++++++---- docs/introduction.rst | 6 ++-- 10 files changed, 27 insertions(+), 10 deletions(-) delete mode 100644 docs/assets/a_plus_t.png rename docs/{images => assets}/abci.png (100%) rename docs/{images => assets}/consensus_logic.png (100%) delete mode 100644 docs/assets/gce1.png delete mode 100644 docs/assets/gce2.png delete mode 100644 docs/assets/statefulset.png delete mode 100644 docs/assets/t_plus_k.png rename docs/{images => assets}/tm-transaction-flow.png (100%) diff --git a/docs/assets/a_plus_t.png b/docs/assets/a_plus_t.png deleted file mode 100644 index 8f5bc5e950cb4ee31a0983f004f780aeb398c6ea..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 13830 zcmaL8b95%bw>BEv<{M3Bl8J5Gwr#($J<-J08(R}w6HRQ}&cx2mobUY3cmKHO_Uf*# zs$Or@oU|?X#Qj(&|pZDj_%M=dgv#bfAL;1Yn1I09fDh}pA zcOz#rFd41IJ0lA-WiumFPp1(xUNA7oFIK9WKutMWZW9MPdZT|b^d5GOpJ*^J zUI7nBBNH1lAgQsLg_S)Y`DJ@IIjNN?AGrpb9HX40h?%98q?faqikG~qiI)GjBW0s! zp)+A*Vj^W{r)OeiV`pcgC1qx0VrF3cJlW}(Sh?BQxtTdg|N9{S9L?F(oLgB`{J+Qg zZ1Is>0)dX)3=Hn>?)2^~^bXDz3`|^HT>m(jndv?y=v+MQfkqy5_AV6vVGuQQF>$tX z1X?-Rlm25gGInqU@{xaP`hQHYbNnA#dzb&7rq2mu@Gx>@V4`RIXVQNJ<>dbVMeXeV zN7@CbZ1#Wk{r@C(QT22*V^B78ad34u`E;B)#lN8(xka4KjDQZ#styje|CvQaO9!BX zi=~4jsfY?YDV3a&iIx4o4(k6x$jNa_*}DLZ>`lz1MES@+Iq0pdOu3na*tpmkIanCQ z*o2sv#5jZ)IoQQmImKB;ghUxRSw;SX6?HIiwKKB^{s(LNf3d9pEB0S7*g1ZVENbR# zzKTmA34F#WH5|BW^M-*sXBU$G3I$}s$M?f=Ww|LOXSpMTZ=3Ea=d z|AfAo{b#&8e+IU_EQK@}7_*0zsF141%9$^0fUes7N9#k^gSlH%aXdLZ8CshtDR?{% zbiO1yw2+`El<4D>0u(luz$-0WegJeRsx2IXdgRBK0ANw5K1JRasF~HKbkF^b7lFer zj?8op7M}WL_V)C8uJrqZ#@6oHi`GNNDmGQ}LA*ID%d(8kB6-*H6puHYxiWcI(BfxN z4%zux%u15lVCjlLc#>Zi9@Rm&qA&mCe~537=ZVu0>wsCV`gr{(=r-xDu=f^cL3a6* z7t&K0P)idZ-huLjyq6kpAn3>Im2^j@R5C4T3njWfL`o9#ioS;`rtQu6gZ1rg7%~1M zu)~Ryp$*_#GrItB8%v&x&Bra`W6OMcNU<5v0YbUuo)6@&os0HCx%Q_}D2w;LrN3qy zUL%-00PE!H%;*1;XecUbUh^45c-3Ed_?i1FN-% zVnR)Y1IlO8F(612CxqJyAVchg46V=re(8u)j~3MC07?~3(9<^3(ogL&gB?WB_PR^^ z-%yMe%AA7Xou-z}lqRPhu3&3Fj3`K3M~;AICdB2YFPhOJe=N|bpyd}JW4?Xo%7zHG zRTfDAE)mMb{pHNI*7CYqeS7h>raZ|Jen+{s-lMUG+Y`MM>SAU-|M}o1U5OJQ&ykhCwJB_0y6r7?a^g>61au>2pqL5ZLk)k;eB9&Qyqs58@sh19KGcvlcA zuU^b6()9$ze54~yBH~Bf_a${&{4$gg{P2kT zw{?%IM2QZRBQE3nd_r!;_%J>AeP?KxMo5US1m;pW!VxaTCQ1QMbIwsqK_W7#G$OqeWPe__UXIM-fUB!ku&n15EML1@8Qpm5^s(pwo@C*cN zvS8^K6Jo+w&MagMF31vF6RDY5_$E93*3?Vey7d*)5RcAlsul;lOY3@igs3RCAMw@= zJlV}9khLu;Diy^rm`5D;MWmC{_``e1eqAUV{8Y}byGJGtmX{DR8U?tGBii+(f}|%} zlrgGRhb$AZw9PD7V22csxHd{eu8WVDe{8R6Py4I2uLtrYZ#gG0486Zd%HnL4!aTE0 z3vRANIN$NaGn8&do=C!jOI-+W!N$w!qaT7=9)p{f0xO|PuXZ!jlsE~#Tkf4sCmDo# zx9F*~(S!qRf?N;@AU(#gy~ti}1Bj3NVfgqGT7jLq%sDXh8g% z`uO$IdF-MbJj-BiM@Q<)A81rhuGITd1Jb&I%Fh}tQ7qK4IckjU@X^d=7gvd|6n(RZ zp`)amNkbZT+`WJ^`+AGa!UqJ)k~eBzdu9ru6$k|&5d1BT3Qw&>_!&JU9e8uvoKM7z zXG{i$U0Tk{_0&i7NOZf$5L+N`1@Fe}X~Ue%ts;fIT#W+7hk%Y|HV)A)(|6J&uF1yd z`A}%KY*K7{)HW*;lZDA1I6xXd{USR7C4kmLFMe+DM3l@f)Ox|rf7bgF2*r&Z@F-Fh zow@G5NzDH@LZRoE+2~f&sy*Z)@%_|r6&lXCrnZEu?#s*xb4Vp!d8B$Gk)oxJOwr|z z7!KFD4DTP`!!OBsz6Brjm~cSUFZM)>X9pg#0R#@czwHCy8QLt%Puus>e*0S)=#O5r zHZUD0w^6MczR6Mk&M)7?MWCPW>9tQSfH~?8BN)*Vcvy4UUa_e;_w=3MI_&9%B|+-C zL_M^c##l2TBhe0b%fpO$XOl!_Yk>cykvBd9#BmJ9ur(%Xc$D&%UJ1WOf*{nz%&4Ds z*Z7H?y~YyQpdEpBm7g|IRqGdq=A^_QOXs;;PzNdVP^+h7A*!@j z+{n>uV&uWWvITzq{UWKGF_(T2bx_u}L?lRZRwIID(BJJ5A*{lQLSp&(pC7yKpLk)u ze^tawhsXW%C9^mIZcEfGg`C%fT^3pl2I5GdJUK8?_Ura+?mKW~vT4)LoAbCZ(RyFC zrm~+XdQz5)DFO*{)nuy$n^Okc@)rL`$~hvj3&L2h%-HUx3O?&>edU)^R%@r!;NY&7 z6e5UGh~qDU+DHAV#)JZx_sm7Vp)0A}0@#C$yAL91k@TTi5CGygCnBi#?->^s^N}O+ zw<8LE7@u+3PG#7S(7@gKer6$n$g%0;LYMVD{VF$Vhd`&wfqrUk;)~wyH;t_76Q$xa z>pZP^_GZc#PK)J@V9$7M7atV-%)rZG#$omCCFOp#M@RNQMcD{H|19NMCSHz@C7fFK z$e1WY9K}NepY7%7wm8olX7j7;iJ&32MD7drnrOvx}IoR|d>Abf}7b`x)20z$8q z9578#erMAz3$DoEpG8Ns5e3gqFm|g<4y?cIdLbZvSes3o8w-4&Q&FhG2B~aE-o*Jg zvE^q)5M6nw88FJoU=MJxON{qZGhDT2ET;yh!g|V&mb6X%e(1Aeel9>#gJ{D9!IcqF zdL!t?cwaf`Z@28gGWEKfXm={dXc=QzQ8l{%SnRQEaXS$4>M8nZxX@VsDp(7#{< zf8*JFLaEg6Y{QYyR5mFq1iIPG9>7DT)}}?|Yp{Eit^edE17mA4w2n@| z+ta&t!3mX3>}7(TO2`$ek^@=n*El)<{>GHCzFNCkJ3Tls=iEk*SW<`CcWq?2Kb&SzxCfEOBN>C5dFvH5UmAbf&#Ktdo*weGX2?YLqZO0h}ss-^{_;?tlCf_QTJxwRiyK8zIGuf^6I zu?_v)i+0anAyv<}$x#yBvg**?Ic$;3W`jfII;Yg={QN>eQahh^&K=F`!deh2ynvnN zJlFVc8m~o!lkwOp9#drofF)BY&B7VEm4XYD3KK#k2cwB1z$Nhnu$b}=eu(te6?1A2 zDJZWAa|o)MnrWed9GR(G+uQ>EjOmcG`cVal4#v_|3deIjcHlLCnZncRw$vWh`_!sH z$RSTh=)jNlCM8Mk2s#MM-%TU{2%#;(hX&TQUa4_y268mS9ru`!!E!2)zG>rhkLM1u2nkKy@XD}ztAiykVS_&sg@(_}(+v>AfWF1YmKWtE z){NiuzK08hoZ7rUP5SrI)O09+fwSx#rEfn|rb!ign8QD3OvMTjzGfKs`MT zsuEO}_>Vt^{+11pdT}D-lt3m_<~HV%Mw!?f;r$VWR~P7Oh6HG|U##)fjQMfl4HL)Qe3uX2tA#U!3bb{It>pJVnV*%n)49Cu9z=AUZIjGb@#NTBDLGD;kD0GU`H#adz8R&+}Bv3i9 z1o+=;5qxUf_-b`S`FSe1KqRU%f|F)_1K>moF21K#%?PYMU1DgtDTLzO9!o@r!hphH z%X*MVzW8k-+?c3qGzGA`FO;+eHNd+lqR<73l>aNtA6*8Qi=&!VOLGq3?5n`@F@_!H z1j(G|ky2Iz_u1OsbwpK8zyZPLv`52LR%7=@p&gc*OnJ83h}k0p%%u@9BPUZ;Z_EyDHtI_CncXSLSA zfUvHo7WN~eV@{9J85icfh@LFTt4)Iwe>h~`0Zz7I{7Fk(TY3fX8^nOT2?d^Gq2%nE za)kYGxQL|z$Pp7gw;T_|nqiA|%93f7hzKf2>y9;u<#jQo#aW?U0=r&Qi>tf-dnD47+zJhWJt2b+?hj8~KcQLL5{N#<4QMGzLry`MJl+DYlD-1o1ag+5A> zmQF(DkqCx95cH$mkvB&Z+)=k zIdC!*GCSb!V<3$(y^7bTo07V@R78XV{vd2+^#kA=OY(&!S8R6gS@ z5A0$`j;Uaos%1CLzUx^8NFEE3`o;xn^k|p2K2p$0hzFdak^ypl8=QL&(D&pgs-fC>pbawwCR2H{jmvZQ+bOkaO92TjJnA>ke_nL z=v#*J=wcUFExmRF9Kd5MFmjUt*P|l-)xB+~?9(?4BU{J)gxez&1lz~`!WTqQ#Ww2c zg-HQYkYEayj`8&n$p91FfVo+;J%ydi*ZNuoOdm{lj^geT%)cp!^wpft?+&fUH5?W%#nw9YWR4>Q%A)ZeaEo0pNTn*Be2OPWhqq zle{-bXPxfbPT(k`ANhHFrB;_!1)|GX_uR8&3}h9^4s3H1>|Zq7|ubJd?`uN z5Ra0S9OsMezIsBt=em^Pjc6-WW;~|^scIE(3a?cq;26XP73f%=*VUE<2Xoqq;BSeL5nj*B2!<* zwLAf_@VyF>z^bt)lE`LoIOLR}VREc)`abBj{2e5$*gzexKMcs}Cdo)8F$bxb z4C3s$8_|$z3ztXBr5jw+Wc}I!{rv6jALO9A&%C3d-*b+;({&+H%U?Z~q-8|xuI(^! zO~7#LlTW_NE}K+hqM}nIzUG5+u@n4w&-=RYax>;4X{LsN?V75wmf0)B(t$qijp~ef z&iw~RIX;OL*+FdUf(hNaA4sBs2pRmn{r%m{?4`7yet; zl!}WzURjj^ziGN{bC+q)-NVZpw||CR!n2ppbL{sW;R%K?~F!iB#r^heO@7NWehz)f-2&^H#?=F0I}Uy<@ej zIMtoe+Wd>Pc&U1;-i>1})x1r~PV8TBtFv_`ldYmu{qKY^v6x~9F6qusT2e_UI*?sn zvx2|7o_lnj*TH`-;76YaA^6t8-5z=))@~Qqke7D%Q1?~Iop+XSyi9;KRpjMS(4L{4 z00GtFiAvk$rT7KWkt3Ey1Vy2n78uf1SVN0Z(9jV0tLVA8+syM9+>C5Fr@~J>Gseq3 zM!AAyLM~S%L{fqz_d!Rji{o4jRM^RrqNZI2?(42)tye7e{avM;4^S!$x>4iwXshBMY?h}q~O>ErDG0rp&*V5S94 zGcLUkbGX6ta#p`xR+1A0kNNesZ899-HC(9`J3GO1X`51uLQz%)v&IdJ;!FKJ>2JQD z8IhRn%Wmw3Yt@^}R&Hl4PaxXO^YkRIEpnOxramUyC-N;OF!hjKguF_&K;&7=sor46 z(I58Vmb^T|Td9_IOI9E-KsS#qGxpnc9U2;v&l=~h%oT)P&qP7~PlyLf+32|PA!+G# z8`|N%#jn3MJXTZqJiXg*RH^KTYtQ)rthdT#Q?+S>*a|jdN_W zzAboBNQ0=0=3VLnjPl1Me^+i!xo4Bv&9s~x{>XQD__YlA*!~5L^kYa!FSMsq11>7w zbqM+wC21(Bdm<8muNoV`V;?`F86nHYinFUFTR&3VcoEI10j9w-R`27(B>a_n0-)slv9hG4 z1sQHR&OKDKpLAJq!YcW7-uvXAFzw}DHgEcg<+8D{;aB;-KON%U8IPjNjp%tD7kxGQ znoHH0l7G#L%z-q=h8E^X(z#3bW{(=Ml2ZNe3hp=ou?IIff(q+*=xk0LhBUPrEj;kmg=V*Gu^Cs6FoHHRL`Toe#Wy;&OeiN4~DR z{fM}5diinNY!%~~SvqVV1??=mBkWM=N>3Sr9jM#yoVNt%D!6nbW`}Kj!e*jkWgK^B zI;L+(S>1tiM8VudG8~hJH&Lg;YgX>;HcR|4Q?pLax^RkJ8ATLK3;X&puS)bY?Vue} zCpets(T9Zw#k;m8apI20Xw{4lH}-DQ*dKaw6$M4ZWWHt&YHbJ#C^@Kckas=3D72xy zo|oU$#11*rq9BhoY4ff7x#MPn=6csfYmS(f@OoC(LDaze~ibsnrCPtcv94 z3ooe?8_zTiB%d=ZM$8FbMQ^(@HBSAJm1~m^Cds|%{DZjQvhi5HIlLcHiR@{ol^S8m z_h0ewXT?(r4uUFRBrIiH68b?x=x~jGVmfD>KEJAFPWW0beVr;Yqx4{TOOp(4eYn`K zrj4(Xq#Dr8Phy@Mm0HcEWDF5~_#@#Tb-VT`iOLISk);9jtvL)gsDhs7D?zsEe$_PR z`bmF(31(IIh(H6aZLaSXQ31@&-{iA23+=49zEgJoKK|T93l%}(`dz6Q1-taaGUv6r z{o?yc;`__yN0+H$^@`^ccwuva;q@?N8_DgTnFsW4YwwUzVsjyPq&@!nlwvJo2cNo# zq>lA*(rM)LM;TrNB0)n~EZL((0RYu(L?|^IzW*TPjyeJ6j=o!5;@nAHQC-}BzQ;Ex>XUJ3_Skj#zb{2Ti=Ko4 zAzH@byg@8MYE1ur5cwu(@z$Rd^)Q^=d94VZZ8@Tdn&XVEZw0O0x?hSa@XC1BvitMu zdXxcI6XL1(l`>Tq!hO@1gHvA9Bl-1~&1M11%JurCEbA6+#)eV2EW*rtA7(W)B)gM@ z%_!EOyugZPRqfPMdu(BS#O)%Vi>7oBs|~_P;qQ>l(n2_k9b_uIH4ssiVg;hga)*3W zoRjyguy3dC$1Yqud5v@^HM13h%t_6*f9OMwyliYyMetH5ue zUIEIjP+OMO0Hkyr^Wv?q_DPIQDu6PzE6uW*iD;f1JELaKqnd|75%}BGaObVDY@X(C zHGI|SyMC@&l5l5rLx4_togy$2SEpsW=5@o}!5A~#0i?gGt27kYN5qL3X00yD7>ecZ zBqw#pPhN>*7f5~$=D(H%_?X)Dj(&@G#x}=*~oYz*A2*79|2@bfMN5<7%}`K zqaay7n_akZ0)2Hr2W?qnBH=++I09*8B1>ZJd!|p3yh5uv3oW%E)pus0(c>hmBGDK! zv;_$)ILwooKer-U+ah5&)gUk@x`5Osss2opdR7)k(B!esrd5J*g`4wKJvXfp$y=u(V%XUd1(Nbn zKchkjJ}{*O%KG7QemK}0t-snw5O14KdS?Z@x^;zO%ZxDRrs2}U^X=tqh)!OOY9zIkeh%oMd+ zuUDINPG4Y|Kl9pSt3qOrK?fz^JR+0tVQIslyvPWR z9A3Gn4?=i?Nyt@GV%KbKIVHy5D+`5kL9~S&%WM1omXeX45%TA$lpckIUyp%ckCuts z;tC$13|aV?W)`e~06%S=YN19P&(!6KPyRNVwi!)vjM8w?*^~b$tZ7PaBefI&YhwYh zWj<5>dAu32*Sx0ypO$lM>E-dWLtJ^>qy3-7Ul-W0g1-vX@R6iph3M2758#|9TDM;| z%*tOicfVB)i+-zBp@94ID+>1Nhvhy|(KZS*9~4Cz)*WtaU5yz+=llnq^cYi=#1hj; z3_oj)Tb@}X?05yQ<$%Ptr)A2^qwl0+0Fv%ovU=_ScJ=-V!)mo?3@!4Uo&&P)i{f^?+iZmW@vHVfSzm*=~XP+xZX)o&;}pzLh0n(X`i!vu~|WN4Vm2>3CV3iEZ6nC@Y-c2ocu1hsgJK z`m3ONWO)_2%1^kKJ(qvWAOi+@(XZZX*dw$Chc@q)B&X=+ABvvXwU}u*{hRZD&t(2k z|9xT`QyZ0qKjP(x5Y}yqGWW`*gVG z5Eln(yTNL5ZJ)W`f6C5U|Momzs}UddT|PU0osZQ^g}29~$OaB1qf;DtJ==@m~pP7_}nxXewn|K9KS zdB5cZ_%oRWl)gKmQM*RvyC;l_^tP#`;P` zE6{06cjYoP)~j;|-mW*hoB>&a>~Gq^j4}zJPc_e(hnJi5wcVgO6l<|nT{QBPVJD^S zDWZ-uF&4T4)oGwz+gXK0<=#~E${a4hgW5#sm0TZbrUjh4WyZ~d;aRDzDcJ=3-j~y4)vW_bow1jkLM4Y{{Hcf z)Qzvn*PGOI?r29%@hHIUn{gsVYSH)2fvTTIjdaV%W%S@>7sB?xA4+2w=WTXN7sgEZ zGx6Eq8uQhqppc6E%~bT;Lx=3G1DIB$Lk|v`PL)#q>T6Pz77|e9j;$Z3jA|PD3h%#2 zHlVGrGg&T!mB1-nR*L0;dQH~)bp2!8_<+NRTIPJW>*!ZMp;i$PmQe(kt{es*XVGjH zsC+yLMjz#nLdAZDLmt%#Yfjn-M&zGC6)Kyb8hku2(xLRJkuSQ+-7Wsx9v0K)^8+OP zd6Af1mo){ums%r-Mf-;# z3=KD0Jk=D8D_&2SdA3&m2AVSn?z>^nHqz4U`aT$s(aH&6A{#2nBjjIQ8_$CeHB@QF z!vSIHaZEvm7S9#SrYp~wUYxEyrA&q5>T4zysvulBEgFLzr7|K=X&;&4!Jf88Vh2Ik z6-2|Hd6gHZQTJ*=Ii%P)(%r_=;q&3t4=v$~U4XqeHjN6z7ID3>5zJJqkY&3DBHnDD zx6C8$xj5dy<~bs;Oc_ZXk?h3sVm7y?wPil@23Ry72UOC@k&f3dU0&S``q%^=?6qeO zFf7P7saI#(3Q76J#+VBPtcUhZhLRVz@H}kmudO>1$6|6ireyY<%1QS3qeq`cppues zDnjnZO8^2PQF$Gwf*q58K-ho&F;-AW{PFn2`+QX3*aHrWG`DTJl}roZoC$Iw>2$J_ zca0UPYNrRJ^^La5TlvX*CJJU|$hD1}u-z-*fKS3a{37L~?Jgxuj!^Gvp@chA$jrc` za;h{9(AGGBCzAglnBL9pD^;WTZqG`&u+)X%YsYdKFoi;kmIJYh)d=^~nebLm8;&Y% zbaAVQP^%tm%*3pU2#QM!w<}D4alj@FLR=TeU&o3M@*LD``SFmo-koCfmJVt+1_j7# zCkX0Gm#Y4;A+hwjU$&52{*JJqWNrKEv-9pFaEyRQSvbv6BAATvtjW9?1>4;rUkggD z1H9j>w^uHS;Td`U+Od^0`bC5QrjyeL)jl-wE3<{rT4_`r?MsUmBkp;sdKDctTe{^6 zd7dE%OE*O{m2_YTe(*y+-c|jJ$3FgeW|GE*t_z}DLVloES*qmDx~=l=-D^|SOX{&E zvDfWp?9fLI(U1K0H*oH!<0L^BM}LK#+I|$4SeX>sz_lbv7Z(=Wm4>o!bgonp)?KsC zixt&vf0FV~oQf}7AiM6W9FPmy4LG#}0AGaJhJ1aoX2nRg|AcsSX;j3&qba5z1SBX6 zJ}7eljOuaj?)>A#4*QY(E0 zic@9ZlUZkKLztuPCU4N5Ut4QSEMldJaqQ<2_zkM($N31;bQ&0Sb-q*m38#VWYdVcR zMu`{qFEkK>fMO5iUQ{IW(=7^Xrutqq;kN}z!sxn7^0`n1uCXnD#HKWxN~9e74PP$K z^c+M~ZUh{-+Pnu~cfB*A>><)zOA|hOjQnj472`uF-8wjUYiDPt zIV?|={b-dPiQu!C11U*F(i!WU!aYos1B@r@aQoLKL^bR{lSu1t9Mp~*)8ZY zI{mD(ZN+K9YQ<&&W`#8XEX?-EH{yJ1u60uvD$z>I$fB6?vn*nky0X&h349=5VTe5+?LGApSh+{NeR#nC zkxYO~rs7*{mnT0zwK~m5s3v+d%9-^*LsT+&daFwhNX~b5qZ?0MEln^hfL>veLvzM9 zlO{)KUipD@A1-t-Z_d&M>&J$=Q7o68x{g?{tlAs|X7Nit9`oeX z3%VZjNM1!bAZ5Rr!T1W62U@X25OsDWfj$IH5gvAXETS8K--ET>=%As?3G=MAd&6R` z1fB|!)qr_?FR0{(AgaSu8>@dQIYA2-$2BsKzJ>rGyp&Bt3lq9*tJJPaInD-W6r9S) zMk*&ppwfduRh6twF|_jM_u1my*T&&L<(}LMRL)fvd1nQPh+%|BqF79sQ9km-L!C2| zaUa|S4ACNtI7an0@1zT4;yGqgdV>+j;}*;%$y-OS&X{o~rV^QlO0p{=vtD|05^E_& zD!=EfgJW62vgR8s$dW{?@@bcfaUgy{Kg#?&PhBKPpVXYzegVrP zb@Pp^rh~=mj1gb_D~5UGq;Rq=jrl@HrYUL0bpIlq4WtS7QIHBv2N}U*lyt3UYwYlP z&&|s6H8m>8UrG(Kb@*8qPG|H&ZOBDRJLI)!+RF)0N~{*l#zM%y(3lI^o>U@X|IrS3{8&Ao>ZtjpTxzMMYB5pi4I2n6{{x9x>~>n;ow`?C zBwl7hxM#L9z}#qB5&;1(>YD>HDnda~bjrm38OpAG?Rk=#j^MZa*QsxLxmErWWJ>eX ziQ+fV9)r9mcUvL~L=*2=bxX%$VP+fV6|L|Z@VD$3(xIwR!t{L=r~g;g|sMr(6&n>G9q zJ_X?*Xq7laZ#g7XuOFIhNh7vcpbL&cYPx~=3P^3U7SrlZG>Dc zlCSCKDS1AOpF_`)ICr4#ZxLC*BF&6?+Qtc458rL0Z#Ae!X492)Dx`#Y6eG)tI4+|B zonX*4&=CT3FJA0toc8VA^Yz1GUqK}|((UfFsX@RDdIURTgsTlGl&qoDGuG!&$>F6p zQ)vKZ0zzz3vN6iiM}*I2N5;*Itrx*EE$o1V=(Jj}G=`2eMxew6I%>q$uz@g@ijoxO z>()i)=H3hn-pHqEiFIDTLWWly(n1x4f?><6)>a7&#S^i$nzfa1R;|aam_u?VvIov5 zMD(4QS1^!8dSx!B4W~U9C^~oT_~5T#hi|z*^j)qILd5g^Ve=-rs16Paq#o|!vBiGAKNeef{_cvi7pE)6~2q#cp6#a2>kqKoEe1}12AUOpu zI&b12e2OK*nz=~%t_O(yMIZ}Ifi9`)0ErWG1=*_seoKj)UN?}$56|mZKI;7v-BPy_ zU3;?>sw1#HD}2dDIL{SK8%BSk&Jw(VRC7VL7ro~W%w3$dS+rMmG&nbKM+1m1)P=03Qu8AFUTXp~l=NflS+$Ip(Pdd#IDRx;yj% zYQK6ly}B@|BJ9qmv;I<>%O0UlsCDI&@WzL&oT!W_*mq*bd%ObcI&FWnx|e|sV>4Q$ zFkd7R9HuE6&R~Nr7wg?jTCl^O0H3Dw<4X{$mlFIt8sLRBfu{4Ud$X6*1}+RNS+A%J zF{!$MpFn}KuFU*)RwvU`B==rx97vhhMR83T-A|dK%CK|ug07DRSKxIoLjztu_QG1x zg&1Upf`IOf$^Sz~elsg{G-dmN9t*)QdgzT<8>e}@NuqWmkO%+41*M<^&D>lTP_Up_ z>Y3Y1th}APA17mYOb7e*3q}BoQ?OQa-P=_uNsmj05iB1|O!AQsvG&m0ZRK-wX%T~{ z))OL(UIr=&U#Jfhk|M)Xd{gPZ$)a5VtqM;lfllsJBAUv=3`0h=7pCQcc>n1U1H%wu zElTLnS2>fPez_`3n=^xifXxUoo_A(cNtD(NM-HsHC2S#PL` zj5dhtcGlc&5Z!GsZ(b)Bn=y;hY~|Ek)!+sy-RJPro^tELqfe9|@3OmG%lEenziJ5* zY0%-(_q`hl{9*r)*LmdGeXPP_ImAJDSMWBy!+V$i6mLddbHdxQf%yt}O`T_Ye7-@0_vDO6bD= ziU{{&y}&PsDv2-r4~r`3{-c5$^)P}q<$Qju7d9ltOLCAG+Q_q(AZ9vM(u))H>VZv2 zZ1Of8nxz8O8Vg0jX{=qh$7fz28>&k+m{|Ar9Y+M~b1Tq1^dM2O?fOpXnoomv?9f9A z+0_xR761C>$P95|W=PPf5Vhi8>a?GS;R~_v61irY$e9;K5Uc_c?^kg61FF~uym))E z&G9yEHfTqtG};Q)yrxENrBk_zUz@$T`H2SmK$KN6X$$QU8{hH;mkS=}y{D>MP?NM7 z&KX~FVE^ou&SI$9xxghsjn0}^Wt-u|huamkm>v))a>~v{lLQI$3+a~mqC?H`mzHsG zP<(XAaN^9_?_;mskn_e%ToD0!&Nq>1fn?F0B2ligxVVO_dQ_I#uRZA(>d4XYi}3;w zMkNWO6!l#=5$4ELt0&8CgBQ4TVo*~Jr!nz6o5%*25tDF|EK)B8IDlAE%P>Zz5Y3Pb zycy1e_P;!HlV+|BH6Mx}E>ORtqU(z)dmJvpVSc~S^uB*-e8)tMK3@%Dw)wX9B%1`e zx{H&D9=!YaP5rkCHN|i zMGErazid_eo5RWaMO+49rJjhJ2B^W{X~MIekg;!AVwZob{#x$KGh#a^fd$MVTQ=c= zbd-0~rio?i?~bl}63N)N&pg)5+EJBgH}DXYQ?)C^o;hg_&n4VA@EPr$`a&%%o& ze@n8gpQ^5^jtWkeP{*Sx5cvD=ox^&3I0$L>$4Fx+Y0M_~11A3EUg(fb?S^z=j75(5 zK)l@!-;O@H!Y_50DvJlkv4-ezs|%)uiop|Lh5CY9*HS7IAN|Y2V4YSqvH?uF{0%2X zJyPA2j|qt?`wQz85z=6*uU;^MB z;*k6a2ZsqqP4S@)!ek?hFf(@es=Muw-J$2My!t!U18r_L_D4@^oE3|}-27U}c}eV6 zgt`7ut(xcZJSp!S$Gpbs*R3zhj|POV*7{n6kNPIl8oid>O#}oc&nEXWm;LLumi^k6 zrN-tL78V|>ffVKbeKC;*JCeigDOi~B{&_*rL7>;q!@vQ1?ga_Q= z#y?*N?gnAN^+sbC-W4nT`^YK?F5!DVoiQIBG6?RqSMWAi6&Bk4NH6-|N4{VNZg23R z>fpowJYkM3!Hg(AwT$xmU2w^I~VRa63U>W_^~7nWbxOC)$nNTF=1TsP z48K5Bftk@STjRA~1CHlza=x(tRp#WeHuAV3V7?<#x7PmIM5BA*o>pQZr)TTcr7vf* z%=xy$a_6U|bcYhZlU>jAW7o6Q;&R8{BzDg+tJJaIsirT-s%^Zy-AqvvkHw}IFLQJ* zO_)c@jMsL7zGY0gwBOQw52rbzK)L{<_(nD`AVbhHOnRpSimPz@>hjDvzvSJcR8Kr1 zjclpSqgG^OLZ{b^|F7pYw&tBt6z(&?C0lh?Unw013nI^C`$Zf^_^7(P4us9E+os(6r3snCjff|--l-cD>z494Pd5|x8y9J@cdyI@Ue%72 z8aDMe&QO{9nJYz8)~=N`^%33SO1R{I>~-h*LJX#6gS;`rHV9}HM6{gmem(r5K$^AR zAbWLQ4HX5)AFTZ{%#!fPTxpm^5<8wB?~LRs+~f>WO*Qc;qL+Bp<8KNqzq8%10#ir* zvRBQq)PmT^$~0k{=NY0-CB{vjx~0!WnhwVuoVw%aJ%^v^jRBor?!XH?i%wqLT|O() zE|f?_A>`mpbr2Ps9RE7$lf~DE)4p4ceq;g><&%x>Ir}9!0jEV%uHE#TM-eO_68W_l z;XZLS81vmn$z0~{%Y7cmB$Qa3jly;UBuOQXu98@w`3J}2^cUaLMcn5?!L{XWm*cMU zW2XKG&p5yM9Bn$?ARuvJi4oFfthVX*v_h@Df1HsS#F7(mar!_Rf!COA^II(N@}v-Z zZc$PFvFG|m`lG?bhpBn>N2|WvcDmV{Fgrm`Nm4PJD!hKS#6}~nYhj; zZK=QEvx9QLRk=L#m1F5x%b3YlRq1{Dvo9Y%`JeSM$_%Uzmi^lcwB{ig2ClhK`5~s4 zQV%g6&pzt7V)J~DHv1}owb!_pmjKZMhlkuG{ZvtH=8Fbq-spuP3T7 zRKDRil!(tx`X?ns$k~Fho8!5+4hw0cNG^LaFsoCpt6sIg zGo%fNaI^1YZ>IcDx=Z5tJLbGn@bjI+lkKtH-(QWqR8a8hveCK7>C;W8xJbdl9L*|-7o+YvK#Wj)m9z&V zBZ!bo1Hu!wayg#)=>dp2pXp#hC zIv8Mw@ysRXN6!-L+aSMH-K5-r&{7Hggus>o6F}9qY#6x8c73m?LC`W&B_`6$U-Cd$ zN;pCsB5m0fLEUj11%+F>Y=by)-S8OtJ98n3An{wIxM82?XbRzJ z^R3JHnCLK34<}2#chaALU$_|<2qJ9=@#wEGZQN|ZR>x!PxuHnmuAr7Q(Nma zF|v{MB~>6tvUls^Xr}(BhO9KO?oP-e!6LWY%=#&ik;}_(iGi82LsVF7-MJ|QArIFh z#DFAG#E#np%+s@BOzf^w;_`_XT5BVv?%|J_7$)kSw>qHxNF3~qZbF}65L94K3FT^b zv!6&$o`&qjKiyS>$Z=g9rBix6+gq~ESIOv&`=a%oLkZc-+98vT#3#;vr5P+B-7Ffx z^d#t!8cX1zlJ}P3ypA{uU!XTuQNDp`MJs&YIA51ZCh*e0?Zl6{*XNNoN0OK+1hhS0%9EHF@Pgl!+04Df4O zl_?KIaQpt+AiL`u#9Q#dPjd)aCp~|f8cgZY;}3M#7tnx)85RXKM_itzi-Yh6I&vZE zX;DES?e`y_NXJ|YU2^HrSr|Bu+O+9#(QwY3MJj$PS?CrZPT|@qK9K93e#QE^l>;>p zLlfz&wqjEFrX9tY<9X2W%da+af`McG=~UI$grU; z>Q;4Ju3z4wK>~1{_A{Oj|6E4~3?J(C zRV&A#@J^DUiD&Vh0}u-j`to$Gyn^Ald9ROn&BvJU+VC*p2`UApPL32)9mBwi&>3ze zk>Cj8_nJzOd7dGM{Ped(oc~;{MrLLT#r9mmzyS(~?Asv`tB0(ZJHx~I*hslS5G9|! z89vdSiZ3a=#Lq~`(SR&86z9<`(I9tt{r0(ZapFH>t>l_CRR8!PJ zT90rB6cHvBsgo?3fT5Cu6TVN$xNUfpNW0O>(OGIb(9F{?JCwGOEsThmj3sIXqnbdUS1raTv7#XbrB0)&J8 zNQvFh(*`?0x7OJXurw_uJ>mm|)ZC?1?;7_EAWxj7eGg-7TWJyvfXJ}{$WM(|Yu${2 zlbfH`%7T=Tyz+JOp+stwH0>^cxc0m__+`_Pue2fEDP>>G@f2gbRzKyLA-(ra*mLi@ zSko2|a4i7kUr_lLYCkU)ODk5FuM+>fFY|%Ig+WI-?S0#X$7mQPwqBhj!TSI9(&2yi zl3ke0JDTofv**o+2~$%xwG{m5_O64E7NNT~_2tW(wKmF5H8Ww)9&z8exJQE*cy&=< zI@f}C^{y{V(h!5hcRE}jqmihz3XPHsIByWK{qY&l)5|1Dk26FK3ZG=H$ID*2AlXEa zcBlJ)HjluZtHMcy3PrX9dAhBBC#u#N%--L#B!59l5gaByf7s7}1vEu;Y+#c`p#vE% z*6X(){%BR+a+nU%CbED0O&fQD^vxTYE{Lc$``*3q=BJz312$15OWY4!lj1hC8eTEcNR5pNSt_S3eK46J9 zfCX41<-H3kNtN+G6;}W!YBqT~In1>B%Y>W?YHf_mpZa2T$ zY+N1(vG zu}6oxB+DA0u3YA806TSfzSrO67vhZugr+s1cJdLe=CvK)^fJlbZ;^KQdyEn*K~kLx&_S|QT}O_XIZeLDJb^}P_7bXO7AuQ@~kM4*V|9u@u>N5gx0ab&v8%_!=qj|}IA*v!}j&IaF{ zM3O9w0b!}s>gyZPfO7<|LDdV@1?p^{E7!zIhvV(JE(>z^=ju!Kw#AGooL>lCs6i?* z<;K2l4|iZ>+1+01PaiUI;A?M*T5U86(mL6u64iMeRf0%==81%toPD@;I5}nII5on0 zxY}*#s6O-2a~;tv;&<-c^huZ5!Q1NT+Cd+OqzhNyX$`MK8{f&VuKr1@nFGnMBhHc* z^=%bjB
  • *rS)KUbg?5=%DJa(NYQtzsYA9*u${WHgeZUe>C9OKmOL%b4;Ivw6l)R z%L3v`&yDBH!)D|*LYyrJBZe;r=bSl88Kiy01$yS%!?VEh5OYVSOqi(e!K#(hc#UhW z8TGA;(}U{y&gdpJU2Z|#2BricMnHdLEU%e=)Or^ME!n=D4Iw27!Mj1Tuvy(NQ~eiO z4AXlE+YfP_?f=x+0G5_0N%mb4d+~O5&6WhtgXp(&Gbnr?Zq4bfwyR4u`8a>VWJYtv z55qyYH;iP!o4n=CgOTit6a+jL0?%K!?J$&VFaq9=W&o^k%w(%y`d$XovOe&+!tx|^Vi8q`L9Jt9E@W!AGni)>?=;ctbVI^ znpl!zgDIo1sjUiGfk#tbPbwUzJ`G3uof0jK06T9}@wRdx;9eC@FMw_v8@^@K{ixGh z8|gLe-5r`YV6y$PB0|ozmPR@<$RLQI|G7c_Rto;*hnHC3eRtPR2jSio~2DLk39-bjDr^6Uz8Bw6y^qqEd%Di1dU$@%8fw9%Jz@TU5oZPXL;u7g=fAA_mK^kZNBroby#XJg$_uCNhTM}(UHjcyogF*s6&=bK)x8V%c26R!XD5YT1BE&2uJWBc9A~via(1Jp&t{!+u_KEDXk@Lw6dd3Nj<(@q7jql+ zGoHlY^(`Vc^fpW7VC;YzBNohdfA2@G!o7VvAc#E?H@CGka;MK5uaAQ*l6|0_0>7XX zS#dz{+^7vL>6?VYBOZ0JqonJSjDFYIn27NrH;?+>SPqHXID5oY@xPz~`g#41ESINN)& z!SxA*WxjDljLmuFM=tJzpU%ywpip9Bq5TJn?9umf<18^<_e?R#Nhkd2tIAf9f_SK9 zd`S|E&o~rCuWX(Tvq2r3xt=Bz8WQb&)*bh5Y50_;$Ka!ds#Dic)WcJ z7axijttkcinI0IY|8-e+yul>JX({K-X;E)tedM%|L+9&B%H9OvmdPv9(`8z)47R%1 zNOl3{^e~NaYtvR8_K;%Hs`_r)j*nbnFB{!{lM8W{L_d=y@g;vPqx^uk5P)-6mZM~w zNOR0$7-pSa5trFqofG<56-klJ*k2uj$pY;2awV}6ERmJj3x%Gw$_s!w`r$+8EHSI0 z6;)3mMo53VzV%d9?n=@N(ysF%c=+O5RCro02i?IBre@5$y z0Q!R-_N%xiZuxh>f9AVA?bU;&@EeZAYS$QMC)UO56YYPKf^ML9VglxP(c1<1^Ke9- z^Ctk}O|f8$%Uf4ByOGA?3;J(cXL-@stQlKIK&kQI(Nzq8@|fov?eBQVCPBjT457WZ zfTn;`JX)$_q}r@k2`};Lk2kw3>rj6ytznqsu*`y3m#!u* zJ4B)X5j=|8sic9tPQGV)SkH8W!;2(F4jD=KJ2#xct$2}ooB9Wsn}pV@lp%)OOjqA)H9*rCYqx;19 zoxp73!ILTy4?1E|&l95eX8uw|C;U9TYN1j5cs~nszyo`-BzTpmzrKz}D@TwY%{1b4 zOpqbroCAfU#yos1v-k~sd&;ERg7&D%?&-BDV>*z3TVN0eC2Zplh ze7>3&n{c-sIO&OgF#D2l_iIpwMT6_m$H4i~XECxd=DJS=InbkoPnkP_amLDV7T_zS zLu94&;BM-{gpi~V*BEnoF3N1Y7@MSW81tPle`V^P5`?xanB(@!XW9b*K8uHB4eOKK zEmMe?Ym|#HZvcHvEdXg(H2Z&5GVmkye@lY0vG~?@JRe<_iQsBTXFYWLFt_3=)Q8zIMM2DSeRK}R+NBS}EYgn$_aF)Jz#(+GF%k0#xQB=z;V{07 zwFsEhD!D7Xhl-e9A&f+jQWmc^)HXER{?c%;?jglIZmucI3vNrGsIz`F zL_8lJ!T=emA-=s2W#K3R5Q z>g0RhcgjYv&^=J*bTZ|io=U{mEMX6UR>@av)4NMgj2Y~990$@lp~SQ~f_Uy_XkfSsxBGsOe4g>1N&)m>_@Ce|bINsqY;>e5YiiJrN(b z{2oMbE-Z_ncu&)t6=_FGz1YeRlvTdne#wo4K>iAt}3wd51wsp;Ytjcxhdw3#*Ui7Mz(Wm_ z?{+)Tfa{+DWT+<(1X01w^UJ&$mET=k&TV`*45a?K&>EcMHmnC8vDQ{Q?&%p5q1yRvPe0n zh-H1&*7{^PTGK-n2EeN-vTKYB68wr8=JM7c^g!hXQ;*fnp+E(cHZ9=a+U-;M z&yr;E(ETKJk5%;dOd9skxKayMSf{RkQO!TMb3x*;5cZSjaVOumKp9Nb%?|Xct$Pypey=VAlzrWh1-s&~y#6Xo*d&}df z^8PHz`e!Bj1ILoOuR+=+Bbi^W7hDs#!RLTA1e7s1ug(X})$DLD0oO_KK$=E>u9G@n&r_{Ns*@E z(l#m`+allZWZU%-bV##U1z?K zV11?_W52!!%LNnJc@di_MCPPF0oTG>75-kB--Y*KwDP1Ml4dfMkCi z5ICSjWdPJ}Z!FOaBs9AyYKs5zVGl%n!!@kt?<1H_nd#$8k->fd(xPsjBBAf$n2oj( zcjEb6ozuj6JE?3!!fN<_h?4{Xc{~^$Mx6cZRl+bVTECrB7KRu1_pLeO_f4vt&&_mzVp`y*1 zp*0$tOmh+Yp$Y(EH1u14PqzXP(YRV6@jx8$9WF>>FXshUSYRRW)N!B;I^|kp^(C1Q zjk|LM?+85K34d0k)p@UZ8w>Khl+|zbYrFvJ$D5X?TED-fKu-q&KwxK*tOy{lsU?Ta z;#SJ%EumU^K<%p@Ff3WgbIgEcyqbj0`n#zZNSgsg{ZUn6tV*wa5Rt|L3REkGBs2)NrgrVFUQ?ks@u* zcFplm9>wOLfrbarX+1l>f9%>8aIp@cMR(=L;(#{Y7_DsYilMQ5)e2z9QGni1d9Wl0 zFaIrab{PY%C#%vGK#r0jhTQ&O#k||-NrKySwt7acp((t2eSfR}#l}r_f9vyIY`Xxl zn*fd@tcsqMnY4_T`aIoA;dxetdxJo~)_%C;X}&Tcsnm-eh}Zc#P=GUy}6AKyiJCq);8Wt;UzBQG4!4mUi1r|3Oj?#r5L=sZRd8wA^Ej?jok-s{rbg4XmRyoLIFq6WFaxx1LrL#5WF^=bv{A zf}e;36!ZenL3Q@SJYqZ9KC4A>ZRKX^58U4id+OtI_yg^Ji3O--&6%t+8E+3T%e98L zlyEH*;(_A!Uj*9p=Y7Vc2leXT(xGp#UbXf2$r1 z1E(eh&Pliz`>}Bdfii7A(*cJE0EgpT^2Ndov^TgdNdTJ6x{na36pGBnyP-ScTAj2K z^cUYP_0=@=>iNxm z4XKCOJKe-!yYiZ^HyIG0$7}2s#`z*I+tERq7MY*r6g9o=uqeVVe@35`+yv0iGUFyA zS@ZMRfHY=i+t~vEhD^|VO_J%r=J;O@cyUuw= zrs-(fcLP68fHX;q72OPzVnIrklSfaDF7PG2Q;DJ>)&6xvI^G(xbErFUZifwxypxON z*#xF)i3ELmu$IGy*1u(K@&0LoKJ)wBZCd$5h)#p*j!yGV7cEfq5d5`*Wbh zn}*a5%J(G*%**sj`3kiY92`-IB8cxn-V7tzv{KZ%JU^ii2bStEP&T*O7_W7ohq#75 zjTHRyW#{i{7$6h2NH%xgj@+dM)LZ6F_^g%GH@- zz4D-wcWRa-N!-OHC5tQsTaC-v`^~kR2aLf5t3j?Im!I|Ju?aSbD0~Xt?iru8($^$1 z9Cup{D=m~*FdZG;EFhEjKt+3>X1o(PK(whS4W)(JTC2fN)d)Gdo>MlwY}ybN6RlJY zg&aKQi>c@dI(|rpuMY4BeDW@$)w_q{HD2KB=$eQ+i(@a>%$v_l!OqE#%`_7_bV?sEiSW z#NQEpSc(>Y-H+i^ zv#0l(!wsE!z0=OycY0*4K|d|Q?-h1L&s%$lIdoef($wec#HULc&Ln8~r&fmd1t4gc zL~NMLZTSbl!ngTb(teyK)OqXBPkCNgSDuXKKA**=oeP}kzGDm*Ng#+U581(!K@~vd zgd`M1YVIF`1_mI0$y5tTD2O~rP|YVc;mUGo&cI>|7s?;`#Am0IN>bx4OON50*%sw5 zvsYRwB_!2Iq)JprZHc-YDF_1RzImgcGskZp!6GD22B&_+;G2EpDHUIvi=>@;IisX! z$sV6M^I-r?YK^JQ-V*b>=&I1H6YpuUegE`3H$14Tzri~X%06a$FA z3K?-W8W-zxRWv;_FF}~A1t}y+f%n_ezn>ZI6saCuC}<+kmN8W{2MJxR{=YK^BUGMLSPISPuIZ%2cZjkSz4nkui*aVrY zKFA*na6cf0nm9DK0hJmL6K6`371lI%b{mv%7|d1FX?)0AKwK__aQaA-PS=!7w&dgP zgIo?iEK>pXf&pSj3htN(#O(^hw`P_20~zU&52}Y0xcU=~j8B$>e{{0aItuV)Yp>!d z0EbkCB!mB+S1A&@M>e+*Ne&{9{zhliIWOyuYZ0~+;~M9%!)7jQp5sco#14Rn{;k%t zmp1tpKfRC4&%pBE-RX{_Sk~XwY{TBA=;zT}*<+GjR@!U5OFESjr#d=7(Up%^G6Q!h z;3eZ0Qq@$!i?m7IR}U_TLN#JIb8V<4E|HNb8h7t0$EJ?%1*v?|gW=Os3#r2fo(j^b zwlaI0JlN?QNF=nJJGl_s!KU6?y`-&=eAI0yZ?jKc7(!N-sDI7N zDBzdtufGVC-gN0;$QzP;ddoGEx)VoFoe^(t2=^kVMFkNFX3cN=X(M=s1#BbVIoqWgL`J9~>}vuZ zJ|H)->~f-X@{4u9jQ@N<0iR~z2>43fad9E7n-=O+ki+@_(;<&Ht*P;&8m0=?0%jOW zfv*tX$=l^dIHLM1c{f@auh$!`e{6ns!~e$F>&B4IS9#Ri;JHX8@i{@2{h^P~zK%U? z&1R$AKE55VS<(N%Xf?}g+7(HB7ULpy%H^ASEbO+Zy3p~c?t|F^OW$aB&$3$92vuA- zH-9I&M@|twl(a_@t?L|J{m^T)>s**ZJ69gxM&I&>@QK^o8@Jl^VCDs|Stw-~ zC-^Cf3fvyroA%?r6irDKGo3h1yfz#*WYEhdm{`3Fw=8(N@a2Wo9ai4w_)um$YBI#9 zZBS+2!*&}|C~pNK6~+h7nPrpJWj>cyd&M(asRv#NRZCdgPKUUkO}h6tHXAr`vJvFI zPBycsMfdf+oud)W4v&;(Mzlam*e;b(-i0KmjJ(;CK-}GZwDypSnTS>j;$nV9$-B_- zEoCrVPTXU5;}0vzuox&_R5h^hqFSi?_%nmDRC&z_5UGhD(9wk)MTLS_1fP;1|L$?7jmTC*KEX^_)-wP z`0h-!gTrFfXr#B}i+r*{9Kp#nF2PfOeCTR+1?;vcQyuh(GZvbw@PQgC5=-3<9#9Ig zVy7|hWD5@NuovFRM_4wHGMl1SE0QQnk&#_WoK`8p!w<8`g2Qdi2isi~Kvm=zb^e%h zXfYWz1M1HJZe&QOp$bxjg@BZ^h@L z7V%ftF}siAneQ&#&8K0FTgJF!%qi9Xs4!(p$Z-jJA_^_E)W>Cs`6HZTJVZ;XB)W4+ zdkHx}l*Q&HbQX(0ueN)td{pJ-yek(_89T-e-6iiCd$+e1ol(`xiLG1oG3^bffLm&+ zq^OI@Z)Nb?igS4p9Oa7pgc={Qf?jNaQ0nUL1TZD&gDjtN45e0vU%g1%q5?F>1rBT6 z4qC#@l-xbo>l1UfJ7pM})&FQyo+uvT2=>49giLg9AoZ)~A- zu&##`mwAMhiUrA3yBiY$7P_!bEly2?e|{{$L=DJ-~lj!gO8Dn;QDg{W))j*uIN%y#ZK6R`hK`)cRjz5g`@THwu2T0Rsp%treE18Xjm zm+2x7lf1e@hVv<0zJC@xt9HPHTuTpi_qfFKiIS}br-&bs)i zbo@WyHbSlg;DSmd?}Yw=NB}B!{CoB{(9;5PJnS5OKK675%7Z<95tmUie>U<$o~gc=Np{> diff --git a/docs/assets/gce2.png b/docs/assets/gce2.png deleted file mode 100644 index 358dcc04b36585b1a891e47973b3dcc7ef5dbc1b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 31246 zcmdqJ^;eW_*#4`4bPOROIYTRw0z-F6$$(0?Aky7k1JWQNr7(&pBArrF0}>Ka(jZ+* z_xGCTd7pRfKVYxD*WN$4Toc^))%SUP&f}bzM_MW*gm(yU+_*uarmBdzapUG8_~(R= z3*O0G4^6&tgXM;rqMV+O=~nKoy4ydGuV`jqZ&^sN!U()9y}i9@xCm=3S@kDoZhGO> zz^Khb9^$J;dx&1IR<>LR2DkGIal&f{0+08nS7-eP`Vzc9H_cX$%~lpoZ~Xl9G&>do z!IGE7$G-`Ev_a0riQqdF9K8_{^6$HF9K{U;4wMDwpErENLR6)+af&U#m;ZgMSRL=b zuPzf{!44(KQBlx;zbr3{;${5r(n*~7$kBbpq&rdn-7UGn&WJ@upkC0)h1_gV>g=%*D{jhQG8_{+o zjcM8Kdrzq1nr!-CdtDxm8;zHmd=5DM%YHjG=g5A2_?^hJUMeH!ro$4anFj97JPGg8 z{yeE)BGWDkpG=$ldN+mYSxTLf8z!slMjZYuc8--8R0S52A zuu7D&!D9|6%MCo5tQk_KpX_E1b_hH_s!nEAnYv#)Z8>i0@3=1eoY7y*y!FN}@b#@> zb-wEJokqTAjFt*O$lnKZ|ua zDfK-+|I^V9mLzvX@X12-XY~wTuP!>p;#VZm9_z&sp3-?;mxf(~K_7A&F$Fuf&e@3)r>XFw~BU+WQx9?5e`|{ON zA)2(hE1tgb4Abf`Fmz*ciiGLX0icm0_G3JoKi-BLJw(RNUR@rWZO*l{yxhC1_rY9Z zr>-zJ`0DgYz1y>lf$ho4qVg~E{i1Uo0~$qIxei~Njw%$$q&)5$H(y`*)8tt~Y+U$PaPteTdZyS)Fje^? z!B+vSuM#Z|$Xhty6efuGl0S^UBTO6}((eNy+}>%}EG?bYDE)oLuAUB8AdxE*ASxu~ zPSQD9D@-)b4{WoTu+0yQyTpdv(jvAWgYZXuZ#YR-VX}KkeZQjIs=MLPBZond7A(F~ z!-=m)Q})V!3>7)jfepfrQ=^~T!+V?>w^vHL#o&KBC>hrs>VKz;1YP)yx6F9Y&72)> z(lloAJf30VEO;zPq?S3HC}7n!0lKhw7P}kpa5W&5OhRK=nW6F)va z{1a$3IC{$8eczAZdgS2YTslRU8COSS#Dm*7;ue*b6xhsLS(R$vOg~)qo_#_8(T` z6ia!-yFpAL9HE{T1QxY~mAbXZ;m@bPpwSGA1*IBae*59?gHx*(kICl^%LdzvNH%2O zY~ZoUQ$Fj4c2Wub&xX|uleJDWQ;roIGmW+1<&%_rRDRex zXZ=3SfR2(%q)Vhc{3v|r!QW{R`op!r)Nh@Ab)qjzs9R|Ooq(SapXAC;xi$7`bn^R+ zt0a--guEXr>Cq<%adCXE>&J58xb<6vbuwUn5=K=LnYJtBM>U^3`{h{c`BQcyF0qEi zpMg&#BK?)-+w}o&6Hf3vSb!$!A~*vBz7cT33lm(mYm){#<|(#u3nh`^s!3cKYy7IC z4psZ>TGE^kIpkp4MRY}~sxGm^d^u@il))2+UBPcaWm9seVZ{So}BI0G& zm1ErUSrsTTP7&umdTJKC^d250yu%h57`oa!K2$B)V@@!_zAB zkvYq}wiKmhrY~5PBeTs|L1O<8d9PMQelNDHn zd7o5s4+PX94*bM4>6|PEU9A<0HUD&@4k}QArTD&09w$!W7Ot!P=0E+It#`NeUrttW zm`nZi|?WRN6r8=!(G<9UwBv67)s<^TiK&LE}X1 zFW^)<{$Ohlk^dw3!VLQYCfMW3!8 z{`GI~y&0!JE*AH5XLcOoEcf*-lEu674<3BSJ&^fW7MfV?aCW|3JJ&#>?_j8Yx1Pn5 zbPxJ10)lyhRljC^VG|wXvMZ;Tk9apX5;J-!E-bOP)HQI&ecJn<(c2NYA+y|hg(t}4 zu-CX*+Bl|vo(S_4zUIXow%8O|@sNPS;KMO|#QrAAWO7Nh5dfDW^=UvGYCX&}*S z!}2XFeMp$^;R{CdZ1OSL6Gd(&eyGU*k=16nUMKh9#1{m*^L4 zQ`actdzMB{?pODf64yBxQb zHr8K+mQsRnq6~r-E3P!kWn51>071!bro_LIRXmC17IA83=kD{q9NFky&^BHlhIs#p zn1O_Yd8e~)Fwe0CS9#>uAP^#it!A#{Eo=7D$rt>1dKPfDUL+B=AK4!Ohw{knZHeDn zjS|@B`UM_}uMn8+^TvO98$?l_gZbz3*1j{Iug{fOiO*y{h{J5DeD$dV~v<%r0^q{Zvi4tfTJM#In+OI z^1ZdZY?iV5jC6dzAAfc;|Cr46MJ21#*-hwa{F5MTW^0!Ti|G2?bH3)2f|qM9HpmOE zl5Tlk2-)TlMAClAHS*BFWvT2=4!$Ze1(qnxDsRqK7m9j#6xMyyH-|LNC*p2#$Io1D ztspfVM5n=m4jlhB!j|sFDzq5&(uAtG@=rX83*?K;SNxg3-ziW%Lo%_uoIa^y(BY`M zWUC42ezJ6+RfdE=CmOkTAtk8L49|qSEAEW8Q&4JSmqjb0Ows8Ne}e6mM2!_GzmdlR zWr%}zD5E+!?hXIaiFSoNleeUOI3`Q~;Q4$(U;#=w{?2V;4oa-vWy84}o25^UT_eF$ zzc;W0S6Jk+>sg%yMVqF+Y4S5=Qyf?Kd9Y-D>TYGY_OL zSWbIDXn!JUAd4Hmiw<+aV#r=fDOfT$(($y7rPSGwI-Xt4bjf!)ziCv9s58#vW+mOnXEw!JKe1gIDZV#d zglf=@7ru~6Amnxkrqz0XToi7imz#6d-DE8{?3>MaPL6)=66n&yVl0|P-F)1Q#%;m# z7IAQ6l6NuS9ue5|!ze3R3+SUYl&L=B)29CVBSInXkf1$iK}8XVLIAdi(g zBY_%WTx68=DZ?!3sVY0GUk;V=kBbq{a=U5+PTl)?; z5*KPZ&F@j)E;#QpxSn+!X+F$XQ{P^cUfLQlU7t(!`fYP`I`4aQCR(}ur?g+F{`07b z1PE0d_Xi}lhg6tN3U~Kc`YZPbW#&u+4<9S+&iX8oNGwE9mdHmEzH^G3^njCQIBeXF z^CvXo$PYRzt|Z7X^|KqB$rp2fKLet9OdQ?_0%XT_64AKNpNwjqynv)II6vCHf8sqB ze|=34_tGo-jr1z{f^(fE91TZ4KIXQ)_cSX| zz&dZc>CjKatM?jqXxmU*l^@Ml6~FPgg7Mcl*9Wh|7nL&I_iK_V(+#7?ewO@HzRnoR z6mX8b6byJPTxvF1VfDP_-R(CO!5op0U~@lyyUAN z&epH)B2=!dkzUiTDj?QKemVbI)Op8xsz8b8xR%ctGnP}+_3kDfdW~M&XoGfrTFSub z=Y{cvrE1@(o0g<^2rQFEIhNz{3R}<}8IzT^5)NxJe)PO~qcN6A?YQ&~^BXrL3S^d7 zE3M7d@?U@j_(;=-X-{c5C+2njG$m)ZhIJTE(D}A#5nh}(AR<~cfpV)+#B`R4l7EpV z9z#l6PS|*)9&XZGW$g}I6XjYN;V182kD%9oXP9}pO?i%+FnXocohB=8=y*Byrg3Z0 zG?B=!Cb0_YS6IiSZhQT5Ts_lD8U`A|HhaFxKKB2iXLEteqr zTb|!W@vZwXS}OWNWXhb|_xnzN_rxB8G0-kbJD%~*7kY!L`KlAN*M^siNPeR9)@$+^ zjW-|pdF7z==2WZwsx${)EWcG3vH!)N?(KFm8IPDP0_5_Sh8C-$K&ln=G6(i^Pjasq#sr{ns?*vM65?)zZx_>&kt68 z$M(N=jlK2sq!)87Hu1QI|FZgIDB|BnXjJ9#YR1pT%71%v@~E#{XsZN!^Vd-B#>5YS zmzHX|dcs{;ODgZf9|$M6YRg<&YfBxb30czKChmEbVX|`OJ|MPQcP&+}fDf7A$aX;R zXQ?_bu^g@qwf7JfTwR^-=DOoR&67^^W6*3}5D;%Y_xf;Wbu?ar)r}#m z9JRoTD*5`=JpUZytWnw!KtcEfE}1ksS-sDq1>VWTog!Hi%9@U*zKaE;@4#$HZWH?* z?=+=bZ2hYJa!!hk6=a=vG~;bHUJG|yHa{V@%X+t$Z%nb;QUs@8|%N}R1?Mq$vztKM~ z3V!Pw5|FZ@F?wOG)!8Fpk|pZ`Xo9Cmf!)aa0Wk$HQ@-`h95MH}Hw!?ycumqW1fYKd z1?kyu0b8iemN%eLN;-g65i9U z(e725;VR65RTy$PKn@#m0nxUYQPfhex2qQ$D?FF&BB{@yta3@w_qFe=f&+D{t4H!R05eNB1Y7%_ zu56HqPMbge^yK!JZWf?>yIW;bXj49fEzTD+x%Ua|*pKz-!0>UQLT@j+`$oDAG7k3^N?QK z-nzEwK_}DZ)U@G)ll<{4#SPozt8wY(Pw|hxAZDWy47PT*y{+!iOZgW6nc-}jYS25A zz+Rd7pr&5(>krB2?+TO-jhY{Sai3o$Uhw(*gn5d641Iavy)8M7EBRyKb~Fbag+a@1i8IH`nyEe19QO_vGZ)m#(Bht^?lqkr&^!Z|DuwmES(S-UUM zIU17y6wl{yS;vRJ>~pCKW0zb-h<$V27t%^l91j}A9+1z`V1W~hb-l)N9hqY(-WNLy zCzUy#b{}ApjfGUUkX^n>jAn2{vL@=ezZM@NXTV3MpC&z!SyYu8D7C<&lL>5V>snCI zIyeTcc>uH8dX_o5jy`<$^G{*B`D3)TOD7e(tNRf8QDl_v)qye& zp@+WiudAIdWD?!~R6MRR|8v*>JU!mA#ff!0R6X0E(Z=4dP(A&#*2~eh2tAI?#5Vbk zt_UdNxcC9`H5XZ7mvv-`TD4FXKMO4bF?nk0)~#zR7bqMVEpPOt<*FguhYFwE>Lodw zABHiTEXO8CKj$Z{A~(KgMHfjtC^RU!Bzg9w`K(t-Ak9OhH&RV1v$VV1T!=A$$hHCt zE6@L4GJ?{2dj8J&0oK1}XQduWK*bnH>y%9P!M?ozH81Y?oybBkZ0qm^11*+1l z;zUIT?R5~6JP;ctD5A^wza;TZ@)sQKDVOZ!s4H#!vnG<{l17)T<(0$wV!YQ}zoJb~ z;u12{sfzJLd*xnN#W1}SDr!gNy}%qeDJBZkaJw}b1LXsVTf2cxIJ4qyjzg{-S4kI- zw%%P%EF%ZAq|wv^{^I;3@iRG*m-0>4ijvTgrxo1rlJzNq4XD53qtHv1MGP}Ep?5uE zQX#fYaP^~qCwzE-o;G`>i-_hy=EXDz$bF#oLNa~pq-5*YgnBB+#%@+ZJ&AcuCAz{u zP-gByq1A%NNtzn@fSaj|PcZ}KABGC*-L~3#KSDn_bNPNUkVe$g&hudrFYDH;O(TO; zk}>cHYJmgaUU~g^T)6epU!;66y07wSbEaL>ptIo*TvBZA3wOV3`&Fij&~1o*@M?jF zTJGWFbRK@DwlE!QOG@raQ{|fy96R9Alz(L1A2F&{NN$%>Q`G-P4#|q`vH0q#DYpxS zo7#q=JFU|2Xfx!n9EXx;?Jkg;U61@YIY^C$Qx!!aJ|}jGdX9@f=6z1|1P9Xik|fBv zX=Ml6D7qr+@v@S)-;lu#cy2;YDf*-H36Zcvk5!Ey>!wc;oRtc5mlYYA0satpGsIBpfg~l_<*6vtE>~XIri60SCbPw&ji^l#t50P>ka2snjd|u>UxMRnp z{}EwQsH|DjGLfi6&uA3h_nu+<0GW3f&YR1h8&`s4ocC^Mfr-tPTr$UQ>jh(5R7Q;xmo@TKC`*r z7vmqyX=Y?*NhgbI7^@T<=}%VWH!tQ-!h{cXcW9SHdaIqE{4~9?OuyV)#qzIH7s`HM zRnPd#s_MMaDtkl3bIgri#H)O}vfuBhm!{I8BZA#)hMRse_SL8DI;O(U$G0?R8ZJt6 zuWJ(hYN{Km?{@42aay+5RVMVhb#vwiERH>IBapYvr9)Ve#HVq|pzoQGNm3%Bk*yR( z2~uQ=Tmj1duGXy$e3aQ9VuA{#AVc>=dHuNVLsI`^MUNzkA@Qy;=YhYvcqvL>S|g>f zlkTru9NZ-^5`;K#SvF+|tDS|o!tBX^a`@xBJhzMPyQL$s*+VNg{Lz1_JldTH$+FJp zc}vtqe#2U*E+0Qu_~{CbThW_t>v|Ure)OIBWvt6SW9HCr_ajF^V{+c zB)HvS=X*QH=BT`Z9MV+rYtH^eI6}L^enJ*zoZg7eDO3FR7vDg(D}}c>wcJ`NtBD3j zIY{P~IJK#6;)nu69Q@pf7B!H13r&spQZs)K%PYExTkVJ`0G*e7cD&S)kouElQQL2e zU+C=n;3xMx#n|N8vHHOsCuQzg{iKSaR4}jnSnJO;ozlP4bf>*1lf;h&-tOV*IE)pj z-b!`r;F>28ieV8U#){&L9aSSb4;r#%z`^HyF5BKX=YN>HfjpUg`FbkkEm4VgOYxIR z!&{pZ4Go8vFVc;U^G@aj8xPw4I6alHU%U6Dq{fuyfh+wImqXp=;fCI)$Aw|8IRl49 zPRq^NjV|v@_LtetZHFoDnU&_`mOY&b>-Vhv%HN4gYGmHT_uO*ChA(l3tNT5)f;KLi znS(mj0)+DeE`K6!+L^+hA3b^FRYWIbeS zRHon3ZNmjxdgs=;{tuECh7>K-KtNl%T=N=%cu$mG*lmVEOdN*_qQ)&g4`km8|0dSG z0(q2_tX6Rxl(Tk&w!E)*n0i2(l&_&6-t7;m+JM@OXe3srRJK;yKu3Qk6slpG#;_D! z{!B`wt}RNq#v|XZ$n1do0&XOJ#gJHZ->iP>?$bEwZKt-B$4?3$`qpwV^X}7pt}y&L z9#r$%WcTsiSG9z1zv%ma(GhMaHJ_nXUxpYf@6Axk3vKR;=adno)-P7pXS?SKb;%O2ho<4J#QcQl=`$@`HmETP7tZ^&*~{|}V$wcptL>}aO@^yKXsf*a@WtHA3Ny(&55@B}NVD$cP0g?4 z--bdI-{_@^V_R6^b9;Z)7P+OQq)Jf_;j+iy>$vG2+8zB+!Mfe`bIEZ9v0Se(jltJ| zR56{q{#FxKHgX@yF>xspjfeUc_+2ldgk5#9S>>aBn9=WH zC}HJ)p(ffNLqPc~Ngn-skzPXJPN{&|#RI#+P4Jb6x3F3+kFqh4Bqs3w(^BK*W7r`A zYUzpi-|v5V`bA4jFWK)Q(j)Z0p0812);RsOu)QJRG$r1<-lk1SrN{omidge*7*C4t)ZXb zQc|BjrgkO8Bvd>88mpPKp~uk8t-W@#!H;}uqOUj@F$mVgZ~dLu&&N+EZL%HH5s5&w zsz?CQvAtC}IPgCW*F`n=aAaeojV)l5HGv@Gls@3608A&2lNHaNGzXkkF@yr#^&KeV zv7D4MAb71Yrir>j-h6-~9j4Q0wlG~GGc3$Ds%XzK60Ctax zZ+A1SCIFV1k>nvp9EaZ{5%1dX?avmi-I;0pRo=~%oia-IYuqf@5o62H%$JtBC${l^ zQRz)zlKAX75FL)58>8hI^ailiGmYLlkn)aDzB5CARG26|_3mA1{mLDQgC7beuVc{+ z6dJGByk~uj7~RlN5$_%2M&P^H#_%EuQfJDZjcJ3>X34msJL>H0t|?bqyFZ4K^;Z%lO*}ltwaH9cv+A z1Jl4W@UCMrIfjAv>z;hJh@)X>p9}~X$3=VDGmICfz8F@2Y`>1ZS6UV?GKG%MM>_A# zw~>;;G99|g+HmRXcyQ0K{+Rf0SKm5OBH>p%M%ICp`;BZN+k_dw=}Mhl-PEzZXl9Fy z6=^dUI-~Q--{!lzAybWJ8a#8aQfnfAxZ*-R4>mA4QgI=hK3Y*5RV>R%Ae$WlD&ar7 zF$RFT-6Cy)M_l>g*5nAdrfAVV-`Bmc^FQpdqw6#VT+3x1>j+V8^c4H6;R37T=Mm44 zx>{oF_Ys`!ayLBo`$daBKAFpq#pxw3w!)98`(q4!G~B6n3lWM^Eer-9d;O5Zn*>IY zZNLx8OuT1)(#fII=4iX&h-lUN<5690VdDNzUz#`K=#=CA2gNseyzk{(p;iX-cafP_ zmR!C|(-zT;>8cw^9LbW(p8?w&uxQ2jq4tk3EE^E|&2*gS*9Rlo-s5r~4`QlpUfq4X z1%d+i%Lzczwg5Kwj@TtqhIEH$Qqist%3M!h?h-Av*3AJ9x>NY%vthlLTKn)daBGN7 z8~hDHw^@&V&>2`;da!q?qr&(MlcsII)lJa7I2U{!e_Q9hTPs}mi|9pC6*G3=!Yx_F zCCD-`t2|A5)%G?#B8+`RAcRZ{OZ^TBy@CJ%~o(t0pZ{)YoceO}et(2ek1Ox(Z>BPihi{8gr1_t;A-?x+~C=49n!J^~Z zu%gFRMsylzimU>f566stB|#N(gpM#t{k7pu4XWZLROtP2jMrvhvQ;hq-aEDKGJ!TR zF}G#3@R5kM{6^m~bMVE`BvuadtKpZj!$OeHW7g`^#3q_OTm( z3t&@Tm^tM%7^78Ub3KBhWnd{24NtngpyPP~Qq*`Qs7Z0YihlF_+_z<4} zu4Mh?>)(f=?a?Dq7vawLVZ#F+#BfPCA%rV|%sNGH(Ie5{_2cvWFa{Sm|5qKiPbQ5u zF9*VIt&&IIV3c0Tv>Mw`PGd;v>T@r3SYi7VK^a{tKaqqrz??vF84}Xd@n&~T)IBLo ztDsP=f##i6z!Q)H5OvC7+vqCZ+vkZLg}PbyF?2#dGYbpHz^rESeVN5k z)UpGK_HRce2yn`?-gyTZGXHTSdsKqk=UOa^x`1`6c_E5^deuP^Hk`nE2=Y&22b5m-i@o-Np_li-8ZWM7po;rDv;=ZZb>DpUMB5K{a4Jb0lZ?t=UpM8bNW=okW>@2PN79SybEXV=&$39Sf96yWS-N7y#S z>OBaiY?JK+r=p8;*0Y;PJU8flEj@4GGpql=Y4@$dCf?SxB`9#UD(T4uK(jMvqQ?}^ z6fzY>6hFD5wC0>zF1=L#Zsk1N0$Zo$`xAZVZ#6LY~D4s|!3gY~}RYm-Lew6QkCJlEnoJWRJ2ELb+1VugVu$9?5K` zyRvH?yNFD@+8RF+_ZHujpP75QS7iAlK&AuK~8m$QR+JIR`KH|C{X_MW9<6D!n z_=$Xw{Rp`m*tG{)=)Qe6RO@U;bHBWYaCq6Tz+;NT5Ej{+DNv<`(xOtIN?uE3C)}{i zr7x+=aOGD%)0{*0@s}Pj;~^asj8O>-k;YKrE)BEE0M#*8UBsX=DJ|F3EwZlGJ8bq% z^PyHZ*JV=_%bQt;PU!!B_aKH)oKxvHs*u7plLFp1M5}0aMQjm9XfaAxlC#&M zx-u*I(NUvqiNx>bYLh>b z1e%?Zb3I0Br4fgkd+eF=^V1f52>ZkKd7XI^c8NA*%hm~i#&$k`fBu2euH}Qhz+Y)4 zx-FtZ_dt%pk<^rk%x}jX)w!rJ#dAm_pSk>HrS0HA%!*~&Bo!@BRkBtCSDY?~8}dcQ zS!?#6iY!6nZ~7;@LX6laas_Jaj5CEY5K@21;ca{4Y~Yfu1ZMB|_oLw#DF?Jg@Sk0y z=6H*H33sp)+&IM5X8NK(h|xsrymxPcd-8Cm3Oyodd*tFMJ%0G9IVDoI80JaFm@M8n zshf-!odC%_8I>3kJ1)i#{iZD2@N@$IQxK3E1rV-RVDWPa#oQ76Y&1?3%7%mTA0`8z z#8QL`%55kSe3-h+grjftOe*7W95wT!bENT`+*bO>p)2#A*z(^CKkM#PEQfDE+zr04868jq*hy#Edbqp?-5T1>0z&R8(=EU3G>rmV~d6t(qzs{Iv z9y$uO7wLMtU)5PyCW*_yBbn8XD^5Cy=-~sRPl-o6Pjr=>hfs$kapq3|zlyut8-m6I z-?VsE9d1|Kuid%d8AWq61~n6>3 z>mdsE6F+)c-%37={hB0}n7){KnzDX+sOGlx+ytLoEJP}?vcO7b*Y{0cU9+1Lj+>G; ze7Z-*Hvge*SoD$0{Uz)55bSDXNxOFI8&S!tZi1960%DkBp3>?%jvyjQoV;b1Qhe~H z6vV&G<>&?{pQXRvU!t4IXX0p~9b+dBt5M&Xw(bN~8g~}r!8`l3)PpisCvfN7)fKV3 zsn~Lh13rlo84^zLti0?5cXH9FI`@sG0cvgWqCDG0F~9T=?E0;twwY?WmT2ga-s-rm z{06NA`Ug$k%@qN);kT0Ri5l?fXkou)@~4Xfn+fL(-&hJ@!m(HLaF;WdL}}&ttSY#j z{`KM=Q%i-k-(+KN&EU8Y!eYaRISh--T~Z*9za!dBw=XHSG+!{7^ZU+I6E# z(wmgvPv#vvi|fq0+j~-T|BRWd+k61_UdWaYPih9Az#L2Khhs_pqQbLS$`B1?>}ht9 zy$ft8AI`XMMUW~!^kDU#%M0egHTM$X@5;?Cr8p`lf9v^jR6;^5@mh?!kCKJjD;q44{p8pw!4>CG#uV+uy^YMH-)K2B_Kddb(y99fZBMHNS= zGx4XJnc2U~f4Y9!wY}$H`k=0yTueDvH{YYiXISizA-mU)rQ!- zHOEK9^7NaXzXomr6hAGa-Gzm4Q2xAjPsxJb+thn{HA%eF){+G$oe8N0J>1KTc_Jn~ zaU2@1cwuxM5FCbL;>3#!nm)hvffGd1XE;!>R*pk+`E*vtIpc2@EGzQBrd$!>5Ne;_p-7<{OHs`ayZAt_vFO2*FxnIFgEJ@C0Xqw4WAD}solMZz zmwnt5(T=qq$-CJC{_tn^PsEa=Mt_jc6aZyn7lQqS{1j{SfEj+4q*6 z+h1~0*Du62HS}P5TA{HLV!S)XJ9P`z?oNJsCa$ZaoVoY&JkA*HtOjh3lk=P@!&oDLnSg;k4GZONtwnf4%Acv&t+{9MY87{Z>Qsed&J5>XBVUlKQEioHx( z)w`hji?jQ(2Mj--B2zW(zq(D*cx^_LVaIN$z!7Ao?Vy9iD#6^qp%k>*{&s^eHbT6!aeXJ49y%e{5dhJ*r*xl%jZF zpF-!hwK5fz3et7et(%1Nm-|TJ2J2%+Pp!b|$wly_kEKuvMfT+DYG>=JH&Lp%og0I6`fKESUo#v!3BqI%b-k+y4QVS*NmAy>K7vcDK~j2c-(QT; z9z%REUqO7TYZ#lLPL6SLN-#L|Up{&dJacY|Ovz41-Tf8Hs(J%#E28U;_>8;xdd3XU zF3jo6fE3Ec48hBP4_`KftqGacEncp|;8as%6q9F=R98$D_wt%3WwPOcN|{ZC4(Sro zl|$ZEh=TlSsakHl)>r)K17U96sW^A*lN`OKxwVL)Bklk`dIlwttkdTLVD-Bd=E21j zd-Bg>wxBGkx$!~Li}4cYQclDmX!XlfDQ973pbhr&1;fIr>72t_VBit;VWyV+I4`=c zS@7wQC<9}<$AneVfT3sRDAa-BDT5S~fB5KSg5!&GFKaK2e%@JnJUffj-N;X=kY#r}4^z)qd-N(b}- z?m!wde!q=_pY_$>(qw9+i5(l5M|+aO{dANed#HiDSWj-0)ugeSNN9ihx@2iB@AkSW z29n|dXgUn&BLZI$*4hvtUToTisn3{{l|`9~ESKmIVo+*?EFI=GZbS+6e_sFJ|M?rm zR6n{_GX<)D2sbb(>*Dxi$nid`|EaU8v{GR6^M8vD@u|2$qhXOo@dW=gE(F3Q38I3? z=+7^x|L2b%p)d=T%1M|5=U*E8W)C=GX_#t08vGA&mQ99%;0jUqGxI<2`%M9K(P!eL z4W`2wv#e4qm}Aw2CszM)UP3Z3rVAFIUPu0qm-hd?hJ@xMpMeU&El?X5zqL*(v1MMU z9AAMc0eqKXYUop_(=X2wjsIqFv!D_{?waPn^C=+7oj^6Hw-h}%=q~{+@&YwphJYbi z`eWeU6u|Ee7&Ghwve)bJ|HY^Jvnmw5l2>(aJW~D*1cIh3F1!fGBl9x30F>47Xlt?t zfZ+P;U)A=u!N95}PKS2rzS?QrZ7>oj$x8RE7age~+&p{sJ$({z5j$X_s(oB)TwnH@ z`>!8BTiWLs!8!4b{g*)W?)O{8`5wb+M?VzE44V879lGM^yhb!d>GHRM7K_Qe%~@P{ z4^#2v2*|P$@3_aEzM9}kh=<5AN(k446^D=rjEY}bc+x^g4CVIXOFrI{33@arhwAG} zV6-=KtbPUR8jI%7Pxn_P7Qg717dV3Yt)1fnrI$IZKfe09ADe)JwCOpWXFVzGz&YAt z0IH};KxHMs&A#*kmJvg*3-Gn zapx4!`OmIPU1RSx>F+%)oi;Q9minagfka2(YN`fIquwiujM<6asqSsNRqc$I5UdeX zwAONcWe+NnsxYMP>?8;<|57Hx2uw|$Sa-~oGoYk9?p;QeHk#wigiCaS)<_I6_8L;6 zF^DE*WRU)z!RLL+(HORHbQ~?w0#N2TfV>U>neUidFMbF9(P?iq#Ne!4$kc@q{TllH8q4(Wgfe1|9r_deRKdcGx0v7CfytuZNGYr~&ubsaGD*#5se*RuK zC1e0tBbJ&u8Dq&UVV7V<|5Hs1(ClWZKX4Qq2lLyYl-rMSZ$|Ne2K%aX(ekJkfNEO% zo*PTkWflr5st3&2(@p-~qBUSqv{mE%$qo41Qz~7-02iKjAJtKr1ZI<6h9o1W`cA*2 zt@Sv9;5m>RIRaZ+yaGV>N|VM6W4D;|xA%L2ix30ej=%!`1!^qU`9q$!;TF6xqZK+^ zV#JjR>3m5eu_f?2iS)7{aL1MbW5#T!#~I@grGAQS0?< zaZm)g%^&$SnGUXSmh_9-$YH@cTsL%!Hk%I!A+ei6JMDI|-L&dS6NNkHz<0}|n;Q>1 z1Sd*26EMly&oor$0_WAO4E`>_Z)d&c0=0BbOMKkrhv1>plNa%tNH%PRn(W%8{a?216WG zNBFQ%jH;QSFQ)w07N6y+=HASW%JD1y;E@3S8i_-*HP7oRHZe zn4@<{MQ4A=K;9W{eI&KlihV1uv4z^owO@F=?$7BaQ^1}a&4TRfI1(5=z^^kPZ$Uua zPxGG_;AlBjyMgB>`5j6o1x*VLV{DG3b&T;1CRhrkSd~DBH-yt+Pf0Gt32gxntUIVo zu}8<&aDl~E|D7}kq@L`}V?)E>z%aZb-MtV>%ks1|vO}N`O*9WAIP=OtJ$Gj>MK?BQPwE)(9%Bsix#wb=C6&fzZFJZ~Oc)}{dL=sR)4r}&7ttL6 zDz(Mi{rd#E>(rAewHV>J>L<-*TByQGX}3aC=3<2k^SHdz53hH1lh1BHHC|wN02Al8 z-7*iEWpO`P200No(h)F+JI%IQr>-`?g7@B$t`LwuGxk3S?dt)B6Wa$VzQ_?qp0zbb z@9BkuJGP8}RmRq=Mf?=m7`F1gsYkL@idCgvVV`9NZQv&7p{H0Tx15N7dye z-PabD91qXIMl$*M+Jo-cJf~nxJ5@Ee=K_pY!&-8WRuyUSVjG^ik0TWY3=8&}Bq6?m zpVsH9*<)vVl7CClkk%`XG+B^sH70u(DqZkok67iQMZvcdIiA^*l%H>AF-bEz6EhZR zK{myJHmsl|zT^h&Zpz4El`)I(CH9vg0r|cwbSitU6*$2j*hL}VDsE`qbo5u@{94t9 zP;WpD42LJ$6f>VLO@x@-MGWfMhTH$n8|~o*X&Kf@?iM-T+ZgL!D5GD0!}WK*7UQH1 zn@Iwb4%g7p+G#iS)jgeW;<*Dkh1YV!5g(SO!pmG?Ju6#vO9{N~w@t@d?G&FZG?7hi zkRL{?NzM2w$^ffYXTRApQNFWV5*FNVL!LVzmgN_ z@JFPQ7W)gN_;E6-rK$P7LJMm2(LRnF1pe5q0*ito2h~eV-F@m=1f`aXqCds~{z2e& zvDNf|20PW+deHgxLxbPNw+f44j$ljBbLWpmqzn@g#b;g0h8|@2JNnxfc%nlSW-p zNpuw)zbLjO#6u~w35q!nWmI2V*DOx$Psr^g-O&-+=hcrQp+9q_9{lpt!cfq2Rf1{WruER_&gv+3i01ahR)hSZg zfIyBFxEy++cQ>c1KdDcqh^$B;)6mHTvE|m-kS@k>rL;R>Vs}Zf6aEx{N;gV&g{lB* zjW2Fwfd!5^`1omml498y85!ttT2BHM)eLWAMM1dW#3e1f`em)}rs$Y&Rd5b>nJ07O z)8t?yauwG=1;6|vtM0hI0Z%wf;Mswz4#AL6M1oFu1gA#k{N8eO zxvjzm+)^=li#YlM9W>AX1J}w@fnt)RfR(?{%v=pOq#SieXb5J&q#Dt8Yx&`WO;*F; zVA@ZpwR>8&ZM_-u%3@tg$^-_&*nX%aP01I#SC8 zL&QG3**XT9CImf1U7b5ncn-JJ9qGGGg5R^qN!3`6zev(_QDDz3Fgl=FrGi;u#vDMhZbO?(yi=*THtJDvAqPWq~p2iH= z#Db)_Ik)x5-U1lx1AkslSA^w zj7K2J664uJSSmeW&In2znPhnzsg{fDs_b&5|Kx^yjtDx+(;Jfy}LK|=h(@?13>2G%RSPd%Gm93{Okqrt7685 z%5%hvDc~oSYtK|OnI?ncjz`rqV}(!!5N5P+SH1ceu}fk$C@$jIbz9rbf$Q5%)rPV0 zwmWuIJ<8QLUCUgP&@(c5G@gvb3L6LZBj&9WDWtM7-+1*!J#k0+i_0_)r2@lHgq(S8L$^_D<*%})-${jV@!$};=TSe&fBXf41V4~tiuP;s+X4AWD!ON-_9G=ym5Op3 z5;N!^lV78k@+{2cTz$T;6MgJ7*l93={Z+i*f0C7j6^L58ws$Z8Q=4hV;0^YtA{$x% zN!BFx~zQo<&cZjkP71f-+{Hnj;s zN;W0Zedm_=2~m6IsfrIzx^6Lf%8AoUMHk%%wVSA^}i`A z4;Vr^p+S@GziDs33>cG>MNay^xpN;DIIgV2#;bqR-v5UN;t!-fz`&-sBR|3WAHyU0 zD->f99r*uDCYCXk+qTs8R&%(?M4LsC2Ws@ePnc)dN@d%*o#J zb~smCUmi9HR{z#2JR`)D7!R3MlBZYt<7|P1E9})Jc%n;zMf3;+Yo8xnfhOKKaPGTr z&eq$%oXT%10Qm<9k;?yy%{ajlSi1By)uHo_8?Z0s7fB*944@7oPA-+H9CHSYS+9U~ zGabplIJ!Lkp+Dv&cJg`NYqON$kcVG`&*1OJg|R-8YqETs z2f~q3;5s@2aR?GlvQ=8Y@S&Sk9x-L$3{~n91JbgiIp>j?+9{*W>`RaU>j6*9b^s11 zHv>`wFawgwjj`VX^Dc1^VErh&LZ)Z`vpk63iY$6y(gA-i{~)6#=)^PdaeGGq6n)cCA9Y`}F&l5GSB9Xl|>v$zu4Y zcoZd`MDHr^3gl{$sWcvow}6Dsf&rcmQ*gF#SPFInDy-&BvC9K3xvFT!7{8Hp(d|CQ zC>L|Aj&0xoe9`WEiTcWt`U1rLb>pfcL%?@whZN{gyG@(E$dbg6gZmsK-IqwuS?GMb zX{0=487g2@9F-J(c313TQT`MOeKLy_>QMz|oOJ^jNWt?|Qr7xX_G)Zq#%dv;|MnMb zl%MWZG&+owB~jh|_@2)vG{eyigs(#AK;7`8S1wf_R1|T74b4pGyeVu3C7u2tsD)2n|7kl0$-^? z9cq<5yk>(!<#=;yBpsyE=Kz|6-SdtS#XoN21W{Z*cWnO1hhCV4@*S@e;GYO#%Khb= zfJQ8>e+#qgqJ_TLd+GNIBsdbi_khCYJJJ#8b%DMM;#&H}7!FAWQT-Ub40CMO0hXHu;2cHA9S z&NMYMGkJaV3|+f6LhU;YHD)jQ*DUJ1-O z1S&1@s*asKx>>r_FR(s_tpjg~REu;)v<6XO^*N*?0E3h00d#oTiU1Bg%T%5q^sHua z&Gwqi=*Nzr5*j010@=#$WX5HaV4;0q)SJ|_L%_Px-=PNXz?hyqU?wRfHr2XV2LpZR z($l;gDFO!`LbK+rs&J)txfo&V>n*+p;4YpQ7Mr# z#1+qP8U5!NbA>e`BWywa30wGOQM%S^jwII=q>9is+6YZCYBR&P{ zBO)h@fz*{+c5SDYQ#6!+PEtld!f7-M<{c8s6zQ-b_V%nnQr6_`VB-a{KBkrm#(TmT z4|Ak5@>s%<+lufnNP~%PUL72kR1a58mg+alZ9Cv!z=f2spHBG!)D3C-{bJ0UD<6Fi z6)*(Cp3+mxlkTp9`7D2KLjJwbps^}k_aT8RSd1BR#sF8Zxd1LVNu#HE6>SM)t^$*2 z2Jn)mh)|D9bSoLymK=z4hm$Xn#l>X;~0#@T&M zT}>J{oAU=ee@z(#%>%$$X&%?JX5}|`hE=c}f?RyHpog!R-iZaR3Q!E{osIsP(~4ZE zpJQQ4Tyqbq*Uf1=^}=FjvAB zk|az{|EvN4g<>%%k|vQ?-%b|8T8V%GO|1n|hswkwvQ-g?thUQWQoTh&9?c0fSmcs=3`$#V&I3IxfH8s1=NB?8!7Vz1c1S&XLe7KdTC$;t^?6PsfHToS<2g9 zP;-Aa3k1!!$Y*1D64ZWX9_f_XfLd=~&9|EJ{}V$(`Fuf{lO5k+`Ah(;uz*;Xh;$7n z;nMHHv!k{7)1c$~$_Ov8oNoiSg1Ox57mAM{$yU3?art00*1_ptX4i-^)Xt6jATGZLr<%T)S<< zXMkKYCLz}s?4crojX)Q!#~C5C0@wvo07~&G3x;Ic83K*vWQ*s1iT39w(Za%Lg@I<8 zp5O!XQz5jz9z7l{V3Fl_MLY*_&OT3@Kpdr07G_dSg3C5CAUgTIgyH-+#j7hZpRL$s z_S=iRjz&5CO+|$CFH$3qETkY90SrCUmz@lU27PSPd!0;&z-m7J?CQj<83YD~ao|+S z4%rPM?AueT5wJjUZUq+A)HguG=h)FJy(f(TBI86Twa8>0+=>mb8B{xtrPT=hKn>p( zFfJV1NjYAw?D&5_DU%7mU%Arr{sOcJpb@ve{*^vAJ^rZ_&;Gcrce)P9=97VR`a5Sv zPAZVHxoW)HhS!eUk&Zz?$Pu9|4V3E!-y-ENkt-?)XXa(#5eP$M+L9I)`}kpREdHl3%nQBCd5GxN$JY7q(X!lnE$K^SbPm7c`ZlxQwg8kBWGt!|cpJ|K zRBAb*Mc%r(greT(YQLyzwwy9_=OBAc6n~J^wBgRJPc&hq9_iSSP5WyQRPDEWq`)3) zoJxg+l&H(hom^sQKrAoJX`YfV;W8{!@L4qoEv7vsNuboI?yku>(TAvF$AzaC^>JBP|YCy0joi$b2;eMM8a|oQoXf?L~XtTNvp-> z+2LPK%f=+I_wVk6C!>MtvHM+!r4fU70Nq;9l&K3IwHdyL=jSRvg{lLOAW-MgZ<8wz ztV#rm^=h=i7=BwgiFo9~tC2*tgN##AK>s`kU$vc|;af$l_H=YcoX1qcP^qdv0%~JU zLV6O+J6c^!l8fv2w=uyh@pu|2>?;79Yl0jSu>wlxJ0EdBe<3pjaEC}2uY+Hve>)T6 ztlt5}v+@P367+kTx9et88O1+}y!lSIxzO6IrJNcDuE#9-J$q?|Ry)vUvFjY;lpdyF zW147ow+A!uddw=JFX}nkjy{4|@sX!<(P4rqStFpngd9pHrEC@=Wz ztE|hvdWXy$m`SS+T2=&&IG9L;WAlps}7Mbn8&iJMb%s`Cb;jbkv>bU%R z&hd{Qh@Vq)TtQsiRF)Z%Ye3JI!MwIxX4ot&N6;HTNa4JR6od-XKF|tfjqnt%$}R`1 zTuc|$g~HI!U!%7pQ@DP;ZSY+QQ(YP~Wr-9PUq*AB2^*FPzti8gD(U$WA0)gGBm@D5LR@!$;I^d3uP?Yf{AI$(IKg<%lkpB?s9j@y4-6AWp2%yyEJ{G z1@?1+G!n0TAwu{{eVAyN_sI+nFh%{jm*F^(-J235Wn{eBJU#o!wwjq2Xsnj{%aC;~ zo-CgYPCuTmQ8<7a@ffWIvD~2|T0yMS4RdafCom-iKfW91m0I$hHxzpvcYPNmtqu^} zSYfoglWKm6y(UpKaitxjx8oe{42!mfk2@a~N)PS-o41TIhn}&c zim470lj`;IM=jM#KoJ%q7ym8c`CulFNv~(97$neBwX5rEAlqDA;~swOOp||4E){ic z@Nr*iB~jvH0kYpQ?8~4-<&vsk!}lOm;ZX`zFUHLD`&yEm2;{0lGsi0T)n79MaA`QL zzWzQhxQ7<_(~LcUazwtP{R~mOQ*F}WVagi%^C74{u!g?le$(Nm@L6+5i zF?A9eb`QZ%#z*q`hC4uGHV1LEH4O~_?ioj?L}_Jl+!1i2H#JA4D?R35P~NW%p>3fK zi3QSJ3)81iCt&n-NOa~vr`_UuTL;pjSs^_^9LGdS7mulE>J?RVaqP%2+662*^H67* z_;*D08l1``9YZeW4;@a(bfQt-P`>V-Qn_3VsO%IB7O)If5XDzYmBfuVW;^(!U{wwb zYZ$L^%L_3bEy7lS$;jb$Dr;DQXp0bcgoeX~BP7qUk3dQpp1egWO#GyV9TMHRCrd3C zclyp(HYAIN8)ug-jNy<~RIWaI&C_EC z&=~s5#1%CtS7UcN!kCw((Ib|_7?t_4@k)Cn(0#G%vPvdP+|&aGJ;^ zvX~4Qp$4E<`7Z@Te&;&)AdZ7ck&*_WSJNwWq)Y(dqQbgN)i(KC3B539Fn9XmjHuNd zO6a5-VxW)`!zm2PBl9R>g7mi88tw7B?(;dRWW2a^GpM-u8!wG7ymW-OEhQ0-wo7+d zVuWMDqe?p7Ke@M|%;26)7J-(#gR^NwM)mYfgz%`m0~*LACj2@!vfguy5m z4_B^G*vGIEy=<&_pudk5V+vHwEa<&!5uHbRv~mth(Kx#>*fHuBVHxM5PS*de(} zhy%0`q6k(&X<%@R9AG81xDC2kS75D{83s=TP8`5WgoT4=A!~Pj$YpURF7Q9c8y#qXv10s>Qdn&yo$$}d1G>??CP z%y~2ZBbFx1yv*1B9^nYyhqBq4n%W!DY&6_V!%Qqh9U>3rLw-siO3)-YrK>%nb92Kk zen)DsGSG*I!H%I+o7wN{k4XaWI2LJ5K1Gi5@Alvp@Y8b)H)@ zWM6ZchPsp4Z7T_J&9FMe?~`F?`!i{>SYD*hjIT65@Ax)N|*J7aybMHW$FO!{;Y znWYU!0V6F^2*(Bn>w|Lu9CwH$Pl|B}V|0!1OT8#W>$K#;K9`y%SG^O9&chh)kL%Dw zXV5K6QDOzQTDm2-%)S2p^ZhTsqypIF=PEOI@fHajC4y( zR?4BoOlj(kST<`>GGon?+-(ivtD{`}JCbmZb+Tu>pNWGAmc$V?LlClFm)Y)LCfq)G z^c!sjbpdK9nW_ZrG55*Q`RfTsB@!1hC0=-J2y)q61wK*uTz!S5LGYOuzo$!>T2+&% zkhk)2m5uo;ROhIX#IL5`WGR@*9~x;o+VFIW6Wt1Cp+(2-h;GVj--@<)OE;11Wl)6I zC6y7z%B>oW5lKOaF-$gkS8NFvZ#sB<*HJfHfXs;DX~haAW-wl`<2HuMttqCZZ^=xX z1Lh_HGY^gBVs9xSc#i}5@RR*Gh2)6{1mF5SW5fvG5Ao%$L+%x739$aem~MacXi!G= z+f6lDdsXp%ljT7~&Ir|){@|4^4dG+Vbs8m0N3Y1I<4NA8n)%QMA)P?Xb`%uyO5|?@ zMAw{)?$992Y-w9N8?&P|Itd?qz$Z4-JQG!jcl;W<;3+a-V9HBpeMYjz4e7);u|^h3 z*IBi`lWT6HliJuUQ*2R1K91zN^*rMlWYWt>tRFlrI3u2{B$^|2%^p#?41tA%%7}oE z>QAm42##+1703p9?)t@U9F)E^6uvbp77Qz?5?+>w(~ zIP_2Q`^ce9*~ibC@O+8|5*4wbQ_*Il7N*om)OCkbx@{@e&Re4))}J0=EiIjk3^g;f zmK58Qj7@rD2~zF`iVQ59Qv`Vm+xk;?5g}QHd}Dk!8wu2{-H5pDD~S~`g0H2 z4U<^*WrP{0lkrEYFL6TKv>Au471#Z{&`?B(@vA$2IwXSTAN1RdjkD!jxxYkyn8wbb zo_L5pCh<-gd*%};P?@FSzgNgF{mo~tG`*_)zQ^|-zi{l8;S@2>@>yrxU7HdH=P!SV z)#6FI`z>cowBS1rt@PsgeZD#Q?Sm4oMfHw;VsBKu1;*GkuhhHwd1*cylR;+1k&@cl za?8^|SM5prB;pP8geu!OWlV2EdF$~Prqlx|w%3a15SfPqW0Bf=2)~6mFFm5Xo<}zY zV+%^qhnb0Qm*x7Wq7R6kqEYV)vu^O7;xn?KaZ|TiKKf5#o`zoC3OFo`s}tpTVCw(Y zQ2y6na}XbvPwVXwB>$GdHh9PQ>2N1yBkL74EH$&oG2zNFeNiOXa|XlSbeRTUf((>m zy>WvX2`|sPMR%kTseRpr&Q!Ty`mL!V#haJTe=dv+4u}bI#vO+`FHIl+-SjkHixNNy$q{ukfQh_o$=%!`Y5X`x|1w|inre}GIey2u!L%<&!0He zA5X)1i?KIQ@_chCp-%BkG@gnCoWhEhs~kse!q z$e8zH7sMo@MP^v!PB*JFAICoz_yf2&)INfxA6Qe(SUe?ejq8B`+~vd&-}!Lj*!!hwkV{^W_^k z<~??GqPTzAZMu|dH(b4DYuFOOxFPym0WzXxI973UmogKP41VIIeJ2tIOQX6D98?b> z>}%q}bKZGR36xQ8t_693q#DrZ6bush=C}>&Iy+S;9p|y&`YSQ~V$nE2ieQYYIw(gx z!tdEX{HgMqojBU{-GS#QuD#iE$jCV4uFvBE-JyRwI1&IkKO9c!(Y*%j3cHcnD}0}F zVWJ)9hvxDLp^2gC&-t(SWvd*{M2R(f9qD97W(NX}KO@hLgQ0t*3gw}*Z)m5&@_w?C zvBwg+*-zK}7W6>(PxOvvh=V8 zzDM~1$WV8UmizLA_!kU)Pgl#@kyc}kb0n<4D|W|wvE!X{@{F~5;%i&dYR*E>n+_qX zCAyO;Lb|0zDv6}ok8dqkA&#nYf7objk|ot4fYA{k(uNbKXR zm`n$w*pLE0^)auB*1WF$?^8%}8&!zKmEi}yRB*%Lngcxo3O|s2qT(g818G%2V_A*L zi5_m0a_-65?n|LqV1RkbuiC`jwmzwO`u ze#ifIJ-{WLr^$FDQde#&=U$}si!u4Jdk`L3)0fC;SIE?)eGcjDpVi`0*X2ib_VqFR zzbcP}cDuGt*>Dh7ug5f-8%6}rYjkE)>xs3OiY6=r<=CrsgNN&f^j}n~>Q7b`eHv&~ zBIVSxxNJ+d(6Z#L=Xj}IE#?05$G6K}HcBx_QtSC8m41uMr|k(x(OI`Rg!S{0#Al}# zWz_X@hQ;c>+wmEv9>Hmj;t;m$>&cT;l9!+U)amrTJo@>X*YYNc2YJAde9Z^*2EH@3Cn((!2ccB9(s zX$UEw3d!AJ?TK%zjOS^E2Cl6)tBHA(cviI6^4TZb;ebzd=Pzo(ft`+K#lp8vuXqi) z)MoP2!?6Od?Z@kyAeXZ~v)S6N=S$@nFB)G7&E`mP9H;yl_$@P8rl(!3ed(9Af|#)E zU3dylZ^;gtDAewdDVV28dBbHmHAeWHTXP-W-osHzNF3h)WG6e}wNdO> zX?275*^S<&)EBHFC;-TnK8;_cX!`e_Qi*1_Q=MZDKu8}wDipcVoY3hb*HT6n_R8(q ziHg&3of+C3(wPwVx=O&7Z9L%qDsR;$dFo_+;rAWC*0#Kiv1BG#yF_Z!>ZrSXzjQ)Y zyJY0SbvS{DnBc;&@Ef$vioI(ri{E`RlgGt+ljj%AJJ&4F5=}QJSnXN1^xQc#cjxVk zl*Z%Au1@kB-G=E8=UqRhq^E3QyM<^-H*O#-NKCgQ%}^zVM`Z;5WCKF5;~)wq*Uo zN2}Xs`7ATRu8$gRzPd}1>Xx@%Gw#q{8@)Mhe>QZsF#Qc0W)X%~PGceVbZ&vhZ&wVk zaX-6?v}K*|#2K%FUok3VOCkM*gO55L!stpB6=z3nWO-GhI? zxvyVY&>*c|L01G2THa1i?r`Wld zx{SB-3)g96Q=|Dhmz=ksP2FdY!hf?fT&v!)cw~_*kXMS(As3YkV~^l^%FzWavt_bl z)TV^O<(m<2;IeV8y(Ocaj@xsrZrfAMd=6t?LyhgwXZrd@zROc@JG!>#jkxI zYN!<6er*fytnU?N({9TjYc);gY4{AUFJROkshbMF?W*VbS@_7fNT8>7d?mr@lcTcj zVJp)Zt($SwAiQKfL}meH+A&kTp708x0?x49YF^pM(W_>g7!uJ5B8+ z+Ph^*2p>hbJNDu0+x5@Gmtzet`Ny_L8~r&2IhHApwVuDsGGf%(>?X7|AZy>3skG#p zDSIpwbGTZRW%ncF`U+o##^co%1?gSG!b(UAB98BY*Zz3+Wf|$L^`Ksh$}Y!jVP~56 z+#ASR=%k`rDdY>(>3$;d>vcq97RP)+lf{bzt=3FTUOuu(dm$ z+#Wo$N+wTA7Sk>GaXBdafMsHA!jNKPNbSQD?>+uj?v~nkpO?R6Pql8#WoV8Yd@WD!3RgC528tcfn z>lBQwI;76szWII9$@6R1U*xYFBAZmygT~^MA^6Xpd4JIyqMUz`L>^WK8wf6Sv0moy z{$YRNCBRd^ikCuI0d)>%t{I2v)FX1VMa=9C*b~KcD|fA%hp%K2b-vGDA!J3H-8mM5 z&iww=+CU0-+S50V8Ta$Q6(i3wL`+p=(Tgy*0i zK~$dtSpxGs(q^_WO^<7|bQ>OAd0uA!bFzpd+Q#s_o3PX@%0QdCaMsMEou;Z2`cuu; zIbv4Ol2IE+?zp;RfX~y!`Iu}D2WOvgbNn)SV=`>h^{9X2Y`YQf;ObABU9xY_d?}2M zV^0JMo3eqI&N7!ynFS^nOL%4LUvthIX2l4kX7+R0YFtG+;WY~@)oJXGWwAXuTC=n( zYCqe0k8yp+TW(M2Zt3{FG|Ga34f3kp*#h640)9pLg3sZ>B@Wi}udkR?;?`HJCh`n9 z`)tEH;wf#47V?g5-68pL<7R5ORID)Cq)(eZIZ-%p&!fIYG{4zjmY7?=MC{UL#ru-a zvab6C(Qcs=2pXV)|Jj9_2F=sXyB}o)Tk+^=+ig!D>snz-7Dt(`_1Fhf!AA zJ=;Diq+%HA+wdx%Cb3Py`_ZGrecJrSv{QGZPpgt8h-9ycby%@#`)n;o%QjSU9)^DG z^{6}(jIjPVi zj*5DckIimJzx3-P4-G_`d(ZQ1pDVr!7}F$vaujb_X@T3EZmX%Rpy(hnCAXToSVk&Z z6a^b#%cbekK{T3{KCd5c+uu@&=AfLW z<=vyRI#oN{V%oaV;b$$~ofT)MQJ?kR`Tpoat!ioEackEuH>3Q$1(iL#>r?sG+xu$4 z@b)OW`)KoVwwyHrk1^r>JJx?79B)HV zP_gJ7-uQP>ATZfjC=jDirx4CGUIKBj3KEEC7Kw9b*+oHIg<8{R6MOuTZw8Ob9u7J& zVlP1z2fPLOJ}jqP)q3O$;MI2-D2%RRVlNs0{z(vll0~DeqmFg+L0{DOO&PSMgg2j( zeW!jmkBt=_c_fruVz(scN1`VG9smoSQM>?i{2ocbzu%d)vo2?SbGQIPI#}QaHnzmy zA%0PNr+&Pj!*_Fl4lDvhhYLo~&0rQ;#0$<(M=Aauz~2NN>(HbrK=NkHzlrH!mltoK ze-9wejr!h;+fai3=IAGP2oS!$HL`yXKu2~<5>T8o32u&lfsXaxr-!`f|9$uVuaonH z9&3v;A(l95FpU5q*OT@)wjen1zjFams9f`VUSvTz;PT2cdsz6P45TTq*iCXpHvhU_ zSL;5>+c`WuSEqfZ6b!^Rgu{2F(S+Q2D@Aoo4e65-O2Z!xo zstpROQy3bYB1T+1LvEi~bc^)AC&vt%%n9(p?HT|0Su5D%X-;Et4wEjzDLWMKnf9pPqshk@1q$h>yH>mXatr9#SmU#Nyxhi_IOf&!s;jgP)~zXFtmIinIjIJ?$U!f@A+B(B!a4f&((~ZTeW&Pog15jX zsaRjkdX4S3jo3Wv zky(LDJIW<5=z!J_b;$TGX<9mBcvF;qBW_>1$`f z>EYWgYWF%6dm0opIr#$xqWD$vrWivs@b?<;@y9Nm{TV<_@zN@3rSR6|QOfV~*r2x0 z!9*K3(`%1}Vaz|j2s8*l`COuTHT6EcVB>wk(KFn6r-Zq=Wlh}2C*N%{%`M7Sc%td- zln(6&6#d`(Luz5qjy*$i-y9Y8L`Zi3eclxxA_T&(-yY4LIi(*R?!|7|C8WW!p`%)7 z9vh|>w&9pt;}w*fa2)GRNiTY#e9rIzM9hq7w8$j(g$i2dMTxF5e?5NuAQMARksVTu zKfF!1QCyYV}-Xk*jtOt&) zwI;FOP4~rHHJqR-=KJA7of>hIXW2y=f(@25rHxaL9ZTc4`!!YUH6B(12lKR* zZ*wR{$)yW&gf0?uDhwtl zZn~+8EUl1lDo?$AcE5re7~mW_5FYYRZr@~h5gzW78geDF)<{`*sPWdk*+y5TdZ~84E270dqG#Z&uNm< zaDCGnd6^S+RM>EFEvBY#BzBBJYX7INk5hqOB1{J#_NaPICd&2AZ`_TW?MEQu7K`r*8zPxj zIcXNcvV8p<64!X>njgPlw2Rcg+jAxD*8IB^qB4Hhl6}L)i(lJ9-Qsyik5(*BZ5Rg^h#Vqi#r9fz_uYny?^`*i2%#H`j==mtcQ-z@fPnyb(UYlH-gz-H#?cz zf%%sDF&bx?KN^JNevG7lQO?~uzN%`0MHkgJXt;{lPpm{-JSSdXoAuV8)#8o2A^qYD zylk|8X!S_$R73BbD(Q@#(R$J|_DtVcz+66J_E;(s+tQS`{k53B0y@rPJX0C^zH|{9 z19R`vE=!1WLE&>Cca(}6N{yomeEUvsrQ*ND} zj1G9Y{ba{w09CeQ(6?WIREoLqGmG%rBNE$>pV`EO>~SRHp*OZ_R4=uq+E+cC%dclR znt&t)%g6iqXO4VF9 zHerLW(kt%l6kjJ(ziljcvG$fh#{lQuoS_m_=uC`u?Hn*f32sFPMv@zsTJ0c z{CTnYLX?n}Y~I9Jtyir^{TR*nQoKRUT%8)FaKLkPq**w5xN85fPI)UyRhe}Ib}W&} z4kdpk{fo8jlUr0_D}`*q-toNG8}jphZPW;{+dfYuzKL%^y&z|#j7*PX1l1IXeo#JV& z95czGk^c|>_{p3pj+k)a&%zE%UR5OH3ZzvG`ChC1qeQFTm_uf8TCyqI{4D~sf=}WT z`q5=uhvV8$lXMAX3f&c&7{W(QF;hT;$hfXCMSZZ8GimQ?{?^aQtNzTP#JQURP%dsg zJfR&UszS}HVn42DaHh)hM@kA{079}RXL`B4e*T)Vd|&7!zdi?EP^z z6^)pS_cJ@fs3<^l3tw%`OlAX$B26nlLhV*Rn*e0(+ygtRab%~>nv5WhryntSEhcts zM##?0C)249v>;wlhW2!QH64yn@;oo{&cVg;C}{LMMN%`fILx0WEjW>)trra~+frR! z!|kA1COMBmMfn%pISAV1>F+;}u|G17cr~S0bPF^fdIV3|ny4bZj>*h57VAPS-pZjx zr}~XQLMIgAdfd3<*Ej9yV1wZ&b@w5XFfz=%+c$2Bb3`vy9;c{+RvIMIKO|uFfvGLk zE=v*;?;sa5V?^c5e$@f+12G-2bb^h;FR!vy4EnjjC3C>-miTP5#}Ee<)Ucvj+O<@a zQYh!J_Nzl1YUxI{SDyJs%2jr8{R?ylA_k?z&7i-xYQI&G9xCcJ>h5;Y+N)`5^Q!L3_$xS@)eS zyz0Q_s2H#BlB%zW@4~OIHOm#5^LZkJhl&zQ@&tqHi==6sq#AJLk)e{olB7n+@oPw= z*rRYp*220-PMR#~r3t*?y2QIzTc zeWf&1r8GQ+?wZ|hOFXp{B-m9Iuce$*buYUqrAf+oOy8UO~zzI?3}5b@9q@WV)E}I>R*%A7mj_SO&vSRjK6uhU%&(lvg;F(dJKoYt6w?&?c~sl?>Zz}wj}3RovFlQvxMyLvS9dsXs+&m z*uzWWmU2>iOddZ~I@Wl~2)U98h>au3DLpo!@sjGa{Y7HC$)g=#-r9x2MEC?`5s`L^ zl4NNhnP+*2Mx!xCPd45)NF5oSWE*!+z~`vFw!sWJgMG9k*0KAYDe#cM<527=J+v_ApiI&lfaz z!Qmm2VyN>R1s5`{DF`SK{XM|J1Tp)!t1@0Ica|4M>N~a23-nwZ(v`gjocI|O!V@H! zCgXvYj=un+XG0d$^k=Rn%5CE!^)OsyPIozd8^AKuF4V;&EnxHQBm!B7X}lQx&K<U@-pvND)^S zcKL*WzrJ>KzjaD?0#m2jz#C9SC?G^7393%usZ=<1GFSs9FamK8amF5eg}2NCi;GU4 zR-VuWK|)Q*mJIgf2c115^Fe0uePsK-U0W^s9#_?tuv+KHaxREa_yGvHnZ`xes5!l2>IpHbkvYYc&Vv)f~CKc#1Puy_LpAY)PZ%CHpro}N~d$bnlq1&vSMSlL~N&gu}R8qn`%EJnv;`4 zhnMGM$|Lik`(5|RvO|-ep&o?HC6x0p^sMT;N3NoAqHzx;%Pj?Hhos)TJS2uP~!=9HR%?U1l@xz-gLow|vEmkhR zOs>IEy<{?pn|*vSNa)VFC>b=16eO@BNy|YX-&E*Cy6xK+%jjWflNf5m|C49x^WTG?IeNT6&GCI^>I?bH4k*1-AEt+tdBVoEp1)2z~51O0%Q|Rj8uqZu@r7s> zTk>w%PN>!B^3I(#xc4aXI=a$f(>F16UNv>%V4(Nh_1-LRn&?{1)Ko}3qwAu;zGPqhwh&G+cfYTm0jMhHiIj?{@h&%{quVN2?@IcHnaH{Y|df3ID3pKMp!<;R_=Un9`QJr+z(hA7}3s z#dp>p7;D0XV}wMKlFX7FjqUFhdKZpPuNs03KYSUgbnV_o;3|R425h^{2g#0*9ofCo z5U@c*5Os_rAk%IN-zEQg9YR+LWjw)s1J@^T2lnnaaJ3-etIYKuE~|*W5rX!9KqHaq zPA!D%6~3CJ0EJD!nEpjmvP#{n0O(3%7XPOaFa?%?z+w`ZXkuDoZNj8};65USi2kPw z6xd1V#)5a9tU{_imTZFn45LmT&TE!!|CejIpnjvwd+n0)aB`po)7SvEE|L_Oy1x;6 zcK+AbC}sBw%jU$l=h>bLj#c(jIOV*jQqZ)N!R1W>4h9b~xLzgg2G4LAp3uy-9J)T8iMuh_Q3P#M5pspZNCXaMrPKGD zgak(f7hj_Xat|VyZF-n12&#L)vm^aQCBD24Nnbv|R0^?7 zAcZ_e;uhc`vOylA>EQ2y1f7}wwlP;B_i&jG&GQTPOFT1x+sjJTD^{`A_#kC}|H0cw zzpKJLaxQ4vxD752ds;yIPZ2+@MCqavQPN@`G4DrBjr*+@hdTbpYBJH%Ze1UGgW1@rDI97i#BrTvH9y{z~Bi$3G^TvN>^I|FmQ`3eE&aW6JxZK_ND>4Xy zA_I1e^GQW}Tr_twDzGSVx@dCg$tj>ywiCxdY5<{6btAruZx?}Z?m<1(^kd!MgD8(c zLEtu$gckaH#|!}lvVJ>)e$dZV8l)B6FwaT&4jRb*_>S>qmWTRnlI{-tw|;&^Q24yp zZ&2>PS#Ec_V0aW_y9%)nj0U5U|9Kk1_dTlTIX4Ld|5$GDR;AkulRD+Y^}jZ#Ur~F# z3${2jH%hDm7=wgugs@*{yiwgz`jU)C_S0CJ*LDG*x`u#>zl>lek+T5<}1 zq#?`v8oHTw#dIj;@UcakYfVKS5~P{c_BX!lS!E!F^|OAK4D{0<;{CWM6fQD9b6WT? zO!HA@88!pqaCDHsiODG#JQ}CYq(sAe^uDfBm$5EvyWv143dzjVawbmtV8~_jc=Dk( zPW!*U*&t)H8Imp#X~tU*#UK8wJpd{GmFMBB6uz)LxoIEUSpq#Et|+9YG(CkLQu>yX zqFTqySfF)q*4&8@*+AETiz%XA6l-=*oSeM;LI!N+h^)!W7R*TR5 zAMtZloZf$}$W1b<_@w0=1l>G!A>2GL^+=N0clP-_OEF86nnva1Ie%)j31H7PX+OOn znW%9w2bx*k-~t^W2sgqxHtNX+C+9Rf=D{ySLG)2)+-(;S4HHhov`JTo{zX#3L}KeX zhIP%}(ERDaPjBJyS8SXm#}}Gd+;8HiLyVrvz+r|yX;qcq#-s{5Z0G_NN1*xEboc##TU|Kbwq>Hr&0V;r^bl5srAjJjvG<TI4?=9@XUJwDf9-g=_aP;GSb3NTFM#oq~$VnGt?_XB+P zxk0P!j)~Ng$Ml2yBjT3vmOHCSaaA`m$(-JZPkU$}!^wef3&94!1L5TmkC4$iEFK;) zekV0mD87a-t3;Sn`e-=6Tk3T)=oYqPQUDPl)>wy1p54Z**l8UKF z;A9dTrA8gB#E_F{rg9S_NufR!@;S`npE1_cP@mMM<#W*7zy&fNBnCI4wR4~fDF{o* zi&c1!2ZH(IRaD7HsG&Y+e^Y4k_HL+;s`*RKm%qscE=&qUMGKHAFz z#Gw$OpWDXC!Xs*xqiLYM*A5;S+PKhg-?;$8nZmo0EpnB7k6c6{(=r0N;%{yH9b+LiT_u0?guoB$ck!{r74x z`QqCnCYx!UcAKMJwx6WP%vOC5)yyxFa&FB`P7Q1fO&0jhv5G|o?W9$%X>He36fH9& z({kI#)181H-9-aJWfcf(iSY07CCYA-+cz|3vQC;RHq6YuxpQ4^bxJByb?D|GU<*V@Mef2R8saF$A0e(I*6d zkj#eci;Yl*qt8RiB%0=FU`A+Sz;r^;2m(|kB6ZGCugrV2ZzAhP$o_ertmFHbYhPbL zWEwZaBA+xQ$b2{Eft*NK>K#R}*r82Aa;M;3KPV5B1_WIaKy9FC1=5>BM%6;53*EE6 zEAQvZez*dN#@ZPmBbI~m5aC!RGDcvG$h@*Dbz?8Jasw9|4tIe1K+D1Ub2?mZtv*Jq zhrK{%OM>CsynbQ!g-#ow=?S2#4VVlavIk21C44{7q|i0~Do_y8zi~2H{flc+wEQHR zg7#1#*8-%pMGI*S&|^rwA@QfQ5q-`##?QXGApzJ~+Fh#*YE^LB}hVp#NZipgmB0XmuUnxC-8+A(u=S|L(81e-m}uoJ!t*bOGL;+BZHR1`DTG6rFcrLuCqhI;K)eE5 zVS)Ifd4Y^@-u}KDNWsCWg!Oxm2d6_IV@5c4P>2Jy-|Hkn;+N;Pv>ih8=|<0=(ITbn zRl8Fsg1#jJ|oY%}Z^_^uMmJ;G+Rl&moR&6RTc6&>%# zJQZ`rE{II$ONuOl5lLI`IXCl5e1u)UhkQ#AF|&Q;^x8g)Z&z}+ z?<&^5eL(+`CoAFvGiC2qtVvxP8zYsuK@)H^cFp3o#7FZ#D3k?0=zo+y!CbgO|9w*{ z3)>dYDC%I)^w=MQbz!TN_Uw_un#?*!=C84Vq6oQk_olh%=8d^7Tv9eRxC^J(g-e}U zO%IFh8F=fyrK+}BGPUY@crKKo=lACZAA@5EJ6Z zPX~HEj0YZ_gWHcG<+b=zx6}-eJw{o-FSh(z>TSEu5f#lMxSY~Nw9IAax2|n>@Lnow zp-Zj6FUM;v?=xJwaBl7Fjo*(6IS6Gmk&0)#&xi}7EtN}F^Z^)Nk@-kW-X~p*Tp=^| zkXz)Z)AeEEsjb1sMek3i?mrib z>+Gl-xFd+heKGBHO`FSZC1NJ}!iqti4+(S*8zT3`3r6yYMyjSRYLES3&+ET*+4X9v zQKEr`Ygb?I;!FQE*0)Z6bbh5ooA(0GpD%Ih6(xDlO0JjI5Oc=(X=Jmcq21c0W^0Fy zm;6D%5UtGd_4;!k=OH zLy&LQxc9gW{_z@;8v%_We@sh4{p=M!l;S=&hk~Sy)sbFOTd(^JwJ5nsCOZWki|qxc z&C#!3>Y-SZR{HHM#eM{u=-@sQ+sG?@D@Q8uJt6)!`>Td$Ym6=l$FMA9NOzrHjz2V0 zm;gfIUsOQYN{Y(-YzTy- zr~zMH+zSVX(jJcV#jO{m;xvwqiv_)vn@$mK&q;3S0Xkyp;O}SJX$)s7Qkpr8gb53DtL%x z(JjAr_5HW8kwCmOpJA`^o~|M$?{WvcxW8Fi=#{!)?XWZAJXvb;>do{!p1DP;Hz=8| zJlneJ!tq)O^l!buS9+kfY-uYkYhF9(s_F@R8_0B$UeXNeSgq-ixYC&ag9Xbphtsp% zZficgQrN_0hp0Hhb+9d$?OZ@kQ@5^9(&l6=c6i27-6g_Ds3FV*y;GLqVQ8@#UDLO* z+WEY~EO-il5%UxWk8)A6d0ygN>lA9Q72n=O5A9>JqM49DPf+z@eKudbs0clLhH5Ml=Bpi9&ymIgmo^CHS2*yhMGm-z_T@n4yAyc>>08~d-U4+Qf&vevN64t zs#fh4ejs#lJJoSE@_R>%uER!%$!(6b22LLmpw8#()`poXRs+(slCmbrDkz$TF0lk6 z@2XlEV(G#c8@i32Q;$0&sA6LTmg6mN&1R&_x}o~9D3j6WiG1R>wwF*apB!0w`Ry`O zbG$DFv_#A>GjysRzg5%6`fQ9zn`qs5EcK=5bIbC;ylgc0?J>#1l< zc0u2DCO?x$wCB9o(A#6a+ZEDYE7KAF6p=v#hqihNtc(h$fWN69@kQ`8@ zx_Y=`DSd!vQLH&^pwOz>XOlqt{S_;9-w{3Z$ea_BiGvSa%tpl$C}6Al^UA2$JFj<*yFXYcT~nx4sQ8 zG-R+C9h!<F%cqDpd_T$ba@pZ0yK^26f`?5RwWUF1oU^DRGMw)U#F$#ps?V{tl~ zLE~C`4~Kdi1+9?dBr@UK^=G)ul8o8n<4F^1ev6{T9S3CbMa@22t*NSFX^w$&J5F+V zr=5ZhVZ8&ROu%_ezGrQJy+pR4PyGR`+oKKZ?FH1%f=P|j!d?Q8Rff;*?yvI1&3(s+ zpO0skoM8fK=1uqNjT>b}?v*aWc^SpQ=C;3Nch;JBDsuEnat^8a$oFS_ZN|mjtj(Ss z?|YYSGU@TSOw}BT9}4E|Z_U$^UHkQ2r)bLkTDL?EkdyYIuJHby%^;K2pl6R;j&w(S z7cS1~+A-=J?_e(K3X5O&E0qz1?fOfzILliCmWEgO7dI=J%I7P-PyEc(%f32<$EEk8 zwue#Mb(K02XreOp^Q+S?yfpYz=ut~_t@x%0<7wIhB(vL;-{fXvjN;SGvYqKGFTE5 z-Cp&te;*H&!PI9pH8%ThFJ_u-- z4Xe@A$E;R0a*rz3MaC@mLz*B5k{3kg#MSVip<2j3EuS5`6xrRLre(MC{l&o1a6#V6 z-Y+bPxxd}AnTuao=_u*h)2}Bw+-x-+bhV^&YYKGY@pB23cryLbfPmJ6C?p@nWv4`| zUq@FH>EDx}u-RZQ!eFHs%lFlgeGkz$&FAA%qP@!`M`G*t1}9uqBjv{vSoEZ!^DPAD zgAk4EDg~8cJ71yyDnnDa#Os;DA~Bs#jVyncYQCQg*Lw>tK(06;Gc9Cqbr?ADq#%3) zsc3*p0lyyPyFonQ55j2R3WIJ5Pr>C5dHShH5G@fB(xk8hFArj{y(Buw3P2HhG91^y z{xjtLv#^770U!zvzewR&5lsB&`fw0U4aFJI)KDYWSND=`s2ddTiDjGdb-%VU&bGC8WHBeG9 z-MF(j7Sz4u-rGNKJcI-H&eigPv-?&j(vtd1yF!^zv(g;pb|Lc{UD+mfo8@t62pBQC z1LDhFmP2{OONszgu5|MnOK`I#=GtUED!nq^n%Y?(gA$uH_{JHzNOHjSxDFYkm2VQXMMv! z47}b3;}&q}1cmRf0-dWdL81dEhiHA)%L*7UE^|y>F==#3CO0oKXM7K58u<`#)%7D!aiS9pF)!0ax2m29Tui@Si~*wafpCU@72Sf%PlXe0`QP6@QW}2nNq3+A zGFrE?&^A*4(0k|2;e&yOVO365{qHxoupHTi#4j6pcg{edP=Dm#01G!eqOh1@ssUj} z(^LJpNmYgOl}?pTsVAh|`UBY)_y3#h8!#^6?rrwnJuU|zgW=M@^*0`XVe93eW2Bsij7NlVz0}pcphU1XCc<1k7s$wFNE#rP!a0OFxG_X#>Kpa*e z@eN-VsyRPR2aQYJQ$FNI3#vZx07(kX8&co`3LqkN?KTkB1&9(@q#Cl7=C>jvxR|geoFlOYpPQeux(acANkm=Sy=9yl%YIPH!!ZyDbggLbkS5C>A%tA}1VLt+Q0xc7b3e~Kc0Wk~ zX+T~YaG|K>fe!`;PQ)PX`}gVKFaI!|pQIjKJGeI;aJyi5fh<`;8-_FBn3V`AVmi(Lte>Dg0)lfU&y}f>CHL!Bx;q`Xdyx!zm znfsaRJ`0}P1y@lOi*hrzB)G!W<&ALN| zp|Hqs&8uQ9vcW7@H&EhoQ|EJn-|Wu~k}TrDTP9?KcE>Y9P%$u!0Gno^2kyfkX(WJi z6X0;jcJATqdq_7Sr$ZPBEXtortG_3MpqvMo9JZ+t#n5U{+K7-0+69NR4DB9& zm?|hy0KwtFm4woVgoIZrD*^4loOut}=Cs275ZDg>`9~UwEc(xJum=idn+ShY zD{bzX+HOeyP6femAQVw5rN?SJmBR9Jbbx7K~< zsWxU}#HkAVHmBy|^nIZRHOBdAHtqj$6&1stgadl|vX5jr@Xj?!?Ji@@zB@_%#n!jl z$AGI2l@UVzG00E;PO7=~MlpPUG&WUUg~b`4PWxb(r=Jd1n!14Fkv9_LXd-cH#p+>Gr4vToMz}Cz>`A>mM6e? z62orLzofLFeA#Ydr{;I2=zo%%{XfV}5wg)x6#IX2(&3CCNRj|Y`+qMXxR)^oQ8dH^ z$n%17;%~0*ht9eK1$5Z|Oo=yC7Z+Ic@*+0AAH>_!%ml@B1JdD}y= z(h_y3=*tblEvluzC|4A5(dk-8wS|N#$?w}K;i@9Z2%h#vORMM9hHF^E-M?k8h(Pi0 z|Ipn6HI1^rsK65o4d5c&l7d|cfFJ`mDu>#75`mPGde1UYm%4B0q6(UtK%v$279hNO{^m)WCq1iM^Zg&(ax6!;4hC zz0g><7VwB6xx}R(ex|YoUR!j1APQ3}Kl2fDI!m%+_YF%I^vGqS3ob(u*X(z?>F&?^ z@8vi_$Z;jp!LvrCIUp4Y9{c$0RqzBM3r`Fzw$1MoGmI9^9d9}NyLQN)8KPC(&oan@ zO#YY=+WvdW%q93Ck6nxCpddI^`OXOJ8DhO*#eoNUXQzfgfI7b2%EOgxP&;&y)c&Xh z=6%#KFMF@-pPz2Ot3fiOq;8`a*Dv&Xb&k|u$^_3rVv^GKiuKy^e`SpamJ6L;chC$2 zby9A#tb4_QF_P|Tib>48i{&(4_TT{&5Q2Qy3qkGc;0YC%8ZK~fey-(FDwZ-<>!? zS!iAWCuQBWIcRNpk;>5Ysd~u8uZnd$gC;z~U1{~&zdM)@w%kB6ozI?hHt3EDniW#7 z&d%0LgS_g6Y!cNlS0fN){RI)-2wYao`O}?Z&I$QXdwo=IuQ)e=1Hr z#^TZguAf(U+lN=w$}=5ji#@Vo+MpLUGkErdmsFbKCf5@+b3AbE%O8d|!s~OG4$(9o zd2Z6~*wo0kEiXm7SNeC?DyUSloT62Jnpx4l3D$ zqRAfmNaDncJyju^>I*p*^k=3O3Quq|D0}7P&|Jz5U9_F0$%3c=@#)VpW}bm|%@3qU zABOT(LZy-J=NW@WqG+1lS&ivH&z2Es7d@Bmf@gyf3PUH@z;D0c<1*{F4&#tf96_wC z)KC|^siFA#C%56_swhh6Eu^0ptsMv2;>1ebF^8|SzR>GKzicasiN2qG$JxOT%-OJm zat_wwD_WikQS;fEm!PUit4hJeg93(*@?E9shI#&^peZLp0p}_@@FC~L+~BLJtEgC* zAe)9B<)PAoRAH2Dwn=_G{5}S{0R0+Zfi}ho~H90GZ%5+p81#u}k012r&jZVL_{3mN389P6tKdjjac?)RD$Q zj?iqV@E%>WufNpJ@Y$j=!vwsqiHAY|ABy->sUOms^#)E~r17J_rEUJ}#=YkH0n%Vu z&B~Ok;QOGvjQ7ttt#652-Wn!y?Nx4(d7^4egtx8nn$-b9mUlGeLV}IJLsW|WN8q=s zN}1S~(|rllLE*u7g5bBVw82#MTu_zk|KESaPj1USPzfHdzHk;|4MI&>Qz=W)^xpph Dksn_w diff --git a/docs/assets/t_plus_k.png b/docs/assets/t_plus_k.png deleted file mode 100644 index bee9fe56e204436a43cd3ec0a9e3755b691d7d5c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 18476 zcmbTdV~{98vnV(=&iIUN+qP}nwr$(CZCht-+qP$RzI)$&yFXs+Mr=oSS9Nu|GNUUh zv%}?NL}8&Ypa1{>V8z9R6n@+H-^&OB?Dt;lM-BJepgRexIVswjIJxRO7z6Md*%}z* ziCgQN8Y>v<8@bz$8*>2w0GpXBsX3`hOK}+5TGQzN2ZqMY+U^${0Dz0v%}(FY(%1>l zz}VE>hMVA~vzGwR+=!b%m06lj+D_2e%v{XF!C28lM#<2_(vaPVfR_i4%Z=lgfVHuc zKAxMkm5n2Z8#lp!(dGE<|7Vz%0Pnv*oGiHs{#z(DX*oPWTL)u2W*SCnLppkTJQfxj zdM0KT7J4c?20D5MTDsqpg_@p;gPDbcffet69R$DJ9E?mj6of?n7q8zZZUQqWCp!*W zT31(B8dpXdTL)8GdUkg9|IlDyp#FuRc67IK(s!e_aU}d73PQ$?h7RU-PUf~Yc>kfO zZ(!@}#7*!k>3^$WZ6_`LzX{to{x3uQnvB*>-;S1^hK|%E9*LMH!jQ#`oD-DONgZb~I=vxWd8ai7W+c=2}aTEN$ zp)oQy;-F(>`?Vq~6CEo99Sa*hJsTYxqY#sj06#N>pfH1w(Erf*KjG4|veFCCv9JgU z3X9Ow3$qBY3)2g*F$uBIF$#zXGO_*-uDFe(lfI3i@&E8O|Kz3b@&3!g z9OnNA3I4w||8HEQ|36lv{S}7xKko7WaF_oX`t`y8bpN;W|33Wh@G-XejTwjE;CN;% zHwFL@(hwKoS8`jw)CTuMUV8i4&P=afVfTpN&cvgL=pNRo%cD=xi=mKdWBZn3``>*{oYUWaXWiUp&ryn|-a?6V`X%f@gntvk zF2yl-q7q7ITe?_+1Up zqjo=%DOCRXf}zL1g7JFi!DF((q)=-vU;2#oKhYT0W~s5$;3x~0hR*#|om!7&jXV800Oat|#!Qquk*M`;BSegf z0~sg6`FAOsp9JI#w04W~W145)rA#Xp9myP3`OVJy-MA-Ut|KvcSJXH=Wx7D+apo2) zFbGbX#(|5};3MYbj!{yyLk%}zrDJ=c0i|c=4sdOe_t6mX9PzFBx@}V8}>Is21BuE5DZQAmI35kG7F~! zD+2x_w#}TJXVcSOP;Mli`i|&vis2Tytv4!t_t~TGgf>VD^e-f;(xNqRwM^)_3`Wx8 zu9_2^N8E^TP}o8zn12wS(m=jjCmK*Dw-A~FDV9#+>DEFj@FT?xuw5yT_vs(C=PidP zSN&MvPM7tMhKI-xEN}XNrrI#jJMI>gv@t#yTJLfwtIzwlYG~YCg?6uZ0&;^+l-Va_ z8WH%ibfmP11%1;yS-^&B=eBGiPd#RBJQli%ts+BJ>UUNzoqUTCHDIPSNO{P_w;Dc|n)>%o*w-`VGN z#p~Ogx^#jJxWU^zLHztS&ndma;q!f7Gi_AcpXe{T_Sr0pA)NVC&gZ$#0?KWbOQdf| zx#3C%goI@o)SiEN%n9A{^mrSC-Y!%7{2bg!<@vFpYGa0fWU2;nKRI}H7YX@*=db?M z{_KK_^<;u?*4}}FhThPXV*Qlkxy-7^7S(aLAHi0KS!G!MYY?~41 zhHJkliYcUhcN=X`#a->#t!FLpk7j^>1T!z=4REILS>v1}^26LjUQNaC3%?!yAA1Gz}J51ypDq#B=J`H^&H-of4+~yVoMU1 zGhiHsh4R;)*npj?8Qca6hA;HcA^U|~sK`Fzozk`YDor3!jj}MGPGt+D40$pu6(HeKnnuaHm|y_rMci*)GzwTEg7!C*8E1===`Vm1QgZ5O!nD`H~~53O+2$(8E4pj3>ePSjl-u zW)ia*+|lZeDAxpKhfi?lPBbgxm9(E&Ce@!wlIDVKIq1BZpsyYCATi#rFQLA$L#RSl_1j%?Ug zeJgRES9x%OGahwCJNCUb@A}7ynrKq)9Cde*Q@l`liItK=33BpC|J<%LOdSB!>d#)8 zqcy`n8F`X3vr*p>c5w@3e8_xy!hElDfix#G%z}ps=|^ZLR{I2j25k1vno3!8M-g~H zUW*#|OLa>Fv(iNk0`d^n@+Nn66gz zZ_i$m;MndmGMI6t*BE}{M=h7V+Vo$*IL0SUAOJ@g5CiVsQZzbPV23Yo%w5`Z#NBxa!aIL9POot#la<9Is#MkMMD4&Nv#N@D>8 z40WcV&9Eouc=$}`*7XImpzlyrx}L}YH2`AL&Q%3|urorgi%PF8knJg6jM-(i$PRR6 z6=|f!5i86pJhg(kNv`U?WXpZL&((Sd?(4@wXH4fYAcz}0C2F55`r4UcUb#U5UaOZO z3>=2dhRX`UX(bSlleOuF<$3p&*EfwJ?Tzc_LN67j+d6lpiO#bq@%fEJo@G|xSioJr zniJiGcq^kQPwm&*yw(f!{OcV)G^*kVw)eHsI}s`0UF$&ZW~H$lheL^-IP0A$r~&6AHAuu!@OR&Taf@y-j(Az z#E76zrglz{DT*>s`G?!MEqw*oCKTU zrg@g9IC_Z@BeP#k)0Y$;b{EyQW8Jp0t0bP|4Q}J@&_|uHTT8I%uL+di1ll0q*FG7= z55p;xxC8NlivBfB=GL9y@W6ioFO_CXN3LpA)faD1(|ee*pu+O{7~WATh|8$Zt{Fwj zzD|cA-|oo-h0yHwR_9nF;Qcj_Jw3+^YK?QFQFv3H{S7=Kx)0nnvZq9k98YT`pv0ih z?@)S5lzb@w%Z=lUBkYV{#!n@oEqhLu8QQEGPxDC#nu`S$C%;3fRfcLb59_t3fpgey z^Ih|SMKx4|vI03KuPA^QBn7;-lMc>GRf|nO0qosrAs6o9K}2$8cc?8MCfCf5?xVY< zm{E^UMT(WZKdlYHLM@1$5~V5M>K;4ZuQ%RNxBC(icGqsLZYe(0z$f=v8b1fLEuDcZ zx_9DN)R+~4#tqckn|rnzt_8V@enHLqRni{mOe?dVon7X?tp(#R-N8zoos4RCs#GQT z;s7i1=+fT!nrnM`!`#~2G{aU-6nt=}y|>g# zZ>MjBozewvmOMMjMnG;-qLm}dk&OoZP^yktC-F;s{=L~sJ+@$WKaPgMS*LQCyLQ<1 zUFCediBnGCl}_>I?}VOUf^bgji2(1-&eW~zdA2k~X;9B5hTU8}zS7N@Yj(=a2)zB^ zci<*nQ!3z*u!ct9M)-%jGHHEJfk>b0h2ibdJ%m>W6VQ%r7ii|M@XXmHt4 zZCM!)J8&p;PB30mBj?{0sn~)^efQ)lz1GV|m6v8 zEl-PhpwXtErG98!a1^$T%m5+-QEJFgn3ul8!mqj;JlE+F*`Nb}B&af49i+h09oW7o zcYNWbO_7H-2t0Ybub+fm6u$h}K}TG$y(8K4+cp@JoUnwP9{A$?X(0>?%vt*!%shUU z>s|p3)p4aW-P^RTLBj~9q|F|HR2K9(gqj)^U}6m5Sg z)3Z|BiIJL}DdCq@9w4@WziSSjlhKP@B|wt2ycO`H&P*3h_dHPqA!I}x2!EcQ%?4i>c7JFnf(=JJ_H1ic`d%=l$!<9U3t5Qvg6wtLTpb|)+5_J_ zm7)%a?gYvIphXO_+QvotmloZ~$*%d$&pz8yZXle3Z~%%B;VU&uq18A1Cso_jH01#SuC_+j>@Q?mie6p$dxN#@{*)|{!AT31*>ib1X9p3A zdhX^xWAe#RTlg!8Ga`Nz159bpBS0ScKs=^jfjI@ZBg-QlF)$*i7zf0e?sC6k9lrIJ z4njo+a#8AI7&B|H!On-@Tj3j8q2|1-64@UpjSow|R+Ai9)_47QT4uI-oytw!qM0Vp z5zxPpM?~?7soV>FT%ikF6nt8$@YxGsAGv8zbncD;y-uA8OLWPsR0II={ICEuqJ{2t zK9P8J(tw%bo(FDB)0j8xeBKn=Z6PJk_GZEXcc4#jr#O916HiyAj=YpUNa)qdaBw|} zR4d4W2CzjjU9Hwb+(S!bhCS00G5C6RtDwI(%q@6hi{_mJUx7Of1GOi!vl<%FCd#<) zpc2}9@Ggy$e6uF1J4zpA;?O6L<)*;Uy8$T>Zg$X>V~2mpPF|c}?Lc*0vC?3w!U2J) zcg%cUxD1br3)O&g0(FRj!|G&V2TgHOwVkF3w4E0{wx74CsNY_G@TY~pKfYhJd_eyt$Iqfoa^ETKl1E~k3Y|E z31YXu?VG6sZ0m)ffVGu+1Ot!C#)cfv8xV|ToBz};wKZ+4Yo`@#KLG)S+} z-T+9r6a(+!!1YyIcI&V(3Ye*n%#RAxB9W+KT16)_UNRQ%yFz%nn=9$(nL<95&oTt% zoRjWsXh6*II4`$???n7i7vG?n0z1u@BQxcDNrC+vme^EFr+M-&!59u80M@FRQ?8`t zEo4hrZSZuKK)3mPV{x>ma%w!4vl?FL30o5uW)}w+zJP{buiL}{mH49r*qWsbLwbE8 znQ2&4BGYG^TcHHjV=|iH@0?(tdVIrR@^ij)!wmhnJf4*l*1zblj_x`!#l!NHe(#z- zytL24$exC5N4rk$(f6z!Y)7x!JTE{DBM|Hbw=E%jU&JTDp0f$V-?Ry8fqm^)zWm~< zQph04_0d#>A8M^`Jo4nx-Hx=u8H+yMmf4zIwBW%Z(S1APYOw&8Gn?dL%dZyxnS;9A z5^m0ldW7ldWQ=cX($)kYeqcKdS#FeYhOq_~wrfOL!D+Y!qFX9YEz)pS-46{&vOlH8 zD{AkCcYczsxH7XQDrMLrtPopBweBOSTqNsh5lkJhX0i!Itn9~CK`-notk(sY?%BN- z^74tO$x>amf*)#9naeu3nkU+x!$QnxzsxyJ?Vs<7ZuirqF)W|WM{@9i>Xs$s+8Jsv zy&0e7&XlF9nyUqoimSV7Wy5}7I-_ibd=&NKyzSX_O}+4M7LKyCpc5fww*>=Boq=!F z44k}I0phKhVDzf*hq|qQk;8Iqk^?5#I zXM|;;3XDtEJK3i<86t87+l9GBLzK^8?-pJ%jGpHD-@@I;v|-R{Use36tuWIsoLaHq zdNc#w0Pjd|ZKb~kVsn*25gmf_9K7ied-mLgO^aYw-X<9^!0_dpQ0DUs%&~hnOcrV~{xTuee|z%M{+;Lbz-{m-wKbZFM@Bi*f$)14 z8QuTg>|*_%r#?XO_~0o^L|jAvz`G1B$6p6D=ls(Yc^4adTQ8^GQY(iFkqHLAwZB78 zC%yKl{_4DS9!#o}y9QyzL+OM%wPSp#R`b!lcFZkl1Jbaa&BUg^ql8-(ix!OXCw6lq zks^LS^P4$h^#fMm`dfk7t8=>QWS_`~lf5b~INMDV=yYh96V0(r-vBbJ$=7286#X=% znsqOV=w)NhTieZ>)vJLh3s_rQLFxxYjm}u(I?A5tT7o-M2E#sA#A5bh6<)+lE7tdk zFHL7SEvtH1&)=j!@O9&Ewe!f$CZ5QSPUnK9XgqiK1jtL@_fF^#4|bKKUoBz^3sR1D zim1?VGW(g(7c~9kI30bTBYKk*?ScXUxl?9+C)I$95a@O`v)S$`37hUKvZHABw$`pJ#Y&e6-6E(sti@rS0jgfbwNYWAP3i!vuE&Yiu0wSQY-pueV15ShGT5>7)_v z`KTz1g?-v>5-iqub1a))MvdgwE(wui&2l@6U`7q8zFFT|+nP#*W()Kv5nT+(lROWj$sm~;6+}KE z*sh~C=$z!2n8~p?&Dr8r`{<3wnG0kH@y!wef|p*ELa%l4 zSeccWzgP0z?JD~xx08j?8d#XaB|F^B8q_cGHnLhD+3xslhpRwJ2pGqZ;gH)r|QYj3|GW$Z)g>NRgXnF>_xgW?P~I)q)D$ zv3@$}CM{L;brEhRNEiqb@Cr?3SE2s9!yltG4?@MwlHWy}d;Ej9t2~vi4obE0fG*|H1SD*RGT?yE-H% z+h+nqUsR&VvsoD_8Zfl3W7fn|Yw))I{+y@v zpM}-Z^nK&>M64mc3LrS3g8Bo=PvfQSmYawY#hcPTXh8RcB{eu`0zsT$XXD<%wO`T zW61z=pdX2$gbgidHVezrp7#96zgposLD`x2COsm8?efC!wHj(5=f`Uvd@B;0F8tAh zUW`y@9KopenOZ+?WjRScBMM+&M0oLSbgG7KJe-lE%?$$C6UzcDE(#Fa`}rOETy*TOOJXN$OgWxUjTJVQ+*5z0UF^MKa7 z4CwoQQsGoSCsRq!D612R|NH#}Usx~8$rU@a!zw$yA#1rhu%zHRrCOA`6E=liqrwU0 zX>;S&^-dHRGk_EP6lPi|`*Y6-ICWu2KY?3{=VUD6LV=;7a*PhLq63M55kaSB&fssW zvx)MiTESzD5|~P_N@|Jqh$2@sCs*`nG`}A%nmr+YvJ_(h!%I0&i*(G3)JCy2U`Hg7x`Za$*u2nXVMLbC#70T6-O>@J!IX(9eqADj_ZAZ8 z?%Bn}0*&=oYyyuB6G|%aRu-aQiKVM-wZAG)wWMXWY6~Q?!Ln*vUAIVCmgAEYbgzQc z$9%Ijk&ggqo70^$#vHu=xAX^acE5HR?auswPA;LqO}ti5`gEsEtQZ6eeVCgwSs!@N zrjGp;Tafs7J~5j^6@KM33y}Sn^eDBZLXqrzXy(a5`!Q1A_yzrvc}TF;xWLV_- z+)4qTBQR4d?vLNfl%_t375yhLJG;#7UVzQ_$9z42?AO5p2jF7S-Q*X-HzA{gclj8v zeylH!gQle@gDpBV#^7PR;1eZxWIEOeHh8wJZxDyJxAfG)E>Z?Z%+b!bWyDHe^@&Qk zbWQg80cw(ozQ`R_f)uCxdwBd4FQdRbQIY@NmLo*Y!qHuu+2evAw}5D6_coi!kQ8Fr za?Z~n=qIOfMYFw@NwF^#-_{c`iSu8-8kM2khQ-9hfXMFcUq2o-hL5t90-$3l{I0Q` z!uGj}wd|A*`^(p%tC6j0Wwb-6r{&We69H0*WEYWRXc`xh78x>w2!i$}-Bv_d8!vkA z_Eisf=#fWD^RRd#>Ax$msZA~pIqr}D@GKgGzv>?VY&6IkaI7iZzIfWVO7^*{$M|{R zIgi|703^X#R2YoD4hg#Dg|0OKr-OsKH_$r^XRboc;lEwkCt9Z_xXb&RUCu88%Q6Q- zr-4v7KhjulYuibyRwi3mfohLp!!%sk)H<=G22x4HFqmxm5bh_dN=>$SB+^I;qmRHk z_4BZ`cDUN5+^pCz%MaOLsEh0~vrl+6QdBJWBx15(VX;gUHuNd9{7rx$yYa-6aT=SD zg(QXebe>bn^VDkJYenAOzu|fZxTaY$E}EBuck0S1T@MI|u8%@%_jXpXmoi+qGsZ~} zu_+gk(+=)ac$Pn{Nn(-k%L{(%Wy z1fO|9^qIe3*Ty>#{~&g-bqN36Y7wS_9KH@ajZ5GY?BgH;*B*=JK1&6n2CLm2C&9;0 z%Mrf@_vq((k*~TjA@6BrzXNG0t{!YJF3Bjh>ihkc^PwKK?C$0YB?pj^V%jS?1fkE3 z%1x?%5VVC}8Iw(KDqt=y$d%_>{8^1A25Ys^99y{f#u|ZlsgyxC9apK-Ge+hSCD%La zNSiPbJj%(+m9-%O=QbT&F^2$cTeQw-q@tcBKBv)64bfi|ZQix82MuxpQt^AAW7q5!Uj5x5PJ<=0?N_Yr^`&v(^NUhFX{o?Seo6g1$ zTI9vCcsAZ4~LjqW3;^08u?lg#+3EcBF$%mO3-yK*vrQUv29fv)FB>oGdnh?gSvE)92^t_ z(Jhpwl$#Jx{`ZAZ&b7 z=|AkZbkoFzQ>5J9Nsw#DUsX28@I6$+gTif*DA$iu^EYZEq_R@f+^})qn>8e1lrJ{q-K887yiAB;qhDCJ!Bdf zn}m3Lh=T|+Oj4RTw8X-)1j7S5tyWsY%64dhUOG5AolbMMo|uw6ntMjD@kNG&kbs$= zLFa{P)eYR?Btc-)U1YG;56IS~eHe};BAFg$6P_quH^QS%ye!IN~k0|@837Fge>4!R3pZoJW8uilN+JJeWtHaUC!+qbYmD zL$Iyx5wCrKW?!y-#N-@Yjhj1Zz6M>ML|O*GGPCEDZFdhpsQ6j?-6b&c(3V)(z?Inb zoXr6tSM1hkX!S`1sFj-OUxQ9Id zKEZp`pef1dqvPPg2fH_yc)SmMej^FEuG?tQOGfO%EZyXd9KS;A`!G)_GMilC}y0PZ(IC-~MsI0OzR1O}C2TogF0z)`a~GYk5&{6j4t@R*CXpf3 z<*PMVjwSK;D$!gp&FUYT)r<4*+X4#fWbZj|d#`N8>)^q>gy+*aR!z8L8+d|N{GtF^f(PYvrKr20UEp_;S}l5X&7ih0l^FOhbY ztlupffK)g;GSjRaV@YWO3!s+OBK-y)N#&2TklvV@Fh3fEiGDFmqn73_vysB}2li(| zNcSK)6irrlqmOTYX7Ar5dc7Dn(!Yh_ygTAcSh z_sOX^1f1IMA^;ArXn=8??O{W;6OOSesnNN*gYbN?6fpG1y~;RGC!|jG@WuUjdqg+n zB@zj5ThoAM6}<%wo@tbn8%3|NNU$=4^naWsglU6GSbF#nTqlhgU?Td_ri$u2=>ku) z59K=Kpz8+Pbs>NnLCxbs8XD=ULd6mW#Gg@MA*}pXJYW~QNBpFqp7v0Hfo|OO7j1TK ziaOX!M^utYoRq}=%xjXWz6IPV+iSFnq=fcab?I$;jI9@32fg-jdA%zVci`J3_9*i)f(9Lje!rM-lD^99R zy>%-}91DtiD0m%T01{e}VwK*H3h~LT?Udg4urm`_a>)RM<;jJUsm{BPt@~W`Rwm)d z9DKfJLU5f!^_B5}VfEIck-#*Dug)_+KbVm#I}ds__T9Ro*LY5jyJ?+3IAP}-oC0MW z`Ut5Yqa1%RxQCCM82+F=oBvcx2HB4!GPntlW1OwLF81&rMp;Z;4Yi{h#*z_?od;_s zvN5iHA3bI9$4-cQD18#wgqMY)4Q*3$&J4{qr15J#(A9V%^2z4c(<}KE8ab%s-oY)SyZHxV$+SW^ zuWop47|PYba{8d|mzHA0vQmYe!*Uy_0XZL}s#k3DPNSe6)NkyS&bM?N!1^BIfyIc( zzuV}+<{7p?`JIO>BhCHP1rAe&@%qXr8#|Sh4 ze~4D>DpBJlfEB}6zZb57s?>k zmm!`~byVf3%+WwwT|LjB{<3>L2*tlIgjUCr#y-E19}+pWX@S3ZCp5}+rYlv0B(`ha zCR-_CSvv)1ryQJ()b!Ir807oISS&k`2m(sg`_oa#cRHc%;4HxGlppC7}` z(*{DtiY{}?MKCBjOsnNAul+itqgwZe6?GL^y0UNu(1A7_SnN>1@KOr7Cxpl!tH9Rx*3E?td%TWa{` zG^c9yXjvNY0}-M!$&B>E)J=m^jRc9hGl`l@`N>7PmB)=yMI`z=l1p5SR1a~OIe|Wz zmDFFUsr}wNu;GvB895E%4?>e!}4TQ8NKi*Apn36CnK zvld$S@9NXAJKau+j+lbJNG%pz2VRO0K7XR|VI(reiz8Cxfv?)73bG~;J+g#qRZ{6> zIu{E?6A-1h!=VUF`)f2FsEW_4@ooDfjv#lbl4HQT1@UeQ~abyg=m5p724u|jeS=d-HY+`VJJwsC`N(JMfiLB;wc$T(qf zuPfsd@sb~WC)z#->azL=1>xqBTeSBDx!$cmeFO-B_{14w>_?mG6;2+=PCRK==02XU zpd-;}r=RPrybrKbts7M5wExs-A*_A&a?;>9?I+%dBT!BlBYL5(oc+98z;gK-x=YAB z>tgGc&5M401nP0GjzhREZtElqv$S`&(p3$}W1SE!)&t}NZxfHEGO})&&uI=5GMnCTiBDX5@#nE|Lax2utc@L1{+FZ zluyLj36Z^%jXT^CH!o`nb4OwEmZ#Rd7%flFaR7p8$;kq3@&|FcX74|S#uJL&=blKNyk~f1&njJ8kq%cXq z`&{0>9vBwZF_O2B%rnyIya*gKZ(-f>-_b0BVTwyBFo$~bN|&v*YJTmFRJoy`g`1e0 zT#%|iBdw`hFA5ueZQsMzZoSj$UTptyK%9o_U_Le}Iw$cFp!n;6vAJ!h-H(%Yz0p=X zW3;#b%wr&tH?Esfj*1%O(XcX1L`9lI?-rqE7N6DY+zix9N z&221Rb95$H+i>Xf{EiAgQ7xTW0G2h-C#Zo@-GluCLKi-6)|bO?5mLb{T&-7Zo{nh4hz? zYzju954U6Z2xK_CYLz$-gh!G4`mQ@D($Y4`k{)%%$TAi(!z+6kc|me%?)roDrGo{4 z*o3>k_91!+!^S6k7Z#N3EE=gBP8?|HvUUbws@0hf5^_Z?S2!{F6$)46_U5GcaZE=K zK%DWqKf~KP%{~4eFs9@@q6Zdf-|mudTjAt1u*~ebNS-zp?LJiFDew@2RRI#;l+mbz zmyh=(F$q|}h*GxFLHDdsqbG9JACBmEbAn+nuMJUJ2>lpzDAkXU05ChJ;W+nqHcjNf zohtRu37^?!#1dr%t7?oVoc|NYmuW5fMCze)x4__>j|<&;6Me^y zEo^?=`D|1`hk|KGAUce3EpkO z<&DgnDCwilaMW)I?D4Oo-l%h_YGmfl{qH|xUCJrVf?}`4NYw&7pG2l2;Nr!O-8hyL zO0FT|GHJ)wTKAvz#&@ibp;846cIN5$)o%3ST1wn4KzJ>Mbt{!Lp~B|()*cx7bz6ua zD{ARV-DShOBdLv3QOcqung|$|-vXj0H=Fr2Tb~ZZ?Q}Gmi>?Kh?kW&|0GI;s|DF;C~H+=Kke5=h}JOPb7CBm4hs$!zi*Lo>FK&?to3nL-zdmNEmI&<2Y zNoB7F5Wo_x>C^_o6#D(sxWiVcwsJ;WudX zH1I>Xx>L|M(Jr_pF#w;AFOU(zKQoYs4xw`5n z3DT3Otd9VgdtLBepOFqN`^Mjgdv~h9P^k|yWqQt(DCS5=k?B8-<1u zes*aT=93M6VL@u_XK9RmHlhS7+gM5F>1v;o2wpddkJ$SpST5nAa|aI}jh_-SR*mTb z#F1EC<>Rz{_N{tQjpwHpGv$kzAEn>w!6ov2KK^YmU0UX4!tLOfq? zWe{b8o1SqIh{#5E{#YD6>S&E%Uya?uhdAe=;K$NU)TNst+7G-*9{O~_(SKv!5C};8 zZ+MncQdMpS5D*7XOi3({Zg0$hHkHeyns z3B29bF1B}oUQu74MUJpc+*Onk9GDNcG7QuCt34JY#);rEUe&m+Z=!!*9jz)ArfE0i zlTNz6tcjeumo%x4xUEmG_}0c}gkM17_;OHInOrqTglIwC0tljq@(#skq+c7%G^>P@ zEGdYSgw-|r7%qVzpHLg!r0q$~apRo;w+F4}%N5IuJ~>z4*1xl;I9A7DGUUz-f=(pI z>SM)?XQM`3B4yJSj2Vg6rgcsE`cQObBMrHtJT$b^0jtuNKnMwXY1R=F8_ouV@NpO* zJaJBCM%t+#T&v!8LmWSFUJgmiY>pP$Qqw5MW#|qGWx?1ex;J|@FTCW*x3jSH#yWP| z;Ig)3#QDZwTimw=RjqYDZ)zJ7Y!v*ah2Xsyv=f9;rw<35)TOStZ0%t)5);)6js7%o zP?JBBfH6d6(U(w`<0}&+aQ;0hb?E&B=zXzko8S(kqwtqfd3V~Ez@18H4?fZ1tddhWuJ94_RPH!qDZ@!&}u<7~>;twy80GXdX5&sEUa zFQ1ALxZ}K{qkLspS${}K4yJudAvv{qL&~`6!^9aTr8c&_mz)qTxPip$iWGZ4bS6b) z9rW-wgSFY|^IR=!T0KH!TgUh8w2Ffw;Ios}W}{h}iC+#uaEuR}j``0I&3*?w4Cvq`?XmPBB43$}5o)BaJG4^66&M#w3qBX!l#oUSnse z`vmxo4$JN$XO)?>%(OU7?IEAzT|F!feryl6T+bPZ7nEH-ovH={p|2a19{vG3`w|hs z3Du-aENoeVjEe*X;?%g9XgEF@_|}$UrT)1zb~5L6d?9y*12jT{7W)3!19g+(TGY~w zmXD-mgpxz(_~rtjo+dZFH=~P~Q?2;6NybF~2*cx^2!NjoB1p9=aodEY{0}qcMD!?k zW>NXnsIR!{&b5@{Z?c%+73PJToWJr<1=>C@E?`jr*LDe{-U+CCJqq0fUS6s|wLBDl zAxPRD7^{Q*nf-!?=W&)0@r0DoT>hr>-Lx{~sxvtvyy4gVV00aSkGJ6Z-4vDVxp3PjDeun)f}(M} z;?M!v-x@MFOVqTA8vB&rGD>1*UWZwkrCsW3&@IgRT7}w-`WbmzQfyD@jK)VHI%&~a zS$Zj3RgiZnu5Um)8p}_tZx@yvMuGzpASfa7lx8_NlrpY^r*a%MOCTBK*>njx{S(1P zPQ)H(@=ho+;CNAw$&q#nHRgu~$tk&Efs*!(k`F@glCg#b4A45S4x%b7qerHFQb1k= za2c#j4+5H;hsTBNlD;ZX*m6`d<;SN|6mm_zjVMoK$-N5ohwa)K7^|$t;w40WiZbL-Yu{H8^qRbr_)$Slk zdNhA$Ac;9-8O}es{{mkno1+DvRZUVkViI1j1OFuPp&++g_Cd2ep-dpv!O{}o$Ls=7 zrGlYGe3$gI))N_$d~ZZ!1wP2xv@cm}Mj?GZ79}IECYT}Pi<%eB8@G=`t_o2fMxjW&SV6*2$wSirI2XPSB13(o0K-Ttn(W#2oRuBUr|0FJ-y6CDc2hyVUpXw?>R z85p+t(Et?ySgNk@)Nw}Je04LRJKhub>Naf*a6C2C-lW=L7<0$^bw?X*S17gG9=k-# z@vGxp`n=Y<@JWkDr%&Er9%6sm0rA@byQn1JVpC+FTt|r5k{rj4 z#bWCL)D}a8U>$h3Wob6FU}q;E1eB&nvlSy4nM0gH^$+K@WV=~HDf8=ygc(fZ3|{rtt}D^!b$l>?;N zk9W%JI@a|T4P;@eZ2_~5)InXu*wu35jC+CYw}y}t^X*F{iO46`@D(pd$D-Fn$Xvqn z$#wIF#-vm>BP1Kf8PkD_M?#lbBQpGe8dhewl`D$*hX?9`Z$@Ji7(J~hM zNy(!Ti&Wa|2m87i&Kp%)QPQsSqnd0b$gUi?FM~8I1odx!e|+FuX9m9op|x7wcgxm~ z>A?5}?YBblcyEw&_enxz)y-)6@f0DCFzbHcu!>@d^s-TJmfSKx-3SPzbobeNB7u2N z$MUn^SWxVUR+bUOyh!9`;cc+={;vQ$2E+M~KXhjA@r8u#Am6gkwW9%8HS<@O4?5$` zfx4%3oX8^Q92wGPFAA&5Eiw+o_I8dVwc!1)!|?8caJ^R^%MRuEzqVt|``4cx+O{}n z^Ccyy1*P4|ICDSGP)zS-8@t#q@OH>ebqy-2-{E;EF~wR0)SCF@Ch#ZTkQ=uXsEcvG zUMODoBApsV>cPxj15XzgMC;`9BxiR?CjWw0n%FJ4h~jGAbSctQ=AcShV!!K4eVZ33+FkR<}^a zw>7J0UG?++CqMr`d^H)BcBjPJQffoYFHAwT~KE@Ms%vl=e7F>wN)&^3Qm>;}~bB2^YJFh2;plPGkQNl-p@xO5mrmI?0`IWUcwW^gfR&4m}?(=&b zbjKSg*6KgS_#oT(_WB<;es=dOxjwf;&bUO5cDa>Fe!BoR8o%Fw+mvR|$xy}V>~gp!F7~F-(JBmp*5v2uM zn(Nx0gF~G!faLNEMDgo!Bjq}X2c+6*@lkK^GAQ$D;L`R?wyx{C$`aIeqAFn1C%4>R z)OGaUp7IgridX=C3ZfFzTp-QhlJ)OjeX)#I<>XlV1Z<@Erm26CBAF@<219qTWAUXB;S@EDwu# zbQx1iDv9hen&4n+vXlcGXE8VgZ}dRCmBEk{Y|@HGrr%X~%`bx{e0bTR1)m+oH=@z!j@NnbPn5tQTaLo{dFW9I?m=k_y(xV(cVZL z2uwXCRW)Vck7svDCQ-DcU4_2|@N*Lx#c*89kmxMLAuhr}QmnYCa=*n7pl~%KD(ijY zbxK#oRZ8XhFw&+#mjv`m9Net1OPj@`7NuJ*BRhg=zyRHppzdx$TF8amcWajM}? z*<}*DH6)8vb)-+Jy|N8I+nV>Uf2d~d?7!=tGWf9=<}_CU<+uNI<=YotMA3RX zqvhq!eEZQg%W4q3s5-^ zw^r2;_4T7#Q<6Fl^nCkCx3aP_7(94zBd{Y#*8cS`KJA-VO8cL1q`RcoC}5hgyP+Mq z_z%@XP`__ is designed to use this Proof-of- The following diagram is Tendermint in a (technical) nutshell. `See here for high resolution version `__. -.. figure:: images/tm-transaction-flow.png +.. figure:: assets/tm-transaction-flow.png From b46da19b74142e9c4bc1a9724c0b5503035a0a0b Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Sat, 16 Sep 2017 15:19:22 -0400 Subject: [PATCH 29/92] docs: organize the directory, #656 --- docs/app-architecture.rst | 2 +- docs/specification.rst | 22 +++++++++---------- docs/{ => specification}/block-structure.rst | 0 .../byzantine-consensus-algorithm.rst | 0 docs/{ => specification}/configuration.rst | 0 docs/{ => specification}/fast-sync.rst | 0 docs/{ => specification}/genesis.rst | 0 .../light-client-protocol.rst | 2 +- docs/{ => specification}/merkle.rst | 0 docs/{ => specification}/rpc.rst | 0 docs/{ => specification}/secure-p2p.rst | 0 docs/{ => specification}/validators.rst | 0 docs/{ => specification}/wire-protocol.rst | 0 docs/using-tendermint.rst | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) rename docs/{ => specification}/block-structure.rst (100%) rename docs/{ => specification}/byzantine-consensus-algorithm.rst (100%) rename docs/{ => specification}/configuration.rst (100%) rename docs/{ => specification}/fast-sync.rst (100%) rename docs/{ => specification}/genesis.rst (100%) rename docs/{ => specification}/light-client-protocol.rst (97%) rename docs/{ => specification}/merkle.rst (100%) rename docs/{ => specification}/rpc.rst (100%) rename docs/{ => specification}/secure-p2p.rst (100%) rename docs/{ => specification}/validators.rst (100%) rename docs/{ => specification}/wire-protocol.rst (100%) diff --git a/docs/app-architecture.rst b/docs/app-architecture.rst index 5a8287f37..e7a0d0e74 100644 --- a/docs/app-architecture.rst +++ b/docs/app-architecture.rst @@ -58,7 +58,7 @@ Tendermint Core RPC The concept is that the ABCI app is completely hidden from the outside world and only communicated through a tested and secured `interface -exposed by the tendermint core <./rpc.html>`__. This interface +exposed by the tendermint core <./specification/rpc.html>`__. This interface exposes a lot of data on the block header and consensus process, which is quite useful for externally verifying the system. It also includes 3(!) methods to broadcast a transaction (propose it for the blockchain, diff --git a/docs/specification.rst b/docs/specification.rst index 02d84c343..2e8b35668 100644 --- a/docs/specification.rst +++ b/docs/specification.rst @@ -7,14 +7,14 @@ Here you'll find details of the Tendermint specification. See `the spec repo `__ where the commit includes a majority of signatures from the last known validator set. From there, all the application state is verifiable with `merkle -proofs <./merkle-trees#iavl-tree>`__. +proofs <./merkle.html#iavl-tree>`__. Properties ---------- diff --git a/docs/merkle.rst b/docs/specification/merkle.rst similarity index 100% rename from docs/merkle.rst rename to docs/specification/merkle.rst diff --git a/docs/rpc.rst b/docs/specification/rpc.rst similarity index 100% rename from docs/rpc.rst rename to docs/specification/rpc.rst diff --git a/docs/secure-p2p.rst b/docs/specification/secure-p2p.rst similarity index 100% rename from docs/secure-p2p.rst rename to docs/specification/secure-p2p.rst diff --git a/docs/validators.rst b/docs/specification/validators.rst similarity index 100% rename from docs/validators.rst rename to docs/specification/validators.rst diff --git a/docs/wire-protocol.rst b/docs/specification/wire-protocol.rst similarity index 100% rename from docs/wire-protocol.rst rename to docs/specification/wire-protocol.rst diff --git a/docs/using-tendermint.rst b/docs/using-tendermint.rst index 0d17db611..fb9882f6b 100644 --- a/docs/using-tendermint.rst +++ b/docs/using-tendermint.rst @@ -117,7 +117,7 @@ Configuration ------------- Tendermint uses a ``config.toml`` for configutation. For details, see -`the documentation <./configuration.html>`__. +`the documentation <./specification/configuration.html>`__. Notable options include the socket address of the application (``proxy_app``), the listenting address of the tendermint peer From 044fe56b43d42d26916d568eae8219382a6d3344 Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Sat, 16 Sep 2017 15:24:30 -0400 Subject: [PATCH 30/92] update changelog --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df01f87a1..d76fabfca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,7 +28,12 @@ BUG FIXES: - Graceful handling/recovery for apps that have non-determinism or fail to halt - Graceful handling/recovery for violations of safety, or liveness -## 0.10.4 (Septemeber 5, 2017) +## 0.11.0 (Date) + +IMPROVEMENTS: + - docs: Added documentation from the tools repo to Read The Docs pipeline + +## 0.10.4 (September 5, 2017) IMPROVEMENTS: - docs: Added Slate docs to each rpc function (see rpc/core) From 583599b19fb671942c333e1f6a9cc03860db46ea Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Mon, 18 Sep 2017 09:38:22 -0400 Subject: [PATCH 31/92] docs: add software.json from website (ecosystem) --- docs/software.json | 177 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 docs/software.json diff --git a/docs/software.json b/docs/software.json new file mode 100644 index 000000000..7c8fc1346 --- /dev/null +++ b/docs/software.json @@ -0,0 +1,177 @@ +{ + "abciApps": [ + { + "name": "Cosmos SDK", + "url": "https://github.com/cosmos/cosmos-sdk", + "language": "Go", + "author": "Cosmos", + "description": "A prototypical account based crypto currency state machine supporting plugins" + }, + { + "name": "cb-ledger", + "url": "https://github.com/block-finance/cpp-abci", + "language": "C++", + "author": "Block Finance", + "description": "Custodian Bank Ledger, integrating central banking with the blockchains of tomorrow." + }, + { + "name": "Clearchain", + "url": "https://github.com/tendermint/clearchain", + "language": "Go", + "author": "Alessio Treglia", + "description": "Application to manage a distributed ledger for money transfers that support multi-currency accounts." + }, + { + "name": "Ethermint", + "url": "https://github.com/tendermint/ethermint", + "language": "Go", + "author": "Tendermint", + "description": "The go-ethereum state machine run as a ABCI app" + }, + { + "name": "Merkle AVL Tree", + "url": "https://github.com/tendermint/merkleeyes", + "language": "Go", + "author": "Tendermint", + "description": "Tendermint IAVL tree implemented as an abci app" + }, + { + "name": "Burrow", + "url": "https://github.com/hyperledger/burrow", + "language": "Go", + "author": "Monax Industries", + "description": "Ethereum Virtual Machine augmented with native permissioning scheme and global key-value store" + }, + { + "name": "Merkle AVL Tree", + "url": "https://github.com/jTMSP/MerkleTree", + "language": "Java", + "author": "jTMSP", + "description": "Tendermint IAVL tree implemented as an abci app" + }, + { + "name": "TMChat", + "url": "https://github.com/wolfposd/TMChat", + "language": "Java", + "author": "jTMSP", + "description": "P2P chat using Tendermint" + }, + { + "name": "Comit", + "url": "https://github.com/zballs/comit", + "language": "Go", + "author": "Zach Balder", + "description": "Public service reporting and tracking" + }, + { + "name": "Passwerk", + "url": "https://github.com/rigelrozanski/passwerk", + "language": "Go", + "author": "Rigel Rozanski", + "description": "Encrypted storage web-utility backed by Tendermint" + } + ], + "abciServers": [ + { + "name": "abci", + "url": "https://github.com/tendermint/abci", + "language": "Go", + "author": "Tendermint" + }, + { + "name": "js-abci", + "url": "https://github.com/tendermint/js-abci", + "language": "Javascript", + "author": "Tendermint" + }, + { + "name": "cpp-tmsp", + "url": "https://github.com/mdyring/cpp-tmsp", + "language": "C++", + "author": "Martin Dyring" + }, + { + "name": "jabci", + "url": "https://github.com/jTendermint/jabci", + "language": "Java", + "author": "jTendermint" + }, + { + "name": "Spearmint", + "url": "https://github.com/dennismckinnon/spearmint", + "language": "Javascript", + "author": "Dennis McKinnon" + }, + { + "name": "ocaml-tmsp", + "url": "https://github.com/zballs/ocaml-tmsp", + "language": "Ocaml", + "author": "Zach Balder" + }, + { + "name": "abci_server", + "url": "https://github.com/KrzysiekJ/abci_server", + "language": "Erlang", + "author": "Krzysztof Jurewicz" + } + ], + "deploymentTools": [ + { + "name": "mintnet-kubernetes", + "url": "https://github.com/tendermint/tools", + "technology": "Docker and Kubernetes", + "author": "Tendermint", + "description": "Deploy a Tendermint test network using Google's kubernetes" + }, + { + "name": "terraforce", + "url": "https://github.com/tendermint/tools", + "technology": "Terraform", + "author": "Tendermint", + "description": "Terraform + our custom terraforce tool; deploy a production Tendermint network with load balancing over multiple AWS availability zones" + }, { + "name": "ansible-tendermint", + "url": "https://github.com/tendermint/tools", + "technology": "Ansible", + "author": "Tendermint", + "description": "Ansible playbooks + Tendermint" + }, + { + "name": "brooklyn-tendermint", + "url": "https://github.com/cloudsoft/brooklyn-tendermint", + "technology": "Clocker for Apache Brooklyn ", + "author": "Cloudsoft", + "description": "Deploy a tendermint test network in docker containers " + } + ], + "competitors": [ + { + "name": "Hyperledger", + "url": "https://github.com/hyperledger/fabric", + "language": "Go", + "author": "Linux Foundation", + "description": "PBFT state machine replication for dynamic set of simple dockerized applications" + }, + { + "name": "Hydrachain", + "url": "https://github.com/HydraChain/hydrachain", + "language": "Python", + "author": "HydraChain", + "description": "Pyethereum adapted to a non-BFT consensus algorithm modelled on Tendermint." + }, + { + "name": "Juno", + "url": "https://github.com/kadena-io/juno", + "language": "Haskell", + "author": "Kadena", + "description": "Variant of Tangaroa, a BFT version of Raft" + }, + { + "name": "HoneyBadgerBFT", + "url": "https://github.com/amiller/HoneyBadgerBFT", + "language": "Python", + "author": "Andrew Miller", + "description": "Fully asynchronous and highly optimized BFT using secret-sharing" + } + ] +} From 17a748c7963a6d3a7d2c3dfb136e762305dd4241 Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Mon, 18 Sep 2017 09:39:23 -0400 Subject: [PATCH 32/92] docs: rename file --- docs/{software.json => ecosystem.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/{software.json => ecosystem.rst} (100%) diff --git a/docs/software.json b/docs/ecosystem.rst similarity index 100% rename from docs/software.json rename to docs/ecosystem.rst From ad79ead93d2d7016b6471fc1b3ef18f9967324f1 Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Mon, 18 Sep 2017 11:52:47 -0400 Subject: [PATCH 33/92] docs: add and re-format the ecosystem from website --- docs/ecosystem.rst | 276 +++++++++++++++--------------------------- docs/index.rst | 8 ++ docs/introduction.rst | 12 +- 3 files changed, 116 insertions(+), 180 deletions(-) diff --git a/docs/ecosystem.rst b/docs/ecosystem.rst index 7c8fc1346..5f5ba644f 100644 --- a/docs/ecosystem.rst +++ b/docs/ecosystem.rst @@ -1,177 +1,99 @@ -{ - "abciApps": [ - { - "name": "Cosmos SDK", - "url": "https://github.com/cosmos/cosmos-sdk", - "language": "Go", - "author": "Cosmos", - "description": "A prototypical account based crypto currency state machine supporting plugins" - }, - { - "name": "cb-ledger", - "url": "https://github.com/block-finance/cpp-abci", - "language": "C++", - "author": "Block Finance", - "description": "Custodian Bank Ledger, integrating central banking with the blockchains of tomorrow." - }, - { - "name": "Clearchain", - "url": "https://github.com/tendermint/clearchain", - "language": "Go", - "author": "Alessio Treglia", - "description": "Application to manage a distributed ledger for money transfers that support multi-currency accounts." - }, - { - "name": "Ethermint", - "url": "https://github.com/tendermint/ethermint", - "language": "Go", - "author": "Tendermint", - "description": "The go-ethereum state machine run as a ABCI app" - }, - { - "name": "Merkle AVL Tree", - "url": "https://github.com/tendermint/merkleeyes", - "language": "Go", - "author": "Tendermint", - "description": "Tendermint IAVL tree implemented as an abci app" - }, - { - "name": "Burrow", - "url": "https://github.com/hyperledger/burrow", - "language": "Go", - "author": "Monax Industries", - "description": "Ethereum Virtual Machine augmented with native permissioning scheme and global key-value store" - }, - { - "name": "Merkle AVL Tree", - "url": "https://github.com/jTMSP/MerkleTree", - "language": "Java", - "author": "jTMSP", - "description": "Tendermint IAVL tree implemented as an abci app" - }, - { - "name": "TMChat", - "url": "https://github.com/wolfposd/TMChat", - "language": "Java", - "author": "jTMSP", - "description": "P2P chat using Tendermint" - }, - { - "name": "Comit", - "url": "https://github.com/zballs/comit", - "language": "Go", - "author": "Zach Balder", - "description": "Public service reporting and tracking" - }, - { - "name": "Passwerk", - "url": "https://github.com/rigelrozanski/passwerk", - "language": "Go", - "author": "Rigel Rozanski", - "description": "Encrypted storage web-utility backed by Tendermint" - } - ], - "abciServers": [ - { - "name": "abci", - "url": "https://github.com/tendermint/abci", - "language": "Go", - "author": "Tendermint" - }, - { - "name": "js-abci", - "url": "https://github.com/tendermint/js-abci", - "language": "Javascript", - "author": "Tendermint" - }, - { - "name": "cpp-tmsp", - "url": "https://github.com/mdyring/cpp-tmsp", - "language": "C++", - "author": "Martin Dyring" - }, - { - "name": "jabci", - "url": "https://github.com/jTendermint/jabci", - "language": "Java", - "author": "jTendermint" - }, - { - "name": "Spearmint", - "url": "https://github.com/dennismckinnon/spearmint", - "language": "Javascript", - "author": "Dennis McKinnon" - }, - { - "name": "ocaml-tmsp", - "url": "https://github.com/zballs/ocaml-tmsp", - "language": "Ocaml", - "author": "Zach Balder" - }, - { - "name": "abci_server", - "url": "https://github.com/KrzysiekJ/abci_server", - "language": "Erlang", - "author": "Krzysztof Jurewicz" - } - ], - "deploymentTools": [ - { - "name": "mintnet-kubernetes", - "url": "https://github.com/tendermint/tools", - "technology": "Docker and Kubernetes", - "author": "Tendermint", - "description": "Deploy a Tendermint test network using Google's kubernetes" - }, - { - "name": "terraforce", - "url": "https://github.com/tendermint/tools", - "technology": "Terraform", - "author": "Tendermint", - "description": "Terraform + our custom terraforce tool; deploy a production Tendermint network with load balancing over multiple AWS availability zones" - }, { - "name": "ansible-tendermint", - "url": "https://github.com/tendermint/tools", - "technology": "Ansible", - "author": "Tendermint", - "description": "Ansible playbooks + Tendermint" - }, - { - "name": "brooklyn-tendermint", - "url": "https://github.com/cloudsoft/brooklyn-tendermint", - "technology": "Clocker for Apache Brooklyn ", - "author": "Cloudsoft", - "description": "Deploy a tendermint test network in docker containers " - } - ], - "competitors": [ - { - "name": "Hyperledger", - "url": "https://github.com/hyperledger/fabric", - "language": "Go", - "author": "Linux Foundation", - "description": "PBFT state machine replication for dynamic set of simple dockerized applications" - }, - { - "name": "Hydrachain", - "url": "https://github.com/HydraChain/hydrachain", - "language": "Python", - "author": "HydraChain", - "description": "Pyethereum adapted to a non-BFT consensus algorithm modelled on Tendermint." - }, - { - "name": "Juno", - "url": "https://github.com/kadena-io/juno", - "language": "Haskell", - "author": "Kadena", - "description": "Variant of Tangaroa, a BFT version of Raft" - }, - { - "name": "HoneyBadgerBFT", - "url": "https://github.com/amiller/HoneyBadgerBFT", - "language": "Python", - "author": "Andrew Miller", - "description": "Fully asynchronous and highly optimized BFT using secret-sharing" - } - ] -} +Tendermint Ecosystem +==================== + +Below are the many applications built using various pieces of the Tendermint stack. We thank the community for their contributions thus far and welcome the addition of new projects. Feel free to submit a pull request to add your project! + +ABCI Applications +----------------- + +Burrow +^^^^^^ + +Ethereum Virtual Machine augmented with native permissioning scheme and global key-value store, written in Go, authored by Monax Industries, and incubated `by Hyperledger `__. + +cb-ledger +^^^^^^^^^ + +Custodian Bank Ledger, integrating central banking with the blockchains of tomorrow, written in C++, and `authored by Block Finance `__. + +Clearchain +^^^^^^^^^^ + +Application to manage a distributed ledger for money transfers that support multi-currency accounts, written in Go, and `authored by Allession Treglia `__. + +Comit +^^^^^ + +Public service reporting and tracking, written in Go, and `authored by Zach Balder `__. + +Cosmos SDK +^^^^^^^^^^ + +A prototypical account based crypto currency state machine supporting plugins, written in Go, and `authored by Cosmos `__. + +Ethermint +^^^^^^^^^ + +The go-ethereum state machine run as a ABCI app, written in Go, `authored by Tendermint `__. + + +Merkle AVL Tree +^^^^^^^^^^^^^^^ + +The following are implementations of the Tendermint IAVL tree as an ABCI application + +Merkleeyes +~~~~~~~~~~ + +Written in Go, `authored by Tendermint `__. + +MerkleTree +~~~~~~~~~~ + +Written in Java, `authored by jTendermint `__. + + +TMChat +^^^^^^ + +P2P chat using Tendermint, written in Java, `authored by woldposd `__. + +Passwerk +^^^^^^^^ + +Encrypted storage web-utility backed by Tendermint, written in Go, `authored by Rigel Rozanski `__. + +ABCI Servers +------------ + + ++-------------------------------------------------------------+--------------------+--------------+ +| **Name** | **Author** | **Language** | +| | | | ++-------------------------------------------------------------+--------------------+--------------+ +| `abci `__ | Tendermint | Go | ++-------------------------------------------------------------+--------------------+--------------+ +| `js abci `__ | Tendermint | Javascript | ++-------------------------------------------------------------+--------------------+--------------+ +| `cpp-tmsp `__ | Martin Dyring | C++ | ++-------------------------------------------------------------+--------------------+--------------+ +| `jabci `__ | jTendermint | Java | ++-------------------------------------------------------------+--------------------+--------------+ +| `Spearmint `__ | Dennis Mckinnon | Javascript | ++-------------------------------------------------------------+--------------------+--------------+ +| `ocaml-tmsp `__ | Zach Balder | Ocaml | ++-------------------------------------------------------------+--------------------+--------------+ +| `abci_server `__ | Krzysztof Jurewicz | Erlang | ++-------------------------------------------------------------+--------------------+--------------+ + +Deployment Tools +---------------- + +See `deploy testnets <./deploy-testnets.html>`__ for information about all the tools built by Tendermint. We have Kubernetes, Ansible, and Terraform integrations. + +Cloudsoft built `brooklyn-tendermint `__ for deploying a tendermint testnet in docker continers. It used Clocker for Apache Brooklyn. + +Competitors (Collaborators?) +---------------------------- + +See `the introduction <./introduction.html>`__ for a comparison of software similar to Tendermint built by members of the blockchain community. diff --git a/docs/index.rst b/docs/index.rst index 56b1163c7..18d83b4f0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -33,6 +33,14 @@ Tendermint Tools deploy-testnets.rst +Tendermint Ecosystem +-------------------- + +.. toctree:: + :maxdepth: 2 + + ecosystem.rst + Tendermint 102 -------------- diff --git a/docs/introduction.rst b/docs/introduction.rst index cc56b1bdc..ec0350aaa 100644 --- a/docs/introduction.rst +++ b/docs/introduction.rst @@ -86,10 +86,10 @@ And we plan to do the same for Bitcoin, ZCash, and various other deterministic a Another example of a cryptocurrency application built on Tendermint is `the Cosmos network `__. -Fabric, Burrow -~~~~~~~~~~~~~~ +Other Blockchain Projects +~~~~~~~~~~~~~~~~~~~~~~~~~ -`Fabric `__, takes a similar approach to Tendermint, but is more opinionated about how the state is managed, +`Fabric `__ takes a similar approach to Tendermint, but is more opinionated about how the state is managed, and requires that all application behaviour runs in potentially many docker containers, modules it calls "chaincode". It uses an implementation of `PBFT `__. from a team at IBM that is @@ -101,6 +101,12 @@ though extending Tendermint to handle non-determinism remains for future work. with additional features for a name-registry, permissions, and native contracts, and an alternative blockchain API. It uses Tendermint as its consensus engine, and provides a particular application state. +`Hydrachain `__ is a Pyethereum adapted to a non-BFT consensus algorithm modelled on Tendermint. + +`Juno `__ is written in Haskell by Kadena and is a variant of Tangaroa, a BFT version of Raft. + +`HoneyBadgerBFT `__ is written in Python by Andrew Miller and is a fully asynchronous and highly optimized BFT using secret-sharing. + ABCI Overview ------------- From 881d2ce31e0593b2174717c0e7c21b65ee18b781 Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Mon, 18 Sep 2017 12:14:23 -0400 Subject: [PATCH 34/92] docs: pull from tools' master branch --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 627390981..d5c49355f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -172,7 +172,7 @@ texinfo_documents = [ ] repo = "https://raw.githubusercontent.com/tendermint/tools/" -branch = "cleanup-and-move-docs" +branch = "master" tools = "./tools" assets = tools + "/assets" From 77408b7bdee0d3bf8309b5fd6c9905aeaa725f93 Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Mon, 18 Sep 2017 13:31:18 -0400 Subject: [PATCH 35/92] docs: using ABCI-CLI --- docs/abci-cli.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/abci-cli.rst b/docs/abci-cli.rst index 05d56b919..5ca672c8a 100644 --- a/docs/abci-cli.rst +++ b/docs/abci-cli.rst @@ -1,5 +1,5 @@ -Using ABCI -========== +Using ABCI-CLI +============== To facilitate testing and debugging of ABCI servers and simple apps, we built a CLI, the ``abci-cli``, for sending ABCI messages from the From 1d3f723cccba2bc547485fa2bb04df53d03a4f21 Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Mon, 18 Sep 2017 13:14:27 -0400 Subject: [PATCH 36/92] docs: remove last section from ecosystem --- docs/ecosystem.rst | 5 ----- docs/introduction.rst | 6 ------ 2 files changed, 11 deletions(-) diff --git a/docs/ecosystem.rst b/docs/ecosystem.rst index 5f5ba644f..876a195d2 100644 --- a/docs/ecosystem.rst +++ b/docs/ecosystem.rst @@ -92,8 +92,3 @@ Deployment Tools See `deploy testnets <./deploy-testnets.html>`__ for information about all the tools built by Tendermint. We have Kubernetes, Ansible, and Terraform integrations. Cloudsoft built `brooklyn-tendermint `__ for deploying a tendermint testnet in docker continers. It used Clocker for Apache Brooklyn. - -Competitors (Collaborators?) ----------------------------- - -See `the introduction <./introduction.html>`__ for a comparison of software similar to Tendermint built by members of the blockchain community. diff --git a/docs/introduction.rst b/docs/introduction.rst index ec0350aaa..cfda74ed7 100644 --- a/docs/introduction.rst +++ b/docs/introduction.rst @@ -101,12 +101,6 @@ though extending Tendermint to handle non-determinism remains for future work. with additional features for a name-registry, permissions, and native contracts, and an alternative blockchain API. It uses Tendermint as its consensus engine, and provides a particular application state. -`Hydrachain `__ is a Pyethereum adapted to a non-BFT consensus algorithm modelled on Tendermint. - -`Juno `__ is written in Haskell by Kadena and is a variant of Tangaroa, a BFT version of Raft. - -`HoneyBadgerBFT `__ is written in Python by Andrew Miller and is a fully asynchronous and highly optimized BFT using secret-sharing. - ABCI Overview ------------- From 740167202fca613828e242f4b8b06a8707bb5a43 Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Mon, 18 Sep 2017 16:23:22 -0400 Subject: [PATCH 37/92] readme & al., update links to docs --- CONTRIBUTING.md | 2 +- README.md | 80 +++++++++++++++++++++++++++++++--------------- docs/ecosystem.rst | 2 +- 3 files changed, 56 insertions(+), 28 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f5ceee3cf..a996acad5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing -Thank you for considering making contributions to Tendermint and related repositories (Basecoin, Merkleeyes, etc.)! +Thank you for considering making contributions to Tendermint and related repositories! Start by taking a look at the [coding repo](https://github.com/tendermint/coding) for overall information on repository workflow and standards. Please follow standard github best practices: fork the repo, branch from the tip of develop, make some commits, and submit a pull request to develop. See the [open issues](https://github.com/tendermint/tendermint/issues) for things we need help with! diff --git a/README.md b/README.md index f6ef76042..7238e51bd 100644 --- a/README.md +++ b/README.md @@ -13,22 +13,17 @@ https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/6874 [![](https://tokei.rs/b1/github/tendermint/tendermint?category=lines)](https://github.com/tendermint/tendermint) -Branch | Tests | Coverage | Report Card -----------|-------|----------|------------- -develop | [![CircleCI](https://circleci.com/gh/tendermint/tendermint/tree/develop.svg?style=shield)](https://circleci.com/gh/tendermint/tendermint/tree/develop) | [![codecov](https://codecov.io/gh/tendermint/tendermint/branch/develop/graph/badge.svg)](https://codecov.io/gh/tendermint/tendermint) | [![Go Report Card](https://goreportcard.com/badge/github.com/tendermint/tendermint/tree/develop)](https://goreportcard.com/report/github.com/tendermint/tendermint/tree/develop) -master | [![CircleCI](https://circleci.com/gh/tendermint/tendermint/tree/master.svg?style=shield)](https://circleci.com/gh/tendermint/tendermint/tree/master) | [![codecov](https://codecov.io/gh/tendermint/tendermint/branch/master/graph/badge.svg)](https://codecov.io/gh/tendermint/tendermint) | [![Go Report Card](https://goreportcard.com/badge/github.com/tendermint/tendermint/tree/master)](https://goreportcard.com/report/github.com/tendermint/tendermint/tree/master) +Branch | Tests | +----------|-------| +master | [![CircleCI](https://circleci.com/gh/tendermint/tendermint/tree/master.svg?style=shield)](https://circleci.com/gh/tendermint/tendermint/tree/master) +develop | [![CircleCI](https://circleci.com/gh/tendermint/tendermint/tree/develop.svg?style=shield)](https://circleci.com/gh/tendermint/tendermint/tree/develop) _NOTE: This is alpha software. Please contact us if you intend to run it in production._ -Tendermint Core is Byzantine Fault Tolerant (BFT) middleware that takes a state transition machine, written in any programming language, +Tendermint Core is Byzantine Fault Tolerant (BFT) middleware that takes a state transition machine - written in any programming language - and securely replicates it on many machines. -For more background, see the [introduction](https://tendermint.com/intro). - -To get started developing applications, see the [application developers guide](https://tendermint.com/docs/guides/app-development). - -### Code of Conduct -Please read, understand and adhere to our [code of conduct](CODE_OF_CONDUCT.md). +For more information, from introduction to install to application development, [Read The Docs](http://tendermint.readthedocs.io/projects/tools/en/master/index.html). ## Install @@ -38,39 +33,72 @@ To install from source, you should be able to: `go get -u github.com/tendermint/tendermint/cmd/tendermint` -For more details (or if it fails), see the [install guide](https://tendermint.com/docs/guides/install-from-source). - -## Contributing - -Yay open source! Please see our [contributing guidelines](CONTRIBUTING.md). +For more details (or if it fails), [read the docs](http://tendermint.readthedocs.io/projects/tools/en/master/install.html). ## Resources ### Tendermint Core -- [Introduction](https://tendermint.com/intro) -- [Docs](https://tendermint.com/docs) -- [Software using Tendermint](https://tendermint.com/ecosystem) +All resources involving the use of, building application on, or developing for, tendermint, can be found at [Read The Docs](http://tendermint.readthedocs.io/projects/tools/en/master/index.html). Additional information about some - and eventually all - of the sub-projects below, can be found at Read The Docs. ### Sub-projects * [ABCI](http://github.com/tendermint/abci), the Application Blockchain Interface * [Go-Wire](http://github.com/tendermint/go-wire), a deterministic serialization library * [Go-Crypto](http://github.com/tendermint/go-crypto), an elliptic curve cryptography library -* [TmLibs](http://github.com/tendermint/tmlibs), an assortment of Go libraries -* [Merkleeyes](http://github.com/tendermint/merkleeyes), a balanced, binary Merkle tree for ABCI apps +* [TmLibs](http://github.com/tendermint/tmlibs), an assortment of Go libraries used internally +* [IAVL](http://github.com/tendermint/iavl), Merkleized IAVL+ Tree implementation ### Tools -* [Deployment, Benchmarking, and Monitoring](https://github.com/tendermint/tools) +* [Deployment, Benchmarking, and Monitoring](http://tendermint.readthedocs.io/projects/tools/en/develop/index.html#tendermint-tools) ### Applications -* [Ethermint](http://github.com/tendermint/ethermint): Ethereum on Tendermint -* [Basecoin](http://github.com/tendermint/basecoin), a cryptocurrency application framework +* [Ethermint](http://github.com/tendermint/ethermint); Ethereum on Tendermint +* [Cosmos SDK](http://github.com/cosmos/cosmos-sdk); a cryptocurrency application framework ### More +* [Master's Thesis on Tendermint](https://atrium.lib.uoguelph.ca/xmlui/handle/10214/9769) +* [Original Whitepaper](https://tendermint.com/static/docs/tendermint.pdf) * [Tendermint Blog](https://blog.cosmos.network/tendermint/home) * [Cosmos Blog](https://blog.cosmos.network) -* [Original Whitepaper (out-of-date)](http://www.the-blockchain.com/docs/Tendermint%20Consensus%20without%20Mining.pdf) -* [Master's Thesis on Tendermint](https://atrium.lib.uoguelph.ca/xmlui/handle/10214/9769) + +## Contributing + +Yay open source! Please see our [contributing guidelines](CONTRIBUTING.md). + +## Versioning + +### SemVer + +Tendermint uses [SemVer](http://semver.org/) to determine when and how the version changes. +According to SemVer, anything in the public API can change at any time before version 1.0.0 + +To provide some stability to Tendermint users in these 0.X.X days, the MINOR version is used +to signal breaking changes across a subset of the total public API. This subset includes all +interfaces exposed to other processes (cli, rpc, p2p, etc.), as well as parts of the following packages: + +- types +- rpc/client +- config +- node + +Exported objects in these packages that are not covered by the versioning scheme +are explicitly marked by `// UNSTABLE` in their go doc comment and may change at any time. +Functions, types, and values in any other package may also change at any time. + +### Upgrades + +In an effort to avoid accumulating technical debt prior to 1.0.0, +we do not guarantee that breaking changes (ie. bumps in the MINOR version) +will work with existing tendermint blockchains. In these cases you will +have to start a new blockchain, or write something custom to get the old +data into the new chain. + +However, any bump in the PATCH version should be compatible with existing histories +(if not please open an [issue](https://github.com/tendermint/tendermint/issues)). + +### Code of Conduct + +Please read, understand and adhere to our [code of conduct](CODE_OF_CONDUCT.md). diff --git a/docs/ecosystem.rst b/docs/ecosystem.rst index 876a195d2..188f50092 100644 --- a/docs/ecosystem.rst +++ b/docs/ecosystem.rst @@ -91,4 +91,4 @@ Deployment Tools See `deploy testnets <./deploy-testnets.html>`__ for information about all the tools built by Tendermint. We have Kubernetes, Ansible, and Terraform integrations. -Cloudsoft built `brooklyn-tendermint `__ for deploying a tendermint testnet in docker continers. It used Clocker for Apache Brooklyn. +Cloudsoft built `brooklyn-tendermint `__ for deploying a tendermint testnet in docker continers. It uses Clocker for Apache Brooklyn. From 60a1f49a5c70a5b4cfd100a8d9f9329b7e359969 Mon Sep 17 00:00:00 2001 From: Dave Bryson Date: Fri, 26 May 2017 14:46:33 +0200 Subject: [PATCH 38/92] updated json response to match spec by @davebryson --- rpc/core/events.go | 4 +-- rpc/lib/client/http_client.go | 2 +- rpc/lib/server/handlers.go | 33 ++++++++++---------- rpc/lib/server/http_server.go | 2 +- rpc/lib/types/types.go | 57 +++++++++++++++++++++++++++-------- rpc/lib/types/types_test.go | 32 ++++++++++++++++++++ 6 files changed, 97 insertions(+), 33 deletions(-) create mode 100644 rpc/lib/types/types_test.go diff --git a/rpc/core/events.go b/rpc/core/events.go index 4671d3417..00fd9a08c 100644 --- a/rpc/core/events.go +++ b/rpc/core/events.go @@ -2,7 +2,7 @@ package core import ( ctypes "github.com/tendermint/tendermint/rpc/core/types" - "github.com/tendermint/tendermint/rpc/lib/types" + rpctypes "github.com/tendermint/tendermint/rpc/lib/types" "github.com/tendermint/tendermint/types" ) @@ -39,7 +39,7 @@ func Subscribe(wsCtx rpctypes.WSRPCContext, event string) (*ctypes.ResultSubscri // NOTE: EventSwitch callbacks must be nonblocking // NOTE: RPCResponses of subscribed events have id suffix "#event" tmResult := &ctypes.ResultEvent{event, msg} - wsCtx.TryWriteRPCResponse(rpctypes.NewRPCResponse(wsCtx.Request.ID+"#event", tmResult, "")) + wsCtx.TryWriteRPCResponse(rpctypes.NewRPCSuccessResponse(wsCtx.Request.ID+"#event", tmResult)) }) return &ctypes.ResultSubscribe{}, nil } diff --git a/rpc/lib/client/http_client.go b/rpc/lib/client/http_client.go index 755c3e79c..bd3846c64 100644 --- a/rpc/lib/client/http_client.go +++ b/rpc/lib/client/http_client.go @@ -146,7 +146,7 @@ func unmarshalResponseBytes(responseBytes []byte, result interface{}) (interface if err != nil { return nil, errors.Errorf("Error unmarshalling rpc response: %v", err) } - errorStr := response.Error + errorStr := response.Error.Message if errorStr != "" { return nil, errors.Errorf("Response error: %v", errorStr) } diff --git a/rpc/lib/server/handlers.go b/rpc/lib/server/handlers.go index 3b81567e4..a4df4b840 100644 --- a/rpc/lib/server/handlers.go +++ b/rpc/lib/server/handlers.go @@ -110,35 +110,35 @@ func makeJSONRPCHandler(funcMap map[string]*RPCFunc, logger log.Logger) http.Han var request types.RPCRequest err := json.Unmarshal(b, &request) if err != nil { - WriteRPCResponseHTTPError(w, http.StatusBadRequest, types.NewRPCResponse("", nil, fmt.Sprintf("Error unmarshalling request: %v", err.Error()))) + WriteRPCResponseHTTP(w, types.RPCParseError("")) return } if len(r.URL.Path) > 1 { - WriteRPCResponseHTTPError(w, http.StatusNotFound, types.NewRPCResponse(request.ID, nil, fmt.Sprintf("Invalid JSONRPC endpoint %s", r.URL.Path))) + WriteRPCResponseHTTP(w, types.RPCInvalidRequestError(request.ID)) return } rpcFunc := funcMap[request.Method] if rpcFunc == nil { - WriteRPCResponseHTTPError(w, http.StatusNotFound, types.NewRPCResponse(request.ID, nil, "RPC method unknown: "+request.Method)) + WriteRPCResponseHTTP(w, types.RPCMethodNotFoundError(request.ID)) return } if rpcFunc.ws { - WriteRPCResponseHTTPError(w, http.StatusMethodNotAllowed, types.NewRPCResponse(request.ID, nil, "RPC method is only for websockets: "+request.Method)) + WriteRPCResponseHTTP(w, types.RPCInternalError(request.ID)) return } args, err := jsonParamsToArgsRPC(rpcFunc, request.Params) if err != nil { - WriteRPCResponseHTTPError(w, http.StatusBadRequest, types.NewRPCResponse(request.ID, nil, fmt.Sprintf("Error converting json params to arguments: %v", err.Error()))) + WriteRPCResponseHTTP(w, types.RPCInvalidParamsError(request.ID)) return } returns := rpcFunc.f.Call(args) logger.Info("HTTPJSONRPC", "method", request.Method, "args", args, "returns", returns) result, err := unreflectResult(returns) if err != nil { - WriteRPCResponseHTTPError(w, http.StatusInternalServerError, types.NewRPCResponse(request.ID, result, err.Error())) + WriteRPCResponseHTTP(w, types.RPCInternalError(request.ID)) return } - WriteRPCResponseHTTP(w, types.NewRPCResponse(request.ID, result, "")) + WriteRPCResponseHTTP(w, types.NewRPCSuccessResponse(request.ID, result)) } } @@ -229,7 +229,7 @@ func makeHTTPHandler(rpcFunc *RPCFunc, logger log.Logger) func(http.ResponseWrit // Exception for websocket endpoints if rpcFunc.ws { return func(w http.ResponseWriter, r *http.Request) { - WriteRPCResponseHTTPError(w, http.StatusMethodNotAllowed, types.NewRPCResponse("", nil, "This RPC method is only for websockets")) + WriteRPCResponseHTTP(w, types.RPCInternalError("")) } } // All other endpoints @@ -237,17 +237,17 @@ func makeHTTPHandler(rpcFunc *RPCFunc, logger log.Logger) func(http.ResponseWrit logger.Debug("HTTP HANDLER", "req", r) args, err := httpParamsToArgs(rpcFunc, r) if err != nil { - WriteRPCResponseHTTPError(w, http.StatusBadRequest, types.NewRPCResponse("", nil, fmt.Sprintf("Error converting http params to args: %v", err.Error()))) + WriteRPCResponseHTTP(w, types.RPCInvalidParamsError("")) return } returns := rpcFunc.f.Call(args) logger.Info("HTTPRestRPC", "method", r.URL.Path, "args", args, "returns", returns) result, err := unreflectResult(returns) if err != nil { - WriteRPCResponseHTTPError(w, http.StatusInternalServerError, types.NewRPCResponse("", nil, err.Error())) + WriteRPCResponseHTTP(w, types.RPCInternalError("")) return } - WriteRPCResponseHTTP(w, types.NewRPCResponse("", result, "")) + WriteRPCResponseHTTP(w, types.NewRPCSuccessResponse("", result)) } } @@ -509,8 +509,7 @@ func (wsc *wsConnection) readRoutine() { var request types.RPCRequest err = json.Unmarshal(in, &request) if err != nil { - errStr := fmt.Sprintf("Error unmarshaling data: %s", err.Error()) - wsc.WriteRPCResponse(types.NewRPCResponse(request.ID, nil, errStr)) + wsc.WriteRPCResponse(types.RPCParseError("")) continue } @@ -518,7 +517,7 @@ func (wsc *wsConnection) readRoutine() { rpcFunc := wsc.funcMap[request.Method] if rpcFunc == nil { - wsc.WriteRPCResponse(types.NewRPCResponse(request.ID, nil, "RPC method unknown: "+request.Method)) + wsc.WriteRPCResponse(types.RPCMethodNotFoundError(request.ID)) continue } var args []reflect.Value @@ -529,7 +528,7 @@ func (wsc *wsConnection) readRoutine() { args, err = jsonParamsToArgsRPC(rpcFunc, request.Params) } if err != nil { - wsc.WriteRPCResponse(types.NewRPCResponse(request.ID, nil, err.Error())) + wsc.WriteRPCResponse(types.RPCInternalError(request.ID)) continue } returns := rpcFunc.f.Call(args) @@ -539,10 +538,10 @@ func (wsc *wsConnection) readRoutine() { result, err := unreflectResult(returns) if err != nil { - wsc.WriteRPCResponse(types.NewRPCResponse(request.ID, nil, err.Error())) + wsc.WriteRPCResponse(types.RPCInternalError(request.ID)) continue } else { - wsc.WriteRPCResponse(types.NewRPCResponse(request.ID, result, "")) + wsc.WriteRPCResponse(types.NewRPCSuccessResponse(request.ID, result)) continue } diff --git a/rpc/lib/server/http_server.go b/rpc/lib/server/http_server.go index 270321b4b..2d6f5f1ba 100644 --- a/rpc/lib/server/http_server.go +++ b/rpc/lib/server/http_server.go @@ -99,7 +99,7 @@ func RecoverAndLogHandler(handler http.Handler, logger log.Logger) http.Handler // For the rest, logger.Error("Panic in RPC HTTP handler", "err", e, "stack", string(debug.Stack())) rww.WriteHeader(http.StatusInternalServerError) - WriteRPCResponseHTTP(rww, types.NewRPCResponse("", nil, fmt.Sprintf("Internal Server Error: %v", e))) + WriteRPCResponseHTTP(rww, types.RPCInternalError("")) } } diff --git a/rpc/lib/types/types.go b/rpc/lib/types/types.go index f4a2cede0..528255a83 100644 --- a/rpc/lib/types/types.go +++ b/rpc/lib/types/types.go @@ -8,6 +8,11 @@ import ( events "github.com/tendermint/tmlibs/events" ) +type RpcError struct { + Code int `json:"code"` + Message string `json:"message"` +} + type RPCRequest struct { JSONRPC string `json:"jsonrpc"` ID string `json:"id"` @@ -50,28 +55,32 @@ func ArrayToRequest(id string, method string, params []interface{}) (RPCRequest, type RPCResponse struct { JSONRPC string `json:"jsonrpc"` - ID string `json:"id"` - Result *json.RawMessage `json:"result"` - Error string `json:"error"` + ID string `json:"id,omitempty"` + Result *json.RawMessage `json:"result,omitempty"` + Error *RpcError `json:"error,omitempty"` } -func NewRPCResponse(id string, res interface{}, err string) RPCResponse { +func NewRPCSuccessResponse(id string, res interface{}) RPCResponse { var raw *json.RawMessage + if res != nil { var js []byte - js, err2 := json.Marshal(res) - if err2 == nil { - rawMsg := json.RawMessage(js) - raw = &rawMsg - } else { - err = err2.Error() + js, err := json.Marshal(res) + if err != nil { + return RPCInternalError(id) } + rawMsg := json.RawMessage(js) + raw = &rawMsg } + + return RPCResponse{JSONRPC: "2.0", ID: id, Result: raw} +} + +func NewRPCErrorResponse(id string, code int, msg string) RPCResponse { return RPCResponse{ JSONRPC: "2.0", ID: id, - Result: raw, - Error: err, + Error: &RpcError{Code: code, Message: msg}, } } @@ -83,6 +92,30 @@ func (resp RPCResponse) String() string { } } +func RPCParseError(id string) RPCResponse { + return NewRPCErrorResponse(id, -32700, "Parse error. Invalid JSON") +} + +func RPCInvalidRequestError(id string) RPCResponse { + return NewRPCErrorResponse(id, -32600, "Invalid Request") +} + +func RPCMethodNotFoundError(id string) RPCResponse { + return NewRPCErrorResponse(id, -32601, "Method not found") +} + +func RPCInvalidParamsError(id string) RPCResponse { + return NewRPCErrorResponse(id, -32602, "Invalid params") +} + +func RPCInternalError(id string) RPCResponse { + return NewRPCErrorResponse(id, -32603, "Internal error") +} + +func RPCServerError(id string) RPCResponse { + return NewRPCErrorResponse(id, -32000, "Server error") +} + //---------------------------------------- // *wsConnection implements this interface. diff --git a/rpc/lib/types/types_test.go b/rpc/lib/types/types_test.go new file mode 100644 index 000000000..14688ecee --- /dev/null +++ b/rpc/lib/types/types_test.go @@ -0,0 +1,32 @@ +package rpctypes + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" +) + +type SampleResult struct { + Value string +} + +func TestResponses(t *testing.T) { + assert := assert.New(t) + + a := NewRPCSuccessResponse("1", &SampleResult{"hello"}) + b, _ := json.Marshal(a) + s := `{"jsonrpc":"2.0","id":"1","result":{"Value":"hello"}}` + assert.Equal(string(s), string(b)) + + d := RPCParseError("1") + e, _ := json.Marshal(d) + f := `{"jsonrpc":"2.0","id":"1","error":{"code":-32700,"message":"Parse error. Invalid JSON"}}` + assert.Equal(string(f), string(e)) + + g := RPCMethodNotFoundError("2") + h, _ := json.Marshal(g) + i := `{"jsonrpc":"2.0","id":"2","error":{"code":-32601,"message":"Method not found"}}` + assert.Equal(string(h), string(i)) + +} From 6c1572c9b8c701bc6fe64ea308282ef1fe962271 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Fri, 26 May 2017 16:56:44 +0200 Subject: [PATCH 39/92] fix invalid memory address or nil pointer dereference --- rpc/lib/client/http_client.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/rpc/lib/client/http_client.go b/rpc/lib/client/http_client.go index bd3846c64..734ac5b17 100644 --- a/rpc/lib/client/http_client.go +++ b/rpc/lib/client/http_client.go @@ -146,9 +146,8 @@ func unmarshalResponseBytes(responseBytes []byte, result interface{}) (interface if err != nil { return nil, errors.Errorf("Error unmarshalling rpc response: %v", err) } - errorStr := response.Error.Message - if errorStr != "" { - return nil, errors.Errorf("Response error: %v", errorStr) + if response.Error != nil && response.Error.Message != "" { + return nil, errors.Errorf("Response error: %v", response.Error.Message) } // unmarshal the RawMessage into the result err = json.Unmarshal(*response.Result, result) From f74de4cb8664f915ea8216bd76ad65d7736d4614 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Fri, 26 May 2017 17:45:09 +0200 Subject: [PATCH 40/92] include optional data field in error object ``` data A Primitive or Structured value that contains additional information about the error. This may be omitted. The value of this member is defined by the Server (e.g. detailed error information, nested errors etc.). ``` --- rpc/lib/server/handlers.go | 22 ++++++++--------- rpc/lib/server/http_server.go | 3 ++- rpc/lib/types/types.go | 45 ++++++++++++++++++++--------------- rpc/lib/types/types_test.go | 6 ++--- 4 files changed, 42 insertions(+), 34 deletions(-) diff --git a/rpc/lib/server/handlers.go b/rpc/lib/server/handlers.go index a4df4b840..070072d8f 100644 --- a/rpc/lib/server/handlers.go +++ b/rpc/lib/server/handlers.go @@ -110,11 +110,11 @@ func makeJSONRPCHandler(funcMap map[string]*RPCFunc, logger log.Logger) http.Han var request types.RPCRequest err := json.Unmarshal(b, &request) if err != nil { - WriteRPCResponseHTTP(w, types.RPCParseError("")) + WriteRPCResponseHTTP(w, types.RPCParseError("", errors.Wrap(err, "Error unmarshalling request"))) return } if len(r.URL.Path) > 1 { - WriteRPCResponseHTTP(w, types.RPCInvalidRequestError(request.ID)) + WriteRPCResponseHTTP(w, types.RPCInvalidRequestError(request.ID, errors.Errorf("Path %s is invalid", r.URL.Path))) return } rpcFunc := funcMap[request.Method] @@ -123,19 +123,19 @@ func makeJSONRPCHandler(funcMap map[string]*RPCFunc, logger log.Logger) http.Han return } if rpcFunc.ws { - WriteRPCResponseHTTP(w, types.RPCInternalError(request.ID)) + WriteRPCResponseHTTP(w, types.RPCInternalError(request.ID, errors.New("Trying to use Websocket method in non-ws context"))) return } args, err := jsonParamsToArgsRPC(rpcFunc, request.Params) if err != nil { - WriteRPCResponseHTTP(w, types.RPCInvalidParamsError(request.ID)) + WriteRPCResponseHTTP(w, types.RPCInvalidParamsError(request.ID, errors.Wrap(err, "Error converting json params to arguments"))) return } returns := rpcFunc.f.Call(args) logger.Info("HTTPJSONRPC", "method", request.Method, "args", args, "returns", returns) result, err := unreflectResult(returns) if err != nil { - WriteRPCResponseHTTP(w, types.RPCInternalError(request.ID)) + WriteRPCResponseHTTP(w, types.RPCInternalError(request.ID, err)) return } WriteRPCResponseHTTP(w, types.NewRPCSuccessResponse(request.ID, result)) @@ -229,7 +229,7 @@ func makeHTTPHandler(rpcFunc *RPCFunc, logger log.Logger) func(http.ResponseWrit // Exception for websocket endpoints if rpcFunc.ws { return func(w http.ResponseWriter, r *http.Request) { - WriteRPCResponseHTTP(w, types.RPCInternalError("")) + WriteRPCResponseHTTP(w, types.RPCInternalError("", errors.New("Trying to use Websocket method in non-ws context"))) } } // All other endpoints @@ -237,14 +237,14 @@ func makeHTTPHandler(rpcFunc *RPCFunc, logger log.Logger) func(http.ResponseWrit logger.Debug("HTTP HANDLER", "req", r) args, err := httpParamsToArgs(rpcFunc, r) if err != nil { - WriteRPCResponseHTTP(w, types.RPCInvalidParamsError("")) + WriteRPCResponseHTTP(w, types.RPCInvalidParamsError("", errors.Wrap(err, "Error converting http params to arguments"))) return } returns := rpcFunc.f.Call(args) logger.Info("HTTPRestRPC", "method", r.URL.Path, "args", args, "returns", returns) result, err := unreflectResult(returns) if err != nil { - WriteRPCResponseHTTP(w, types.RPCInternalError("")) + WriteRPCResponseHTTP(w, types.RPCInternalError("", err)) return } WriteRPCResponseHTTP(w, types.NewRPCSuccessResponse("", result)) @@ -509,7 +509,7 @@ func (wsc *wsConnection) readRoutine() { var request types.RPCRequest err = json.Unmarshal(in, &request) if err != nil { - wsc.WriteRPCResponse(types.RPCParseError("")) + wsc.WriteRPCResponse(types.RPCParseError("", errors.Wrap(err, "Error unmarshaling request"))) continue } @@ -528,7 +528,7 @@ func (wsc *wsConnection) readRoutine() { args, err = jsonParamsToArgsRPC(rpcFunc, request.Params) } if err != nil { - wsc.WriteRPCResponse(types.RPCInternalError(request.ID)) + wsc.WriteRPCResponse(types.RPCInternalError(request.ID, errors.Wrap(err, "Error converting json params to arguments"))) continue } returns := rpcFunc.f.Call(args) @@ -538,7 +538,7 @@ func (wsc *wsConnection) readRoutine() { result, err := unreflectResult(returns) if err != nil { - wsc.WriteRPCResponse(types.RPCInternalError(request.ID)) + wsc.WriteRPCResponse(types.RPCInternalError(request.ID, err)) continue } else { wsc.WriteRPCResponse(types.NewRPCSuccessResponse(request.ID, result)) diff --git a/rpc/lib/server/http_server.go b/rpc/lib/server/http_server.go index 2d6f5f1ba..7623337db 100644 --- a/rpc/lib/server/http_server.go +++ b/rpc/lib/server/http_server.go @@ -12,6 +12,7 @@ import ( "time" "github.com/pkg/errors" + types "github.com/tendermint/tendermint/rpc/lib/types" "github.com/tendermint/tmlibs/log" ) @@ -99,7 +100,7 @@ func RecoverAndLogHandler(handler http.Handler, logger log.Logger) http.Handler // For the rest, logger.Error("Panic in RPC HTTP handler", "err", e, "stack", string(debug.Stack())) rww.WriteHeader(http.StatusInternalServerError) - WriteRPCResponseHTTP(rww, types.RPCInternalError("")) + WriteRPCResponseHTTP(rww, types.RPCInternalError("", e.(error))) } } diff --git a/rpc/lib/types/types.go b/rpc/lib/types/types.go index 528255a83..b649d1b94 100644 --- a/rpc/lib/types/types.go +++ b/rpc/lib/types/types.go @@ -5,13 +5,13 @@ import ( "fmt" "strings" + "github.com/pkg/errors" + events "github.com/tendermint/tmlibs/events" ) -type RpcError struct { - Code int `json:"code"` - Message string `json:"message"` -} +//---------------------------------------- +// REQUEST type RPCRequest struct { JSONRPC string `json:"jsonrpc"` @@ -52,6 +52,13 @@ func ArrayToRequest(id string, method string, params []interface{}) (RPCRequest, } //---------------------------------------- +// RESPONSE + +type RpcError struct { + Code int `json:"code"` + Message string `json:"message"` + Data string `json:"data,omitempty"` +} type RPCResponse struct { JSONRPC string `json:"jsonrpc"` @@ -67,7 +74,7 @@ func NewRPCSuccessResponse(id string, res interface{}) RPCResponse { var js []byte js, err := json.Marshal(res) if err != nil { - return RPCInternalError(id) + return RPCInternalError(id, errors.Wrap(err, "Error marshalling response")) } rawMsg := json.RawMessage(js) raw = &rawMsg @@ -76,11 +83,11 @@ func NewRPCSuccessResponse(id string, res interface{}) RPCResponse { return RPCResponse{JSONRPC: "2.0", ID: id, Result: raw} } -func NewRPCErrorResponse(id string, code int, msg string) RPCResponse { +func NewRPCErrorResponse(id string, code int, msg string, data string) RPCResponse { return RPCResponse{ JSONRPC: "2.0", ID: id, - Error: &RpcError{Code: code, Message: msg}, + Error: &RpcError{Code: code, Message: msg, Data: data}, } } @@ -92,28 +99,28 @@ func (resp RPCResponse) String() string { } } -func RPCParseError(id string) RPCResponse { - return NewRPCErrorResponse(id, -32700, "Parse error. Invalid JSON") +func RPCParseError(id string, err error) RPCResponse { + return NewRPCErrorResponse(id, -32700, "Parse error. Invalid JSON", err.Error()) } -func RPCInvalidRequestError(id string) RPCResponse { - return NewRPCErrorResponse(id, -32600, "Invalid Request") +func RPCInvalidRequestError(id string, err error) RPCResponse { + return NewRPCErrorResponse(id, -32600, "Invalid Request", err.Error()) } func RPCMethodNotFoundError(id string) RPCResponse { - return NewRPCErrorResponse(id, -32601, "Method not found") + return NewRPCErrorResponse(id, -32601, "Method not found", "") } -func RPCInvalidParamsError(id string) RPCResponse { - return NewRPCErrorResponse(id, -32602, "Invalid params") +func RPCInvalidParamsError(id string, err error) RPCResponse { + return NewRPCErrorResponse(id, -32602, "Invalid params", err.Error()) } -func RPCInternalError(id string) RPCResponse { - return NewRPCErrorResponse(id, -32603, "Internal error") +func RPCInternalError(id string, err error) RPCResponse { + return NewRPCErrorResponse(id, -32603, "Internal error", err.Error()) } -func RPCServerError(id string) RPCResponse { - return NewRPCErrorResponse(id, -32000, "Server error") +func RPCServerError(id string, err error) RPCResponse { + return NewRPCErrorResponse(id, -32000, "Server error", err.Error()) } //---------------------------------------- @@ -133,7 +140,7 @@ type WSRPCContext struct { } //---------------------------------------- -// sockets +// SOCKETS // // Determine if its a unix or tcp socket. // If tcp, must specify the port; `0.0.0.0` will return incorrectly as "unix" since there's no port diff --git a/rpc/lib/types/types_test.go b/rpc/lib/types/types_test.go index 14688ecee..bab42124a 100644 --- a/rpc/lib/types/types_test.go +++ b/rpc/lib/types/types_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "testing" + "github.com/pkg/errors" "github.com/stretchr/testify/assert" ) @@ -19,14 +20,13 @@ func TestResponses(t *testing.T) { s := `{"jsonrpc":"2.0","id":"1","result":{"Value":"hello"}}` assert.Equal(string(s), string(b)) - d := RPCParseError("1") + d := RPCParseError("1", errors.New("Hello world")) e, _ := json.Marshal(d) - f := `{"jsonrpc":"2.0","id":"1","error":{"code":-32700,"message":"Parse error. Invalid JSON"}}` + f := `{"jsonrpc":"2.0","id":"1","error":{"code":-32700,"message":"Parse error. Invalid JSON","data":"Hello world"}}` assert.Equal(string(f), string(e)) g := RPCMethodNotFoundError("2") h, _ := json.Marshal(g) i := `{"jsonrpc":"2.0","id":"2","error":{"code":-32601,"message":"Method not found"}}` assert.Equal(string(h), string(i)) - } From e1fd587ddd5589678376ed242cb5867162fa2814 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 29 May 2017 14:48:18 +0300 Subject: [PATCH 41/92] we now omit error if empty --- test/app/counter_test.sh | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/test/app/counter_test.sh b/test/app/counter_test.sh index cc5c38b25..0198f85cd 100644 --- a/test/app/counter_test.sh +++ b/test/app/counter_test.sh @@ -23,39 +23,42 @@ function getCode() { else # this wont actually work if theres an error ... echo "$R" | jq .code - fi + fi } function sendTx() { TX=$1 if [[ "$GRPC_BROADCAST_TX" == "" ]]; then - RESPONSE=`curl -s localhost:46657/broadcast_tx_commit?tx=0x$TX` - ERROR=`echo $RESPONSE | jq .error` + RESPONSE=$(curl -s localhost:46657/broadcast_tx_commit?tx=0x"$TX") + IS_ERR=$(echo "$RESPONSE" | jq 'has("error")') + ERROR=$(echo "$RESPONSE" | jq '.error') ERROR=$(echo "$ERROR" | tr -d '"') # remove surrounding quotes - RESPONSE=`echo $RESPONSE | jq .result` + RESPONSE=$(echo "$RESPONSE" | jq '.result') else if [ -f grpc_client ]; then rm grpc_client fi echo "... building grpc_client" - go build -o grpc_client grpc_client.go - RESPONSE=`./grpc_client $TX` + go build -o grpc_client grpc_client.go + RESPONSE=$(./grpc_client "$TX") + IS_ERR=false ERROR="" fi echo "RESPONSE" - echo $RESPONSE + echo "$RESPONSE" - echo $RESPONSE | jq . &> /dev/null + echo "$RESPONSE" | jq . &> /dev/null IS_JSON=$? if [[ "$IS_JSON" != "0" ]]; then + IS_ERR=true ERROR="$RESPONSE" fi - APPEND_TX_RESPONSE=`echo $RESPONSE | jq .deliver_tx` - APPEND_TX_CODE=`getCode "$APPEND_TX_RESPONSE"` - CHECK_TX_RESPONSE=`echo $RESPONSE | jq .check_tx` - CHECK_TX_CODE=`getCode "$CHECK_TX_RESPONSE"` + APPEND_TX_RESPONSE=$(echo "$RESPONSE" | jq '.deliver_tx') + APPEND_TX_CODE=$(getCode "$APPEND_TX_RESPONSE") + CHECK_TX_RESPONSE=$(echo "$RESPONSE" | jq '.check_tx') + CHECK_TX_CODE=$(getCode "$CHECK_TX_RESPONSE") echo "-------" echo "TX $TX" @@ -63,7 +66,7 @@ function sendTx() { echo "ERROR $ERROR" echo "----" - if [[ "$ERROR" != "" ]]; then + if $IS_ERR; then echo "Unexpected error sending tx ($TX): $ERROR" exit 1 fi From b700ed8e3104ed9cefcadbb654901fe73f2868b4 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 18 Jul 2017 13:01:34 +0300 Subject: [PATCH 42/92] remove check for non-empty message as it should always be present --- rpc/lib/client/http_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpc/lib/client/http_client.go b/rpc/lib/client/http_client.go index 734ac5b17..b64aea30a 100644 --- a/rpc/lib/client/http_client.go +++ b/rpc/lib/client/http_client.go @@ -146,7 +146,7 @@ func unmarshalResponseBytes(responseBytes []byte, result interface{}) (interface if err != nil { return nil, errors.Errorf("Error unmarshalling rpc response: %v", err) } - if response.Error != nil && response.Error.Message != "" { + if response.Error != nil { return nil, errors.Errorf("Response error: %v", response.Error.Message) } // unmarshal the RawMessage into the result From 2252071866fd0cb5995926c947c251a5a1f83403 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 18 Jul 2017 13:06:37 +0300 Subject: [PATCH 43/92] update changelog --- CHANGELOG.md | 55 ++++++++++++++++++++++++++-------------------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d76fabfca..71de7925c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,34 +10,35 @@ BREAKING CHANGES: FEATURES: - Peer reputation management -- Use the chain as its own CA for nodes and validators -- Tooling to run multiple blockchains/apps, possibly in a single process -- State syncing (without transaction replay) -- Improved support for querying history and state +- Use the chain as its own CA for nodes and validators +- Tooling to run multiple blockchains/apps, possibly in a single process +- State syncing (without transaction replay) +- Improved support for querying history and state - Add authentication and rate-limitting to the RPC IMPROVEMENTS: -- Improve subtleties around mempool caching and logic -- Consensus optimizations: +- Improve subtleties around mempool caching and logic +- Consensus optimizations: - cache block parts for faster agreement after round changes - propagate block parts rarest first -- Better testing of the consensus state machine (ie. use a DSL) +- Better testing of the consensus state machine (ie. use a DSL) - Auto compiled serialization/deserialization code instead of go-wire reflection -BUG FIXES: -- Graceful handling/recovery for apps that have non-determinism or fail to halt +BUG FIXES: +- Graceful handling/recovery for apps that have non-determinism or fail to halt - Graceful handling/recovery for violations of safety, or liveness ## 0.11.0 (Date) IMPROVEMENTS: - docs: Added documentation from the tools repo to Read The Docs pipeline + - rpc: updated json response to match http://www.jsonrpc.org/specification spec ## 0.10.4 (September 5, 2017) IMPROVEMENTS: -- docs: Added Slate docs to each rpc function (see rpc/core) -- docs: Ported all website docs to Read The Docs +- docs: Added Slate docs to each rpc function (see rpc/core) +- docs: Ported all website docs to Read The Docs - config: expose some p2p params to tweak performance: RecvRate, SendRate, and MaxMsgPacketPayloadSize - rpc: Upgrade the websocket client and server, including improved auto reconnect, and proper ping/pong @@ -77,7 +78,7 @@ IMPROVEMENTS: FEATURES: - Use `--trace` to get stack traces for logged errors -- types: GenesisDoc.ValidatorHash returns the hash of the genesis validator set +- types: GenesisDoc.ValidatorHash returns the hash of the genesis validator set - types: GenesisDocFromFile parses a GenesiDoc from a JSON file IMPROVEMENTS: @@ -101,7 +102,7 @@ Also includes the Grand Repo-Merge of 2017. BREAKING CHANGES: - Config and Flags: - - The `config` map is replaced with a [`Config` struct](https://github.com/tendermint/tendermint/blob/master/config/config.go#L11), + - The `config` map is replaced with a [`Config` struct](https://github.com/tendermint/tendermint/blob/master/config/config.go#L11), containing substructs: `BaseConfig`, `P2PConfig`, `MempoolConfig`, `ConsensusConfig`, `RPCConfig` - This affects the following flags: - `--seeds` is now `--p2p.seeds` @@ -114,16 +115,16 @@ containing substructs: `BaseConfig`, `P2PConfig`, `MempoolConfig`, `ConsensusCon ``` [p2p] laddr="tcp://1.2.3.4:46656" - + [consensus] timeout_propose=1000 ``` - Use viper and `DefaultConfig() / TestConfig()` functions to handle defaults, and remove `config/tendermint` and `config/tendermint_test` - - Change some function and method signatures to + - Change some function and method signatures to - Change some [function and method signatures](https://gist.github.com/ebuchman/640d5fc6c2605f73497992fe107ebe0b) accomodate new config - Logger - - Replace static `log15` logger with a simple interface, and provide a new implementation using `go-kit`. + - Replace static `log15` logger with a simple interface, and provide a new implementation using `go-kit`. See our new [logging library](https://github.com/tendermint/tmlibs/log) and [blog post](https://tendermint.com/blog/abstracting-the-logger-interface-in-go) for more details - Levels `warn` and `notice` are removed (you may need to change them in your `config.toml`!) - Change some [function and method signatures](https://gist.github.com/ebuchman/640d5fc6c2605f73497992fe107ebe0b) to accept a logger @@ -166,7 +167,7 @@ IMPROVEMENTS: - Limit `/blockchain_info` call to return a maximum of 20 blocks - Use `.Wrap()` and `.Unwrap()` instead of eg. `PubKeyS` for `go-crypto` types - RPC JSON responses use pretty printing (via `json.MarshalIndent`) -- Color code different instances of the consensus for tests +- Color code different instances of the consensus for tests - Isolate viper to `cmd/tendermint/commands` and do not read config from file for tests @@ -194,7 +195,7 @@ IMPROVEMENTS: - WAL uses #ENDHEIGHT instead of #HEIGHT (#HEIGHT will stop working in 0.10.0) - Peers included via `--seeds`, under `seeds` in the config, or in `/dial_seeds` are now persistent, and will be reconnected to if the connection breaks -BUG FIXES: +BUG FIXES: - Fix bug in fast-sync where we stop syncing after a peer is removed, even if they're re-added later - Fix handshake replay to handle validator set changes and results of DeliverTx when we crash after app.Commit but before state.Save() @@ -210,7 +211,7 @@ message RequestQuery{ bytes data = 1; string path = 2; uint64 height = 3; - bool prove = 4; + bool prove = 4; } message ResponseQuery{ @@ -234,7 +235,7 @@ type BlockMeta struct { } ``` -- `ValidatorSet.Proposer` is exposed as a field and persisted with the `State`. Use `GetProposer()` to initialize or update after validator-set changes. +- `ValidatorSet.Proposer` is exposed as a field and persisted with the `State`. Use `GetProposer()` to initialize or update after validator-set changes. - `tendermint gen_validator` command output is now pure JSON @@ -277,7 +278,7 @@ type BlockID struct { } ``` -- `Vote` data type now includes validator address and index: +- `Vote` data type now includes validator address and index: ``` type Vote struct { @@ -297,7 +298,7 @@ type Vote struct { FEATURES: -- New message type on the ConsensusReactor, `Maj23Msg`, for peers to alert others they've seen a Maj23, +- New message type on the ConsensusReactor, `Maj23Msg`, for peers to alert others they've seen a Maj23, in order to track and handle conflicting votes intelligently to prevent Byzantine faults from causing halts: ``` @@ -320,7 +321,7 @@ IMPROVEMENTS: - Less verbose logging - Better test coverage (37% -> 49%) - Canonical SignBytes for signable types -- Write-Ahead Log for Mempool and Consensus via tmlibs/autofile +- Write-Ahead Log for Mempool and Consensus via tmlibs/autofile - Better in-process testing for the consensus reactor and byzantine faults - Better crash/restart testing for individual nodes at preset failure points, and of networks at arbitrary points - Better abstraction over timeout mechanics @@ -400,7 +401,7 @@ FEATURES: - TMSP and RPC support TCP and UNIX sockets - Addition config options including block size and consensus parameters - New WAL mode `cswal_light`; logs only the validator's own votes -- New RPC endpoints: +- New RPC endpoints: - for starting/stopping profilers, and for updating config - `/broadcast_tx_commit`, returns when tx is included in a block, else an error - `/unsafe_flush_mempool`, empties the mempool @@ -421,14 +422,14 @@ BUG FIXES: Strict versioning only began with the release of v0.7.0, in late summer 2016. The project itself began in early summer 2014 and was workable decentralized cryptocurrency software by the end of that year. -Through the course of 2015, in collaboration with Eris Industries (now Monax Indsutries), +Through the course of 2015, in collaboration with Eris Industries (now Monax Indsutries), many additional features were integrated, including an implementation from scratch of the Ethereum Virtual Machine. That implementation now forms the heart of [Burrow](https://github.com/hyperledger/burrow). In the later half of 2015, the consensus algorithm was upgraded with a more asynchronous design and a more deterministic and robust implementation. -By late 2015, frustration with the difficulty of forking a large monolithic stack to create alternative cryptocurrency designs led to the +By late 2015, frustration with the difficulty of forking a large monolithic stack to create alternative cryptocurrency designs led to the invention of the Application Blockchain Interface (ABCI), then called the Tendermint Socket Protocol (TMSP). The Ethereum Virtual Machine and various other transaction features were removed, and Tendermint was whittled down to a core consensus engine -driving an application running in another process. +driving an application running in another process. The ABCI interface and implementation were iterated on and improved over the course of 2016, until versioned history kicked in with v0.7.0. From e36c79f7137769af5ce500ab164d94ddc347d838 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 18 Jul 2017 13:08:51 +0300 Subject: [PATCH 44/92] capitalize RpcError --- rpc/lib/client/ws_client.go | 4 ++-- rpc/lib/types/types.go | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/rpc/lib/client/ws_client.go b/rpc/lib/client/ws_client.go index f56c45e99..7bf3fc2a9 100644 --- a/rpc/lib/client/ws_client.go +++ b/rpc/lib/client/ws_client.go @@ -422,8 +422,8 @@ func (c *WSClient) readRoutine() { c.ErrorsCh <- err continue } - if response.Error != "" { - c.ErrorsCh <- errors.Errorf(response.Error) + if response.Error != nil { + c.ErrorsCh <- errors.New(response.Error.Message) continue } c.Logger.Info("got response", "resp", response.Result) diff --git a/rpc/lib/types/types.go b/rpc/lib/types/types.go index b649d1b94..8d71d6f21 100644 --- a/rpc/lib/types/types.go +++ b/rpc/lib/types/types.go @@ -54,7 +54,7 @@ func ArrayToRequest(id string, method string, params []interface{}) (RPCRequest, //---------------------------------------- // RESPONSE -type RpcError struct { +type RPCError struct { Code int `json:"code"` Message string `json:"message"` Data string `json:"data,omitempty"` @@ -64,7 +64,7 @@ type RPCResponse struct { JSONRPC string `json:"jsonrpc"` ID string `json:"id,omitempty"` Result *json.RawMessage `json:"result,omitempty"` - Error *RpcError `json:"error,omitempty"` + Error *RPCError `json:"error,omitempty"` } func NewRPCSuccessResponse(id string, res interface{}) RPCResponse { @@ -87,12 +87,12 @@ func NewRPCErrorResponse(id string, code int, msg string, data string) RPCRespon return RPCResponse{ JSONRPC: "2.0", ID: id, - Error: &RpcError{Code: code, Message: msg, Data: data}, + Error: &RPCError{Code: code, Message: msg, Data: data}, } } func (resp RPCResponse) String() string { - if resp.Error == "" { + if resp.Error == nil { return fmt.Sprintf("[%s %v]", resp.ID, resp.Result) } else { return fmt.Sprintf("[%s %s]", resp.ID, resp.Error) From 7fadde0b373a82b7698f455fb8843c00b31295d7 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 18 Sep 2017 12:02:15 -0700 Subject: [PATCH 45/92] check for request ID after receiving it --- rpc/lib/client/http_client.go | 2 +- rpc/lib/client/ws_client.go | 4 ++-- rpc/lib/server/handlers.go | 11 +++++++++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/rpc/lib/client/http_client.go b/rpc/lib/client/http_client.go index b64aea30a..1fbaedfae 100644 --- a/rpc/lib/client/http_client.go +++ b/rpc/lib/client/http_client.go @@ -75,7 +75,7 @@ func NewJSONRPCClient(remote string) *JSONRPCClient { } func (c *JSONRPCClient) Call(method string, params map[string]interface{}, result interface{}) (interface{}, error) { - request, err := types.MapToRequest("", method, params) + request, err := types.MapToRequest("jsonrpc-client", method, params) if err != nil { return nil, err } diff --git a/rpc/lib/client/ws_client.go b/rpc/lib/client/ws_client.go index 7bf3fc2a9..1407073ae 100644 --- a/rpc/lib/client/ws_client.go +++ b/rpc/lib/client/ws_client.go @@ -195,7 +195,7 @@ func (c *WSClient) Send(ctx context.Context, request types.RPCRequest) error { // Call the given method. See Send description. func (c *WSClient) Call(ctx context.Context, method string, params map[string]interface{}) error { - request, err := types.MapToRequest("", method, params) + request, err := types.MapToRequest("ws-client", method, params) if err != nil { return err } @@ -205,7 +205,7 @@ func (c *WSClient) Call(ctx context.Context, method string, params map[string]in // CallWithArrayParams the given method with params in a form of array. See // Send description. func (c *WSClient) CallWithArrayParams(ctx context.Context, method string, params []interface{}) error { - request, err := types.ArrayToRequest("", method, params) + request, err := types.ArrayToRequest("ws-client", method, params) if err != nil { return err } diff --git a/rpc/lib/server/handlers.go b/rpc/lib/server/handlers.go index 070072d8f..5cb3baddd 100644 --- a/rpc/lib/server/handlers.go +++ b/rpc/lib/server/handlers.go @@ -113,6 +113,11 @@ func makeJSONRPCHandler(funcMap map[string]*RPCFunc, logger log.Logger) http.Han WriteRPCResponseHTTP(w, types.RPCParseError("", errors.Wrap(err, "Error unmarshalling request"))) return } + // A Notification is a Request object without an "id" member. + // The Server MUST NOT reply to a Notification, including those that are within a batch request. + if request.ID == "" { + return + } if len(r.URL.Path) > 1 { WriteRPCResponseHTTP(w, types.RPCInvalidRequestError(request.ID, errors.Errorf("Path %s is invalid", r.URL.Path))) return @@ -513,6 +518,12 @@ func (wsc *wsConnection) readRoutine() { continue } + // A Notification is a Request object without an "id" member. + // The Server MUST NOT reply to a Notification, including those that are within a batch request. + if request.ID == "" { + continue + } + // Now, fetch the RPCFunc and execute it. rpcFunc := wsc.funcMap[request.Method] From 95875c55fc7cf94a5e1213f1ed720f3b2ca4f3b5 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 18 Sep 2017 12:04:05 -0700 Subject: [PATCH 46/92] ID must be present in both request and response from the spec: This member is REQUIRED. It MUST be the same as the value of the id member in the Request Object. If there was an error in detecting the id in the Request object (e.g. Parse error/Invalid Request), it MUST be Null. --- rpc/lib/types/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpc/lib/types/types.go b/rpc/lib/types/types.go index 8d71d6f21..0267c529c 100644 --- a/rpc/lib/types/types.go +++ b/rpc/lib/types/types.go @@ -62,7 +62,7 @@ type RPCError struct { type RPCResponse struct { JSONRPC string `json:"jsonrpc"` - ID string `json:"id,omitempty"` + ID string `json:"id"` Result *json.RawMessage `json:"result,omitempty"` Error *RPCError `json:"error,omitempty"` } From f8b152972f32b7833e4ef6b2479aa57db8ccb938 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 18 Sep 2017 12:26:26 -0700 Subject: [PATCH 47/92] return method not found error if somebody tries to access WS method in non-ws context --- rpc/lib/server/handlers.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/rpc/lib/server/handlers.go b/rpc/lib/server/handlers.go index 5cb3baddd..0fafd2a84 100644 --- a/rpc/lib/server/handlers.go +++ b/rpc/lib/server/handlers.go @@ -123,14 +123,10 @@ func makeJSONRPCHandler(funcMap map[string]*RPCFunc, logger log.Logger) http.Han return } rpcFunc := funcMap[request.Method] - if rpcFunc == nil { + if rpcFunc == nil || rpcFunc.ws { WriteRPCResponseHTTP(w, types.RPCMethodNotFoundError(request.ID)) return } - if rpcFunc.ws { - WriteRPCResponseHTTP(w, types.RPCInternalError(request.ID, errors.New("Trying to use Websocket method in non-ws context"))) - return - } args, err := jsonParamsToArgsRPC(rpcFunc, request.Params) if err != nil { WriteRPCResponseHTTP(w, types.RPCInvalidParamsError(request.ID, errors.Wrap(err, "Error converting json params to arguments"))) @@ -234,7 +230,7 @@ func makeHTTPHandler(rpcFunc *RPCFunc, logger log.Logger) func(http.ResponseWrit // Exception for websocket endpoints if rpcFunc.ws { return func(w http.ResponseWriter, r *http.Request) { - WriteRPCResponseHTTP(w, types.RPCInternalError("", errors.New("Trying to use Websocket method in non-ws context"))) + WriteRPCResponseHTTP(w, types.RPCMethodNotFoundError("")) } } // All other endpoints From cf9a03f698dd1454d78936de2ea46dcb4afa1780 Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Mon, 18 Sep 2017 16:42:24 -0400 Subject: [PATCH 48/92] docs: organize install a bit better --- docs/install.rst | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index b353aba87..1746daa9d 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -1,17 +1,24 @@ -Install from Source -=================== +Install Tendermint +================== + +From Binary +----------- + +To download pre-built binaries, see the `Download page `__. -This page provides instructions on installing Tendermint from source. To -download pre-built binaries, see the `Download page `__. +From Source +----------- + +You'll need `go`, maybe `glide` and the tendermint source code. Install Go ----------- +^^^^^^^^^^ Make sure you have `installed Go `__ and set the ``GOPATH``. -Install Tendermint ------------------- +Get Source Code +^^^^^^^^^^^^^^^ You should be able to install the latest with a simple @@ -19,13 +26,14 @@ You should be able to install the latest with a simple go get github.com/tendermint/tendermint/cmd/tendermint -Run ``tendermint --help`` for more. +Run ``tendermint --help`` and ``tendermint version`` to ensure your +installation worked. If the installation failed, a dependency may been updated and become incompatible with the latest Tendermint master branch. We solve this using the ``glide`` tool for dependency management. -Fist, install ``glide``: +First, install ``glide``: :: @@ -45,7 +53,7 @@ still cloned to the correct location in the ``$GOPATH``. The latest Tendermint Core version is now installed. Reinstall -~~~~~~~~~ +--------- If you already have Tendermint installed, and you make updates, simply @@ -79,7 +87,7 @@ Since the third option just uses ``glide`` right away, it should always work. Troubleshooting -~~~~~~~~~~~~~~~ +--------------- If ``go get`` failing bothers you, fetch the code using ``git``: @@ -92,7 +100,7 @@ If ``go get`` failing bothers you, fetch the code using ``git``: go install ./cmd/tendermint Run -~~~ +^^^ To start a one-node blockchain with a simple in-process application: From a3d925ac1d8ac177510bfb40dd439c3048b170af Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Mon, 18 Sep 2017 16:52:00 -0400 Subject: [PATCH 49/92] pr fixes --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 7238e51bd..420057976 100644 --- a/README.md +++ b/README.md @@ -13,17 +13,17 @@ https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/6874 [![](https://tokei.rs/b1/github/tendermint/tendermint?category=lines)](https://github.com/tendermint/tendermint) -Branch | Tests | -----------|-------| -master | [![CircleCI](https://circleci.com/gh/tendermint/tendermint/tree/master.svg?style=shield)](https://circleci.com/gh/tendermint/tendermint/tree/master) -develop | [![CircleCI](https://circleci.com/gh/tendermint/tendermint/tree/develop.svg?style=shield)](https://circleci.com/gh/tendermint/tendermint/tree/develop) +Branch | Tests | Coverage +----------|-------|---------- +master | [![CircleCI](https://circleci.com/gh/tendermint/tendermint/tree/master.svg?style=shield)](https://circleci.com/gh/tendermint/tendermint/tree/master) | [![codecov](https://codecov.io/gh/tendermint/tendermint/branch/master/graph/badge.svg)](https://codecov.io/gh/tendermint/tendermint) +develop | [![CircleCI](https://circleci.com/gh/tendermint/tendermint/tree/develop.svg?style=shield)](https://circleci.com/gh/tendermint/tendermint/tree/develop) | [![codecov](https://codecov.io/gh/tendermint/tendermint/branch/develop/graph/badge.svg)](https://codecov.io/gh/tendermint/tendermint) _NOTE: This is alpha software. Please contact us if you intend to run it in production._ Tendermint Core is Byzantine Fault Tolerant (BFT) middleware that takes a state transition machine - written in any programming language - and securely replicates it on many machines. -For more information, from introduction to install to application development, [Read The Docs](http://tendermint.readthedocs.io/projects/tools/en/master/index.html). +For more information, from introduction to install to application development, [Read The Docs](http://tendermint.readthedocs.io/projects/tools/en/master). ## Install @@ -39,7 +39,7 @@ For more details (or if it fails), [read the docs](http://tendermint.readthedocs ### Tendermint Core -All resources involving the use of, building application on, or developing for, tendermint, can be found at [Read The Docs](http://tendermint.readthedocs.io/projects/tools/en/master/index.html). Additional information about some - and eventually all - of the sub-projects below, can be found at Read The Docs. +All resources involving the use of, building application on, or developing for, tendermint, can be found at [Read The Docs](http://tendermint.readthedocs.io/projects/tools/en/master). Additional information about some - and eventually all - of the sub-projects below, can be found at Read The Docs. ### Sub-projects @@ -99,6 +99,6 @@ data into the new chain. However, any bump in the PATCH version should be compatible with existing histories (if not please open an [issue](https://github.com/tendermint/tendermint/issues)). -### Code of Conduct +## Code of Conduct Please read, understand and adhere to our [code of conduct](CODE_OF_CONDUCT.md). From 70b95135e6681d0a19b98acf426df48b54d91e8e Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Mon, 18 Sep 2017 17:49:43 -0400 Subject: [PATCH 50/92] consensus: use filepath for windows compatibility, closes #595 --- consensus/state.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/consensus/state.go b/consensus/state.go index f6ee5d45e..2523aac60 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -4,7 +4,7 @@ import ( "bytes" "errors" "fmt" - "path" + "path/filepath" "reflect" "sync" "time" @@ -394,7 +394,7 @@ func (cs *ConsensusState) Wait() { // OpenWAL opens a file to log all consensus messages and timeouts for deterministic accountability func (cs *ConsensusState) OpenWAL(walFile string) (err error) { - err = cmn.EnsureDir(path.Dir(walFile), 0700) + err = cmn.EnsureDir(filepath.Dir(walFile), 0700) if err != nil { cs.Logger.Error("Error ensuring ConsensusState wal dir", "err", err.Error()) return err From b0017c54600fec858801e4c49599c69eab7a513f Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 21 Sep 2017 12:34:37 -0400 Subject: [PATCH 51/92] update CHANGELOG [ci skip] --- CHANGELOG.md | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71de7925c..9aa179de6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## Roadmap BREAKING CHANGES: -- Upgrade the header to support better proves on validtors, results, evidence, and possibly more +- Upgrade the header to support better proofs on validtors, results, evidence, and possibly more - Better support for injecting randomness - Pass evidence/voteInfo through ABCI - Upgrade consensus for more real-time use of evidence @@ -30,9 +30,24 @@ BUG FIXES: ## 0.11.0 (Date) +BREAKING: + - state: every validator set change is persisted to disk, which required some changes to the `State` structure. + - cmd: if there is no genesis, exit immediately instead of waiting around for one to show. + - p2p: new `p2p.Peer` interface used for all reactor methods (instead of `*p2p.Peer` struct). + - types: `Signer.Sign` returns an error. + - rpc: various changes to match JSONRPC spec (http://www.jsonrpc.org/specification), including breaking ones: + - requests that previously returned HTTP code 4XX now return 200 with an error code in the JSONRPC. + - `rpctypes.RPCResponse` uses new `RPCError` type instead of `string`. + +FEATURES: + - rpc: `/validators?height=X` allows querying of validators at previous heights. + - rpc: Leaving the `height` param empty for `/block`, `/validators`, and `/commit` will return the value for the latest height. + IMPROVEMENTS: - - docs: Added documentation from the tools repo to Read The Docs pipeline - - rpc: updated json response to match http://www.jsonrpc.org/specification spec + - docs: Moved all docs from the website and tools repo in, converted to `.rst`, and cleaned up for presentation on `tendermint.readthedocs.io` + +BUG FIXES: + - fix WAL openning issue on Windows ## 0.10.4 (September 5, 2017) From 29bfcb0a31da9647c5ff057cf25850a23a0d45d6 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 11 Sep 2017 15:45:12 -0400 Subject: [PATCH 52/92] minor comments/changes --- blockchain/pool.go | 2 +- blockchain/reactor.go | 2 +- types/genesis.go | 5 ----- types/priv_validator.go | 1 + 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/blockchain/pool.go b/blockchain/pool.go index e1288c9fa..924880c08 100644 --- a/blockchain/pool.go +++ b/blockchain/pool.go @@ -352,7 +352,7 @@ func (peer *bpPeer) setLogger(l log.Logger) { func (peer *bpPeer) resetMonitor() { peer.recvMonitor = flow.New(time.Second, time.Second*40) - var initialValue = float64(minRecvRate) * math.E + initialValue := float64(minRecvRate) * math.E peer.recvMonitor.SetREMA(initialValue) } diff --git a/blockchain/reactor.go b/blockchain/reactor.go index efa6e2f05..e4b63ca2a 100644 --- a/blockchain/reactor.go +++ b/blockchain/reactor.go @@ -28,7 +28,7 @@ const ( statusUpdateIntervalSeconds = 10 // check if we should switch to consensus reactor switchToConsensusIntervalSeconds = 1 - maxBlockchainResponseSize = types.MaxBlockSize + 2 + maxBlockchainResponseSize = types.MaxBlockSize + 2 // TODO ) type consensusReactor interface { diff --git a/types/genesis.go b/types/genesis.go index 23b14c9e9..17078b6fe 100644 --- a/types/genesis.go +++ b/types/genesis.go @@ -12,11 +12,6 @@ import ( cmn "github.com/tendermint/tmlibs/common" ) -//------------------------------------------------------------ -// we store the gendoc in the db - -var GenDocKey = []byte("GenDocKey") - //------------------------------------------------------------ // core types for a genesis definition diff --git a/types/priv_validator.go b/types/priv_validator.go index b96e988db..3e84e7f36 100644 --- a/types/priv_validator.go +++ b/types/priv_validator.go @@ -15,6 +15,7 @@ import ( "github.com/tendermint/tmlibs/log" ) +// TODO: type ? const ( stepNone = 0 // Used to distinguish the initial state stepPropose = 1 From 1f3e4d2d9ac42c5da8ec18bebb0284e88fceaf62 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 11 Sep 2017 16:28:00 -0400 Subject: [PATCH 53/92] move PartSetSize out of the config, into ConsensusParams --- blockchain/reactor.go | 15 ++++++++++----- config/config.go | 7 ------- config/consensus.go | 27 +++++++++++++++++++++++++++ consensus/replay_test.go | 17 ++++++++--------- consensus/state.go | 6 ++++-- consensus/state_test.go | 14 +++++++------- types/block.go | 5 ----- types/genesis.go | 29 +++++++++++++++++++++-------- 8 files changed, 77 insertions(+), 43 deletions(-) create mode 100644 config/consensus.go diff --git a/blockchain/reactor.go b/blockchain/reactor.go index e4b63ca2a..fa3c70799 100644 --- a/blockchain/reactor.go +++ b/blockchain/reactor.go @@ -28,7 +28,6 @@ const ( statusUpdateIntervalSeconds = 10 // check if we should switch to consensus reactor switchToConsensusIntervalSeconds = 1 - maxBlockchainResponseSize = types.MaxBlockSize + 2 // TODO ) type consensusReactor interface { @@ -124,7 +123,7 @@ func (bcR *BlockchainReactor) RemovePeer(peer p2p.Peer, reason interface{}) { // Receive implements Reactor by handling 4 types of messages (look below). func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) { - _, msg, err := DecodeMessage(msgBytes) + _, msg, err := DecodeMessage(msgBytes, bcR.maxMsgSize()) if err != nil { bcR.Logger.Error("Error decoding message", "err", err) return @@ -163,6 +162,12 @@ func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) } } +// maxMsgSize returns the maximum allowable size of a +// message on the blockchain reactor. +func (bcR *BlockchainReactor) maxMsgSize() int { + return bcR.state.GenesisDoc.ConsensusParams.MaxBlockSizeBytes + 2 +} + // Handle messages from the poolReactor telling the reactor what to do. // NOTE: Don't sleep in the FOR_LOOP or otherwise slow it down! // (Except for the SYNC_LOOP, which is the primary purpose and must be synchronous.) @@ -221,7 +226,7 @@ FOR_LOOP: // We need both to sync the first block. break SYNC_LOOP } - firstParts := first.MakePartSet(types.DefaultBlockPartSize) + firstParts := first.MakePartSet(bcR.state.GenesisDoc.ConsensusParams.BlockPartSizeBytes) firstPartsHeader := firstParts.Header() // Finally, verify the first block using the second's commit // NOTE: we can probably make this more efficient, but note that calling @@ -290,11 +295,11 @@ var _ = wire.RegisterInterface( // DecodeMessage decodes BlockchainMessage. // TODO: ensure that bz is completely read. -func DecodeMessage(bz []byte) (msgType byte, msg BlockchainMessage, err error) { +func DecodeMessage(bz []byte, maxSize int) (msgType byte, msg BlockchainMessage, err error) { msgType = bz[0] n := int(0) r := bytes.NewReader(bz) - msg = wire.ReadBinary(struct{ BlockchainMessage }{}, r, maxBlockchainResponseSize, &n, &err).(struct{ BlockchainMessage }).BlockchainMessage + msg = wire.ReadBinary(struct{ BlockchainMessage }{}, r, maxSize, &n, &err).(struct{ BlockchainMessage }).BlockchainMessage if err != nil && n != len(bz) { err = errors.New("DecodeMessage() had bytes left over") } diff --git a/config/config.go b/config/config.go index ec1f85edb..23da4f405 100644 --- a/config/config.go +++ b/config/config.go @@ -4,8 +4,6 @@ import ( "fmt" "path/filepath" "time" - - "github.com/tendermint/tendermint/types" // TODO: remove ) // Config defines the top level configuration for a Tendermint node @@ -320,10 +318,6 @@ type ConsensusConfig struct { CreateEmptyBlocks bool `mapstructure:"create_empty_blocks"` CreateEmptyBlocksInterval int `mapstructure:"create_empty_blocks_interval"` - // TODO: This probably shouldn't be exposed but it makes it - // easy to write tests for the wal/replay - BlockPartSize int `mapstructure:"block_part_size"` - // Reactor sleep duration parameters are in ms PeerGossipSleepDuration int `mapstructure:"peer_gossip_sleep_duration"` PeerQueryMaj23SleepDuration int `mapstructure:"peer_query_maj23_sleep_duration"` @@ -386,7 +380,6 @@ func DefaultConsensusConfig() *ConsensusConfig { MaxBlockSizeBytes: 1, // TODO CreateEmptyBlocks: true, CreateEmptyBlocksInterval: 0, - BlockPartSize: types.DefaultBlockPartSize, // TODO: we shouldnt be importing types PeerGossipSleepDuration: 100, PeerQueryMaj23SleepDuration: 2000, } diff --git a/config/consensus.go b/config/consensus.go new file mode 100644 index 000000000..5b10911d8 --- /dev/null +++ b/config/consensus.go @@ -0,0 +1,27 @@ +package config + +import ( + "fmt" +) + +type ConsensusParams struct { + MaxBlockSizeBytes int `json:"max_block_size_bytes"` + BlockPartSizeBytes int `json:"block_part_size_bytes"` +} + +func DefaultConsensusParams() *ConsensusParams { + return &ConsensusParams{ + MaxBlockSizeBytes: 22020096, // 21MB + BlockPartSizeBytes: 65536, // 64kB, + } +} + +func (params *ConsensusParams) Validate() error { + if params.MaxBlockSizeBytes <= 0 { + return fmt.Errorf("MaxBlockSizeBytes must be greater than 0. Got %d", params.MaxBlockSizeBytes) + } + if params.BlockPartSizeBytes <= 0 { + return fmt.Errorf("BlockPartSizeBytes must be greater than 0. Got %d", params.BlockPartSizeBytes) + } + return nil +} diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 78cdaf7bc..179e8696c 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -267,8 +267,6 @@ func testReplayCrashBeforeWriteVote(t *testing.T, thisCase *testCase, lineNum in var ( NUM_BLOCKS = 6 // number of blocks in the test_data/many_blocks.cswal mempool = types.MockMempool{} - - testPartSize int ) //--------------------------------------- @@ -320,7 +318,6 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) { config.Consensus.SetWalFile(walFile) privVal := types.LoadPrivValidator(config.PrivValidatorFile()) - testPartSize = config.Consensus.BlockPartSize wal, err := NewWAL(walFile, false) if err != nil { @@ -384,6 +381,7 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) { } func applyBlock(st *sm.State, blk *types.Block, proxyApp proxy.AppConns) { + testPartSize := st.GenesisDoc.ConsensusParams.BlockPartSizeBytes err := st.ApplyBlock(nil, proxyApp.Consensus(), blk, blk.MakePartSet(testPartSize).Header(), mempool) if err != nil { panic(err) @@ -503,7 +501,7 @@ func makeBlockchainFromWAL(wal *WAL) ([]*types.Block, []*types.Commit, error) { // if its not the first one, we have a full block if blockParts != nil { var n int - block := wire.ReadBinary(&types.Block{}, blockParts.GetReader(), types.MaxBlockSize, &n, &err).(*types.Block) + block := wire.ReadBinary(&types.Block{}, blockParts.GetReader(), 0, &n, &err).(*types.Block) blocks = append(blocks, block) } blockParts = types.NewPartSetFromHeader(*p) @@ -524,7 +522,7 @@ func makeBlockchainFromWAL(wal *WAL) ([]*types.Block, []*types.Commit, error) { } // grab the last block too var n int - block := wire.ReadBinary(&types.Block{}, blockParts.GetReader(), types.MaxBlockSize, &n, &err).(*types.Block) + block := wire.ReadBinary(&types.Block{}, blockParts.GetReader(), 0, &n, &err).(*types.Block) blocks = append(blocks, block) return blocks, commits, nil } @@ -563,7 +561,7 @@ func stateAndStore(config *cfg.Config, pubKey crypto.PubKey) (*sm.State, *mockBl state := sm.MakeGenesisStateFromFile(stateDB, config.GenesisFile()) state.SetLogger(log.TestingLogger().With("module", "state")) - store := NewMockBlockStore(config) + store := NewMockBlockStore(config, state.GenesisDoc.ConsensusParams) return state, store } @@ -572,13 +570,14 @@ func stateAndStore(config *cfg.Config, pubKey crypto.PubKey) (*sm.State, *mockBl type mockBlockStore struct { config *cfg.Config + params *cfg.ConsensusParams chain []*types.Block commits []*types.Commit } // TODO: NewBlockStore(db.NewMemDB) ... -func NewMockBlockStore(config *cfg.Config) *mockBlockStore { - return &mockBlockStore{config, nil, nil} +func NewMockBlockStore(config *cfg.Config, params *cfg.ConsensusParams) *mockBlockStore { + return &mockBlockStore{config, params, nil, nil} } func (bs *mockBlockStore) Height() int { return len(bs.chain) } @@ -586,7 +585,7 @@ func (bs *mockBlockStore) LoadBlock(height int) *types.Block { return bs.chain[h func (bs *mockBlockStore) LoadBlockMeta(height int) *types.BlockMeta { block := bs.chain[height-1] return &types.BlockMeta{ - BlockID: types.BlockID{block.Hash(), block.MakePartSet(bs.config.Consensus.BlockPartSize).Header()}, + BlockID: types.BlockID{block.Hash(), block.MakePartSet(bs.params.BlockPartSizeBytes).Header()}, Header: block.Header, } } diff --git a/consensus/state.go b/consensus/state.go index 2523aac60..8a19a052d 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -983,7 +983,8 @@ func (cs *ConsensusState) createProposalBlock() (block *types.Block, blockParts txs := cs.mempool.Reap(cs.config.MaxBlockSizeTxs) return types.MakeBlock(cs.Height, cs.state.ChainID, txs, commit, - cs.state.LastBlockID, cs.state.Validators.Hash(), cs.state.AppHash, cs.config.BlockPartSize) + cs.state.LastBlockID, cs.state.Validators.Hash(), + cs.state.AppHash, cs.state.GenesisDoc.ConsensusParams.BlockPartSizeBytes) } // Enter: `timeoutPropose` after entering Propose. @@ -1417,7 +1418,8 @@ func (cs *ConsensusState) addProposalBlockPart(height int, part *types.Part, ver // Added and completed! var n int var err error - cs.ProposalBlock = wire.ReadBinary(&types.Block{}, cs.ProposalBlockParts.GetReader(), types.MaxBlockSize, &n, &err).(*types.Block) + cs.ProposalBlock = wire.ReadBinary(&types.Block{}, cs.ProposalBlockParts.GetReader(), + cs.state.GenesisDoc.ConsensusParams.MaxBlockSizeBytes, &n, &err).(*types.Block) // NOTE: it's possible to receive complete proposal blocks for future rounds without having the proposal cs.Logger.Info("Received complete proposal block", "height", cs.ProposalBlock.Height, "hash", cs.ProposalBlock.Hash()) if cs.Step == RoundStepPropose && cs.isProposalComplete() { diff --git a/consensus/state_test.go b/consensus/state_test.go index 81ef016be..6f804fcb4 100644 --- a/consensus/state_test.go +++ b/consensus/state_test.go @@ -180,7 +180,7 @@ func TestBadProposal(t *testing.T) { height, round := cs1.Height, cs1.Round vs2 := vss[1] - partSize := config.Consensus.BlockPartSize + partSize := cs1.state.GenesisDoc.ConsensusParams.BlockPartSizeBytes proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1) voteCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringVote(), 1) @@ -327,7 +327,7 @@ func TestLockNoPOL(t *testing.T) { vs2 := vss[1] height := cs1.Height - partSize := config.Consensus.BlockPartSize + partSize := cs1.state.GenesisDoc.ConsensusParams.BlockPartSizeBytes timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1) timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1) @@ -493,7 +493,7 @@ func TestLockPOLRelock(t *testing.T) { cs1, vss := randConsensusState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] - partSize := config.Consensus.BlockPartSize + partSize := cs1.state.GenesisDoc.ConsensusParams.BlockPartSizeBytes timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1) timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1) @@ -608,7 +608,7 @@ func TestLockPOLUnlock(t *testing.T) { cs1, vss := randConsensusState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] - partSize := config.Consensus.BlockPartSize + partSize := cs1.state.GenesisDoc.ConsensusParams.BlockPartSizeBytes proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1) timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1) @@ -703,7 +703,7 @@ func TestLockPOLSafety1(t *testing.T) { cs1, vss := randConsensusState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] - partSize := config.Consensus.BlockPartSize + partSize := cs1.state.GenesisDoc.ConsensusParams.BlockPartSizeBytes proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1) timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1) @@ -824,7 +824,7 @@ func TestLockPOLSafety2(t *testing.T) { cs1, vss := randConsensusState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] - partSize := config.Consensus.BlockPartSize + partSize := cs1.state.GenesisDoc.ConsensusParams.BlockPartSizeBytes proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1) timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1) @@ -999,7 +999,7 @@ func TestHalt1(t *testing.T) { cs1, vss := randConsensusState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] - partSize := config.Consensus.BlockPartSize + partSize := cs1.state.GenesisDoc.ConsensusParams.BlockPartSizeBytes proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1) timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1) diff --git a/types/block.go b/types/block.go index fee62e98c..c8cdf81a9 100644 --- a/types/block.go +++ b/types/block.go @@ -14,11 +14,6 @@ import ( "github.com/tendermint/tmlibs/merkle" ) -const ( - MaxBlockSize = 22020096 // 21MB TODO make it configurable - DefaultBlockPartSize = 65536 // 64kB TODO: put part size in parts header? -) - // Block defines the atomic unit of a Tendermint blockchain type Block struct { *Header `json:"header"` diff --git a/types/genesis.go b/types/genesis.go index 17078b6fe..b7762c576 100644 --- a/types/genesis.go +++ b/types/genesis.go @@ -10,6 +10,8 @@ import ( "github.com/tendermint/go-crypto" "github.com/tendermint/go-wire/data" cmn "github.com/tendermint/tmlibs/common" + + cfg "github.com/tendermint/tendermint/config" ) //------------------------------------------------------------ @@ -24,10 +26,11 @@ type GenesisValidator struct { // GenesisDoc defines the initial conditions for a tendermint blockchain, in particular its validator set. type GenesisDoc struct { - GenesisTime time.Time `json:"genesis_time"` - ChainID string `json:"chain_id"` - Validators []GenesisValidator `json:"validators"` - AppHash data.Bytes `json:"app_hash"` + GenesisTime time.Time `json:"genesis_time"` + ChainID string `json:"chain_id"` + ConsensusParams *cfg.ConsensusParams `json:"consensus_params"` + Validators []GenesisValidator `json:"validators"` + AppHash data.Bytes `json:"app_hash"` } // SaveAs is a utility method for saving GenensisDoc as a JSON file. @@ -56,6 +59,19 @@ func (genDoc *GenesisDoc) ValidatorHash() []byte { func GenesisDocFromJSON(jsonBlob []byte) (*GenesisDoc, error) { genDoc := GenesisDoc{} err := json.Unmarshal(jsonBlob, &genDoc) + + // validate genesis + if genDoc.ChainID == "" { + return nil, errors.Errorf("Genesis doc must include non-empty chain_id") + } + if genDoc.ConsensusParams == nil { + genDoc.ConsensusParams = cfg.DefaultConsensusParams() + } else { + if err := genDoc.ConsensusParams.Validate(); err != nil { + return nil, err + } + } + return &genDoc, err } @@ -67,10 +83,7 @@ func GenesisDocFromFile(genDocFile string) (*GenesisDoc, error) { } genDoc, err := GenesisDocFromJSON(jsonBlob) if err != nil { - return nil, errors.Wrap(err, "Error reading GenesisDoc") - } - if genDoc.ChainID == "" { - return nil, errors.Errorf("Genesis doc %v must include non-empty chain_id", genDocFile) + return nil, errors.Wrap(err, cmn.Fmt("Error reading GenesisDoc at %v", genDocFile)) } return genDoc, nil } From 14abdd57f3f8091d9e5855dc0c6e27f6973284d0 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 11 Sep 2017 16:48:41 -0400 Subject: [PATCH 54/92] genDoc.ValidateAndComplete --- state/state.go | 9 +++------ types/genesis.go | 42 ++++++++++++++++++++++++++++++++---------- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/state/state.go b/state/state.go index 7e22e04d1..4e0fa75b5 100644 --- a/state/state.go +++ b/state/state.go @@ -324,12 +324,9 @@ func MakeGenesisStateFromFile(db dbm.DB, genDocFile string) *State { // // Used in tests. func MakeGenesisState(db dbm.DB, genDoc *types.GenesisDoc) *State { - if len(genDoc.Validators) == 0 { - cmn.Exit(cmn.Fmt("The genesis file has no validators")) - } - - if genDoc.GenesisTime.IsZero() { - genDoc.GenesisTime = time.Now() + err := genDoc.ValidateAndComplete() + if err != nil { + cmn.Exit(cmn.Fmt("Error in genesis file: %v", err)) } // Make validators slice diff --git a/types/genesis.go b/types/genesis.go index b7762c576..79bf63c8b 100644 --- a/types/genesis.go +++ b/types/genesis.go @@ -52,26 +52,48 @@ func (genDoc *GenesisDoc) ValidatorHash() []byte { return vset.Hash() } -//------------------------------------------------------------ -// Make genesis state from file - -// GenesisDocFromJSON unmarshalls JSON data into a GenesisDoc. -func GenesisDocFromJSON(jsonBlob []byte) (*GenesisDoc, error) { - genDoc := GenesisDoc{} - err := json.Unmarshal(jsonBlob, &genDoc) +// ValidateAndComplete checks that all necessary fields are present +// and fills in defaults for optional fields left empty +func (genDoc *GenesisDoc) ValidateAndComplete() error { - // validate genesis if genDoc.ChainID == "" { - return nil, errors.Errorf("Genesis doc must include non-empty chain_id") + return errors.Errorf("Genesis doc must include non-empty chain_id") } + if genDoc.ConsensusParams == nil { genDoc.ConsensusParams = cfg.DefaultConsensusParams() } else { if err := genDoc.ConsensusParams.Validate(); err != nil { - return nil, err + return err } } + if len(genDoc.Validators) == 0 { + return errors.Errorf("The genesis file must have at least one validator") + } + + if genDoc.GenesisTime.IsZero() { + genDoc.GenesisTime = time.Now() + } + + return nil +} + +//------------------------------------------------------------ +// Make genesis state from file + +// GenesisDocFromJSON unmarshalls JSON data into a GenesisDoc. +func GenesisDocFromJSON(jsonBlob []byte) (*GenesisDoc, error) { + genDoc := GenesisDoc{} + err := json.Unmarshal(jsonBlob, &genDoc) + if err != nil { + return nil, err + } + + if err := genDoc.ValidateAndComplete(); err != nil { + return nil, err + } + return &genDoc, err } From 2b6db268cfb1cacfe8fb70347f68a8622c87fb50 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 12 Sep 2017 15:20:19 -0400 Subject: [PATCH 55/92] genesis json tests and mv ConsensusParams to types --- blockchain/reactor.go | 4 +- config/consensus.go | 27 -------------- consensus/replay_test.go | 8 ++-- consensus/state.go | 4 +- consensus/state_test.go | 14 +++---- state/state.go | 28 ++++++++------ types/genesis.go | 14 +++---- types/genesis_test.go | 79 ++++++++++++++++++++++++++++++++++++++++ types/params.go | 32 ++++++++++++++++ 9 files changed, 149 insertions(+), 61 deletions(-) delete mode 100644 config/consensus.go create mode 100644 types/genesis_test.go create mode 100644 types/params.go diff --git a/blockchain/reactor.go b/blockchain/reactor.go index fa3c70799..fccfcdf8f 100644 --- a/blockchain/reactor.go +++ b/blockchain/reactor.go @@ -165,7 +165,7 @@ func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) // maxMsgSize returns the maximum allowable size of a // message on the blockchain reactor. func (bcR *BlockchainReactor) maxMsgSize() int { - return bcR.state.GenesisDoc.ConsensusParams.MaxBlockSizeBytes + 2 + return bcR.state.Params().MaxBlockSizeBytes + 2 } // Handle messages from the poolReactor telling the reactor what to do. @@ -226,7 +226,7 @@ FOR_LOOP: // We need both to sync the first block. break SYNC_LOOP } - firstParts := first.MakePartSet(bcR.state.GenesisDoc.ConsensusParams.BlockPartSizeBytes) + firstParts := first.MakePartSet(bcR.state.Params().BlockPartSizeBytes) firstPartsHeader := firstParts.Header() // Finally, verify the first block using the second's commit // NOTE: we can probably make this more efficient, but note that calling diff --git a/config/consensus.go b/config/consensus.go deleted file mode 100644 index 5b10911d8..000000000 --- a/config/consensus.go +++ /dev/null @@ -1,27 +0,0 @@ -package config - -import ( - "fmt" -) - -type ConsensusParams struct { - MaxBlockSizeBytes int `json:"max_block_size_bytes"` - BlockPartSizeBytes int `json:"block_part_size_bytes"` -} - -func DefaultConsensusParams() *ConsensusParams { - return &ConsensusParams{ - MaxBlockSizeBytes: 22020096, // 21MB - BlockPartSizeBytes: 65536, // 64kB, - } -} - -func (params *ConsensusParams) Validate() error { - if params.MaxBlockSizeBytes <= 0 { - return fmt.Errorf("MaxBlockSizeBytes must be greater than 0. Got %d", params.MaxBlockSizeBytes) - } - if params.BlockPartSizeBytes <= 0 { - return fmt.Errorf("BlockPartSizeBytes must be greater than 0. Got %d", params.BlockPartSizeBytes) - } - return nil -} diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 179e8696c..3fa885176 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -381,7 +381,7 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) { } func applyBlock(st *sm.State, blk *types.Block, proxyApp proxy.AppConns) { - testPartSize := st.GenesisDoc.ConsensusParams.BlockPartSizeBytes + testPartSize := st.Params().BlockPartSizeBytes err := st.ApplyBlock(nil, proxyApp.Consensus(), blk, blk.MakePartSet(testPartSize).Header(), mempool) if err != nil { panic(err) @@ -561,7 +561,7 @@ func stateAndStore(config *cfg.Config, pubKey crypto.PubKey) (*sm.State, *mockBl state := sm.MakeGenesisStateFromFile(stateDB, config.GenesisFile()) state.SetLogger(log.TestingLogger().With("module", "state")) - store := NewMockBlockStore(config, state.GenesisDoc.ConsensusParams) + store := NewMockBlockStore(config, state.Params()) return state, store } @@ -570,13 +570,13 @@ func stateAndStore(config *cfg.Config, pubKey crypto.PubKey) (*sm.State, *mockBl type mockBlockStore struct { config *cfg.Config - params *cfg.ConsensusParams + params *types.ConsensusParams chain []*types.Block commits []*types.Commit } // TODO: NewBlockStore(db.NewMemDB) ... -func NewMockBlockStore(config *cfg.Config, params *cfg.ConsensusParams) *mockBlockStore { +func NewMockBlockStore(config *cfg.Config, params *types.ConsensusParams) *mockBlockStore { return &mockBlockStore{config, params, nil, nil} } diff --git a/consensus/state.go b/consensus/state.go index 8a19a052d..1a5b26e62 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -984,7 +984,7 @@ func (cs *ConsensusState) createProposalBlock() (block *types.Block, blockParts return types.MakeBlock(cs.Height, cs.state.ChainID, txs, commit, cs.state.LastBlockID, cs.state.Validators.Hash(), - cs.state.AppHash, cs.state.GenesisDoc.ConsensusParams.BlockPartSizeBytes) + cs.state.AppHash, cs.state.Params().BlockPartSizeBytes) } // Enter: `timeoutPropose` after entering Propose. @@ -1419,7 +1419,7 @@ func (cs *ConsensusState) addProposalBlockPart(height int, part *types.Part, ver var n int var err error cs.ProposalBlock = wire.ReadBinary(&types.Block{}, cs.ProposalBlockParts.GetReader(), - cs.state.GenesisDoc.ConsensusParams.MaxBlockSizeBytes, &n, &err).(*types.Block) + cs.state.Params().MaxBlockSizeBytes, &n, &err).(*types.Block) // NOTE: it's possible to receive complete proposal blocks for future rounds without having the proposal cs.Logger.Info("Received complete proposal block", "height", cs.ProposalBlock.Height, "hash", cs.ProposalBlock.Hash()) if cs.Step == RoundStepPropose && cs.isProposalComplete() { diff --git a/consensus/state_test.go b/consensus/state_test.go index 6f804fcb4..9ae052033 100644 --- a/consensus/state_test.go +++ b/consensus/state_test.go @@ -180,7 +180,7 @@ func TestBadProposal(t *testing.T) { height, round := cs1.Height, cs1.Round vs2 := vss[1] - partSize := cs1.state.GenesisDoc.ConsensusParams.BlockPartSizeBytes + partSize := cs1.state.Params().BlockPartSizeBytes proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1) voteCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringVote(), 1) @@ -327,7 +327,7 @@ func TestLockNoPOL(t *testing.T) { vs2 := vss[1] height := cs1.Height - partSize := cs1.state.GenesisDoc.ConsensusParams.BlockPartSizeBytes + partSize := cs1.state.Params().BlockPartSizeBytes timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1) timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1) @@ -493,7 +493,7 @@ func TestLockPOLRelock(t *testing.T) { cs1, vss := randConsensusState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] - partSize := cs1.state.GenesisDoc.ConsensusParams.BlockPartSizeBytes + partSize := cs1.state.Params().BlockPartSizeBytes timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1) timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1) @@ -608,7 +608,7 @@ func TestLockPOLUnlock(t *testing.T) { cs1, vss := randConsensusState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] - partSize := cs1.state.GenesisDoc.ConsensusParams.BlockPartSizeBytes + partSize := cs1.state.Params().BlockPartSizeBytes proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1) timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1) @@ -703,7 +703,7 @@ func TestLockPOLSafety1(t *testing.T) { cs1, vss := randConsensusState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] - partSize := cs1.state.GenesisDoc.ConsensusParams.BlockPartSizeBytes + partSize := cs1.state.Params().BlockPartSizeBytes proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1) timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1) @@ -824,7 +824,7 @@ func TestLockPOLSafety2(t *testing.T) { cs1, vss := randConsensusState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] - partSize := cs1.state.GenesisDoc.ConsensusParams.BlockPartSizeBytes + partSize := cs1.state.Params().BlockPartSizeBytes proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1) timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1) @@ -999,7 +999,7 @@ func TestHalt1(t *testing.T) { cs1, vss := randConsensusState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] - partSize := cs1.state.GenesisDoc.ConsensusParams.BlockPartSizeBytes + partSize := cs1.state.Params().BlockPartSizeBytes proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1) timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1) diff --git a/state/state.go b/state/state.go index 4e0fa75b5..0ea15acaf 100644 --- a/state/state.go +++ b/state/state.go @@ -65,6 +65,19 @@ type State struct { logger log.Logger } +// GetState loads the most recent state from the database, +// or creates a new one from the given genesisFile and persists the result +// to the database. +func GetState(stateDB dbm.DB, genesisFile string) *State { + state := LoadState(stateDB) + if state == nil { + state = MakeGenesisStateFromFile(stateDB, genesisFile) + state.Save() + } + + return state +} + // LoadState loads the State from the database. func LoadState(db dbm.DB) *State { return loadState(db, stateKey) @@ -248,17 +261,10 @@ func (s *State) GetValidators() (*types.ValidatorSet, *types.ValidatorSet) { return s.LastValidators, s.Validators } -// GetState loads the most recent state from the database, -// or creates a new one from the given genesisFile and persists the result -// to the database. -func GetState(stateDB dbm.DB, genesisFile string) *State { - state := LoadState(stateDB) - if state == nil { - state = MakeGenesisStateFromFile(stateDB, genesisFile) - state.Save() - } - - return state +// Params returns the consensus parameters used for +// validating blocks +func (s *State) Params() *types.ConsensusParams { + return s.GenesisDoc.ConsensusParams } //------------------------------------------------------------------------ diff --git a/types/genesis.go b/types/genesis.go index 79bf63c8b..822a4e166 100644 --- a/types/genesis.go +++ b/types/genesis.go @@ -10,8 +10,6 @@ import ( "github.com/tendermint/go-crypto" "github.com/tendermint/go-wire/data" cmn "github.com/tendermint/tmlibs/common" - - cfg "github.com/tendermint/tendermint/config" ) //------------------------------------------------------------ @@ -26,11 +24,11 @@ type GenesisValidator struct { // GenesisDoc defines the initial conditions for a tendermint blockchain, in particular its validator set. type GenesisDoc struct { - GenesisTime time.Time `json:"genesis_time"` - ChainID string `json:"chain_id"` - ConsensusParams *cfg.ConsensusParams `json:"consensus_params"` - Validators []GenesisValidator `json:"validators"` - AppHash data.Bytes `json:"app_hash"` + GenesisTime time.Time `json:"genesis_time"` + ChainID string `json:"chain_id"` + ConsensusParams *ConsensusParams `json:"consensus_params"` + Validators []GenesisValidator `json:"validators"` + AppHash data.Bytes `json:"app_hash"` } // SaveAs is a utility method for saving GenensisDoc as a JSON file. @@ -61,7 +59,7 @@ func (genDoc *GenesisDoc) ValidateAndComplete() error { } if genDoc.ConsensusParams == nil { - genDoc.ConsensusParams = cfg.DefaultConsensusParams() + genDoc.ConsensusParams = DefaultConsensusParams() } else { if err := genDoc.ConsensusParams.Validate(); err != nil { return err diff --git a/types/genesis_test.go b/types/genesis_test.go new file mode 100644 index 000000000..9743a9610 --- /dev/null +++ b/types/genesis_test.go @@ -0,0 +1,79 @@ +package types + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + crypto "github.com/tendermint/go-crypto" +) + +func TestGenesis(t *testing.T) { + + // test some bad ones from raw json + testCases := [][]byte{ + []byte{}, // empty + []byte{1, 1, 1, 1, 1}, // junk + []byte(`{}`), // empty + []byte(`{"chain_id": "mychain"}`), // missing validators + []byte(`{"validators": [{"data":abcd}`), // missing validators + []byte(`{"validators":[{"pub_key": + {"type":"ed25519","data":"961EAB8752E51A03618502F55C2B6E09C38C65635C64CCF3173ED452CF86C957"}, + "amount":10,"name":""}]}`), // missing chain_id + } + + for _, testCase := range testCases { + _, err := GenesisDocFromJSON(testCase) + assert.NotNil(t, err, "expected error for empty genDoc json") + } + + // test a good one by raw json + genDocBytes := []byte(`{"genesis_time":"0001-01-01T00:00:00Z","chain_id":"test-chain-QDKdJr","consensus_params":null,"validators":[{"pub_key":{"type":"ed25519","data":"961EAB8752E51A03618502F55C2B6E09C38C65635C64CCF3173ED452CF86C957"},"amount":10,"name":""}],"app_hash":""}`) + _, err := GenesisDocFromJSON(genDocBytes) + assert.Nil(t, err, "expected no error for good genDoc json") + + // create a base gendoc from struct + baseGenDoc := &GenesisDoc{ + ChainID: "abc", + Validators: []GenesisValidator{{crypto.GenPrivKeyEd25519().PubKey(), 10, "myval"}}, + } + genDocBytes, err = json.Marshal(baseGenDoc) + assert.Nil(t, err, "error marshalling genDoc") + + // test base gendoc and check consensus params were filled + genDoc, err := GenesisDocFromJSON(genDocBytes) + assert.Nil(t, err, "expected no error for valid genDoc json") + assert.NotNil(t, genDoc.ConsensusParams, "expected consensus params to be filled in") + + // create json with consensus params filled + genDocBytes, err = json.Marshal(genDoc) + assert.Nil(t, err, "error marshalling genDoc") + genDoc, err = GenesisDocFromJSON(genDocBytes) + assert.Nil(t, err, "expected no error for valid genDoc json") + + // test with invalid consensus params + genDoc.ConsensusParams.MaxBlockSizeBytes = 0 + genDocBytes, err = json.Marshal(genDoc) + assert.Nil(t, err, "error marshalling genDoc") + genDoc, err = GenesisDocFromJSON(genDocBytes) + assert.NotNil(t, err, "expected error for genDoc json with block size of 0") +} + +func TestConsensusParams(t *testing.T) { + testCases := []struct { + params *ConsensusParams + valid bool + }{ + {&ConsensusParams{1, 1}, true}, + {&ConsensusParams{1, 0}, false}, + {&ConsensusParams{0, 1}, false}, + {&ConsensusParams{0, 0}, false}, + } + for _, testCase := range testCases { + if testCase.valid { + assert.Nil(t, testCase.params.Validate(), "expected no error for valid params") + } else { + assert.NotNil(t, testCase.params.Validate(), "expected error for non valid params") + } + } +} diff --git a/types/params.go b/types/params.go new file mode 100644 index 000000000..cff9ae3d3 --- /dev/null +++ b/types/params.go @@ -0,0 +1,32 @@ +package types + +import ( + "github.com/pkg/errors" +) + +// ConsensusParams contains consensus critical parameters +// that determine the validity of blocks. +type ConsensusParams struct { + MaxBlockSizeBytes int `json:"max_block_size_bytes"` + BlockPartSizeBytes int `json:"block_part_size_bytes"` +} + +// DefaultConsensusParams returns a default ConsensusParams. +func DefaultConsensusParams() *ConsensusParams { + return &ConsensusParams{ + MaxBlockSizeBytes: 22020096, // 21MB + BlockPartSizeBytes: 65536, // 64kB, + } +} + +// Validate validates the ConsensusParams to ensure all values +// are within their allowed limits, and returns an error if they are not. +func (params *ConsensusParams) Validate() error { + if params.MaxBlockSizeBytes <= 0 { + return errors.Errorf("MaxBlockSizeBytes must be greater than 0. Got %d", params.MaxBlockSizeBytes) + } + if params.BlockPartSizeBytes <= 0 { + return errors.Errorf("BlockPartSizeBytes must be greater than 0. Got %d", params.BlockPartSizeBytes) + } + return nil +} From d343560108e247a91fb05a3bbb082c1946de62df Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 15 Sep 2017 23:52:09 -0400 Subject: [PATCH 56/92] adr: add 005 consensus params --- docs/architecture/adr-005-consensus-params.md | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 docs/architecture/adr-005-consensus-params.md diff --git a/docs/architecture/adr-005-consensus-params.md b/docs/architecture/adr-005-consensus-params.md new file mode 100644 index 000000000..6f43fe7d9 --- /dev/null +++ b/docs/architecture/adr-005-consensus-params.md @@ -0,0 +1,77 @@ +# ADR 005: Consensus Params + +## Context + +Consensus critical parameters controlling blockchain capacity have until now been hard coded, loaded from a local config, or neglected. +Since they may be need to be different in different networks, and potentially to evolve over time within +networks, we seek to initialize them in a genesis file, and expose them through the ABCI. + +While we have some specific parameters now, like maximum block and transaction size, we expect to have more in the future, +such as a period over which evidence is valid, or the frequency of checkpoints. + +## Decision + +### ConsensusParams + +A new `ConsensusParams` is optionally included in the `genesis.json` file, +and loaded into the `State`. Any items not included are set to their default value. +A value of 0 is undefined (see ABCI, below). A value of -1 is used to indicate the parameter does not apply. +No consensus critical parameters should ever be found in the `config.toml`. + +``` +type ConsensusParams struct { + BlockSizeParams + TxSizeParams + BlockGossipParams +} + +type BlockSizeParams struct { + BlockSizeBytes int + BlockSizeTxs int + BlockSizeGas int +} + +type TxSizeParams struct { + TxSizeBytes int + TxSizeGas int +} + +type BlockGossipParams struct { + BlockPartSizeBytes int +} +``` + +The `ConsensusParams` can evolve over time by adding new structs that cover different aspects of the consensus rules. + +### ABCI + +#### InitChain + +InitChain currently takes the initial validator set. It should be extended to also take the ConsensusParams. +In fact, it might as well just consume the whole Genesis. + +#### EndBlock + +The EndBlock response includes a `ConsensusParams`, which includes BlockSizeParams and TxSizeParams, but not BlockGossipParams. +Other param struct can be added to `ConsensusParams` in the future. +The `0` value is used to denote no change. +Any other value will update that parameter in the `State.ConsensusParams`, to be applied for the next block. +Tendermint should have hard-coded upper limits as sanity checks. + +## Status + +Proposed. + +## Consequences + +### Positive + +- Alternative capacity limits and consensus parameters can be specified without re-compiling the software. +- They can also change over time under the control of the application + +### Negative + +- More exposed parameters is more complexity +- Different rules at different heights in the blockchain complicates fast sync + +### Neutral From 3a03fe5a15e012d1fd2cf61cd8c8e44cdf109a62 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 16 Sep 2017 00:16:49 -0400 Subject: [PATCH 57/92] updated to match adr 005 --- blockchain/reactor.go | 2 +- consensus/state.go | 2 +- docs/architecture/adr-005-consensus-params.md | 24 +++++--- types/genesis_test.go | 19 ++++-- types/params.go | 60 ++++++++++++++++--- 5 files changed, 84 insertions(+), 23 deletions(-) diff --git a/blockchain/reactor.go b/blockchain/reactor.go index fccfcdf8f..fb68aadda 100644 --- a/blockchain/reactor.go +++ b/blockchain/reactor.go @@ -165,7 +165,7 @@ func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) // maxMsgSize returns the maximum allowable size of a // message on the blockchain reactor. func (bcR *BlockchainReactor) maxMsgSize() int { - return bcR.state.Params().MaxBlockSizeBytes + 2 + return bcR.state.Params().BlockSizeParams.MaxBytes + 2 } // Handle messages from the poolReactor telling the reactor what to do. diff --git a/consensus/state.go b/consensus/state.go index 1a5b26e62..ac1656209 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -1419,7 +1419,7 @@ func (cs *ConsensusState) addProposalBlockPart(height int, part *types.Part, ver var n int var err error cs.ProposalBlock = wire.ReadBinary(&types.Block{}, cs.ProposalBlockParts.GetReader(), - cs.state.Params().MaxBlockSizeBytes, &n, &err).(*types.Block) + cs.state.Params().BlockSizeParams.MaxBytes, &n, &err).(*types.Block) // NOTE: it's possible to receive complete proposal blocks for future rounds without having the proposal cs.Logger.Info("Received complete proposal block", "height", cs.ProposalBlock.Height, "hash", cs.ProposalBlock.Hash()) if cs.Step == RoundStepPropose && cs.isProposalComplete() { diff --git a/docs/architecture/adr-005-consensus-params.md b/docs/architecture/adr-005-consensus-params.md index 6f43fe7d9..678de42eb 100644 --- a/docs/architecture/adr-005-consensus-params.md +++ b/docs/architecture/adr-005-consensus-params.md @@ -13,10 +13,12 @@ such as a period over which evidence is valid, or the frequency of checkpoints. ### ConsensusParams +No consensus critical parameters should ever be found in the `config.toml`. + A new `ConsensusParams` is optionally included in the `genesis.json` file, and loaded into the `State`. Any items not included are set to their default value. A value of 0 is undefined (see ABCI, below). A value of -1 is used to indicate the parameter does not apply. -No consensus critical parameters should ever be found in the `config.toml`. +The parameters are used to determine the validity of a block (and tx) via the union of all relevant parameters. ``` type ConsensusParams struct { @@ -26,14 +28,14 @@ type ConsensusParams struct { } type BlockSizeParams struct { - BlockSizeBytes int - BlockSizeTxs int - BlockSizeGas int + MaxBytes int + MaxTxs int + MaxGas int } type TxSizeParams struct { - TxSizeBytes int - TxSizeGas int + MaxBytes int + MaxGas int } type BlockGossipParams struct { @@ -43,12 +45,16 @@ type BlockGossipParams struct { The `ConsensusParams` can evolve over time by adding new structs that cover different aspects of the consensus rules. +The `BlockPartSizeBytes` and the `BlockSizeParams.MaxBytes` are enforced to be greater than 0. +The former because we need a part size, the latter so that we always have at least some sanity check over the size of blocks. + ### ABCI #### InitChain -InitChain currently takes the initial validator set. It should be extended to also take the ConsensusParams. -In fact, it might as well just consume the whole Genesis. +InitChain currently takes the initial validator set. It should be extended to also take parts of the ConsensusParams. +There is some case to be made for it to take the entire Genesis, except there may be things in the genesis, +like the BlockPartSize, that the app shouldn't really know about. #### EndBlock @@ -75,3 +81,5 @@ Proposed. - Different rules at different heights in the blockchain complicates fast sync ### Neutral + +- The TxSizeParams, which checks validity, may be in conflict with the config's `max_block_size_tx`, which determines proposal sizes diff --git a/types/genesis_test.go b/types/genesis_test.go index 9743a9610..7d7731f13 100644 --- a/types/genesis_test.go +++ b/types/genesis_test.go @@ -52,22 +52,31 @@ func TestGenesis(t *testing.T) { assert.Nil(t, err, "expected no error for valid genDoc json") // test with invalid consensus params - genDoc.ConsensusParams.MaxBlockSizeBytes = 0 + genDoc.ConsensusParams.BlockSizeParams.MaxBytes = 0 genDocBytes, err = json.Marshal(genDoc) assert.Nil(t, err, "error marshalling genDoc") genDoc, err = GenesisDocFromJSON(genDocBytes) assert.NotNil(t, err, "expected error for genDoc json with block size of 0") } +func newConsensusParams(blockSize, partSize int) *ConsensusParams { + return &ConsensusParams{ + BlockSizeParams: &BlockSizeParams{MaxBytes: blockSize}, + BlockGossipParams: &BlockGossipParams{BlockPartSizeBytes: partSize}, + } + +} + func TestConsensusParams(t *testing.T) { + testCases := []struct { params *ConsensusParams valid bool }{ - {&ConsensusParams{1, 1}, true}, - {&ConsensusParams{1, 0}, false}, - {&ConsensusParams{0, 1}, false}, - {&ConsensusParams{0, 0}, false}, + {newConsensusParams(1, 1), true}, + {newConsensusParams(1, 0), false}, + {newConsensusParams(0, 1), false}, + {newConsensusParams(0, 0), false}, } for _, testCase := range testCases { if testCase.valid { diff --git a/types/params.go b/types/params.go index cff9ae3d3..71ffe60c9 100644 --- a/types/params.go +++ b/types/params.go @@ -7,26 +7,70 @@ import ( // ConsensusParams contains consensus critical parameters // that determine the validity of blocks. type ConsensusParams struct { - MaxBlockSizeBytes int `json:"max_block_size_bytes"` - BlockPartSizeBytes int `json:"block_part_size_bytes"` + *BlockSizeParams `json:"block_size_params"` + *TxSizeParams `json:"tx_size_params"` + *BlockGossipParams `json:"block_gossip_params"` +} + +// BlockSizeParams contain limits on the block size. +type BlockSizeParams struct { + MaxBytes int `json:"max_bytes"` // NOTE: must not be 0 + MaxTxs int `json:"max_txs"` + MaxGas int `json:"max_gas"` +} + +// TxSizeParams contain limits on the tx size. +type TxSizeParams struct { + MaxBytes int `json:"max_bytes"` + MaxGas int `json:"max_gas"` +} + +// BlockGossipParams determine consensus critical elements of how blocks are gossiped +type BlockGossipParams struct { + BlockPartSizeBytes int `json:"block_part_size_bytes"` // NOTE: must not be 0 } // DefaultConsensusParams returns a default ConsensusParams. func DefaultConsensusParams() *ConsensusParams { return &ConsensusParams{ - MaxBlockSizeBytes: 22020096, // 21MB - BlockPartSizeBytes: 65536, // 64kB, + DefaultBlockSizeParams(), + DefaultTxSizeParams(), + DefaultBlockGossipParams(), + } +} + +// DefaultBlockSizeParams returns a default BlockSizeParams. +func DefaultBlockSizeParams() *BlockSizeParams { + return &BlockSizeParams{ + MaxBytes: 22020096, // 21MB + MaxTxs: 100000, + MaxGas: -1, + } +} + +// DefaultTxSizeParams returns a default TxSizeParams. +func DefaultTxSizeParams() *TxSizeParams { + return &TxSizeParams{ + MaxBytes: 10240, // 10kB + MaxGas: -1, + } +} + +// DefaultBlockGossipParams returns a default BlockGossipParams. +func DefaultBlockGossipParams() *BlockGossipParams { + return &BlockGossipParams{ + BlockPartSizeBytes: 65536, // 64kB, } } // Validate validates the ConsensusParams to ensure all values // are within their allowed limits, and returns an error if they are not. func (params *ConsensusParams) Validate() error { - if params.MaxBlockSizeBytes <= 0 { - return errors.Errorf("MaxBlockSizeBytes must be greater than 0. Got %d", params.MaxBlockSizeBytes) + if params.BlockSizeParams.MaxBytes <= 0 { + return errors.Errorf("BlockSizeParams.MaxBytes must be greater than 0. Got %d", params.BlockSizeParams.MaxBytes) } - if params.BlockPartSizeBytes <= 0 { - return errors.Errorf("BlockPartSizeBytes must be greater than 0. Got %d", params.BlockPartSizeBytes) + if params.BlockGossipParams.BlockPartSizeBytes <= 0 { + return errors.Errorf("BlockGossipParams.BlockPartSizeBytes must be greater than 0. Got %d", params.BlockGossipParams.BlockPartSizeBytes) } return nil } From 715e74186ca89d147cdd75c26295d1f2ff0b4827 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 18 Sep 2017 17:00:09 -0400 Subject: [PATCH 58/92] fixes from review --- types/genesis_test.go | 31 ++++++++++++++++--------------- types/params.go | 11 +++++++++++ 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/types/genesis_test.go b/types/genesis_test.go index 7d7731f13..75184e5a6 100644 --- a/types/genesis_test.go +++ b/types/genesis_test.go @@ -12,11 +12,12 @@ func TestGenesis(t *testing.T) { // test some bad ones from raw json testCases := [][]byte{ - []byte{}, // empty - []byte{1, 1, 1, 1, 1}, // junk - []byte(`{}`), // empty - []byte(`{"chain_id": "mychain"}`), // missing validators - []byte(`{"validators": [{"data":abcd}`), // missing validators + []byte{}, // empty + []byte{1, 1, 1, 1, 1}, // junk + []byte(`{}`), // empty + []byte(`{"chain_id": "mychain"}`), // missing validators + []byte(`{"chain_id": "mychain", "validators": []`), // missing validators + []byte(`{"chain_id": "mychain", "validators": [{}]`), // missing validators []byte(`{"validators":[{"pub_key": {"type":"ed25519","data":"961EAB8752E51A03618502F55C2B6E09C38C65635C64CCF3173ED452CF86C957"}, "amount":10,"name":""}]}`), // missing chain_id @@ -24,13 +25,13 @@ func TestGenesis(t *testing.T) { for _, testCase := range testCases { _, err := GenesisDocFromJSON(testCase) - assert.NotNil(t, err, "expected error for empty genDoc json") + assert.Error(t, err, "expected error for empty genDoc json") } // test a good one by raw json genDocBytes := []byte(`{"genesis_time":"0001-01-01T00:00:00Z","chain_id":"test-chain-QDKdJr","consensus_params":null,"validators":[{"pub_key":{"type":"ed25519","data":"961EAB8752E51A03618502F55C2B6E09C38C65635C64CCF3173ED452CF86C957"},"amount":10,"name":""}],"app_hash":""}`) _, err := GenesisDocFromJSON(genDocBytes) - assert.Nil(t, err, "expected no error for good genDoc json") + assert.NoError(t, err, "expected no error for good genDoc json") // create a base gendoc from struct baseGenDoc := &GenesisDoc{ @@ -38,25 +39,25 @@ func TestGenesis(t *testing.T) { Validators: []GenesisValidator{{crypto.GenPrivKeyEd25519().PubKey(), 10, "myval"}}, } genDocBytes, err = json.Marshal(baseGenDoc) - assert.Nil(t, err, "error marshalling genDoc") + assert.NoError(t, err, "error marshalling genDoc") // test base gendoc and check consensus params were filled genDoc, err := GenesisDocFromJSON(genDocBytes) - assert.Nil(t, err, "expected no error for valid genDoc json") + assert.NoError(t, err, "expected no error for valid genDoc json") assert.NotNil(t, genDoc.ConsensusParams, "expected consensus params to be filled in") // create json with consensus params filled genDocBytes, err = json.Marshal(genDoc) - assert.Nil(t, err, "error marshalling genDoc") + assert.NoError(t, err, "error marshalling genDoc") genDoc, err = GenesisDocFromJSON(genDocBytes) - assert.Nil(t, err, "expected no error for valid genDoc json") + assert.NoError(t, err, "expected no error for valid genDoc json") // test with invalid consensus params genDoc.ConsensusParams.BlockSizeParams.MaxBytes = 0 genDocBytes, err = json.Marshal(genDoc) - assert.Nil(t, err, "error marshalling genDoc") + assert.NoError(t, err, "error marshalling genDoc") genDoc, err = GenesisDocFromJSON(genDocBytes) - assert.NotNil(t, err, "expected error for genDoc json with block size of 0") + assert.Error(t, err, "expected error for genDoc json with block size of 0") } func newConsensusParams(blockSize, partSize int) *ConsensusParams { @@ -80,9 +81,9 @@ func TestConsensusParams(t *testing.T) { } for _, testCase := range testCases { if testCase.valid { - assert.Nil(t, testCase.params.Validate(), "expected no error for valid params") + assert.NoError(t, testCase.params.Validate(), "expected no error for valid params") } else { - assert.NotNil(t, testCase.params.Validate(), "expected error for non valid params") + assert.Error(t, testCase.params.Validate(), "expected error for non valid params") } } } diff --git a/types/params.go b/types/params.go index 71ffe60c9..b55ceb8c3 100644 --- a/types/params.go +++ b/types/params.go @@ -4,6 +4,10 @@ import ( "github.com/pkg/errors" ) +const ( + maxBlockSizeBytes = 104857600 // 100MB +) + // ConsensusParams contains consensus critical parameters // that determine the validity of blocks. type ConsensusParams struct { @@ -66,11 +70,18 @@ func DefaultBlockGossipParams() *BlockGossipParams { // Validate validates the ConsensusParams to ensure all values // are within their allowed limits, and returns an error if they are not. func (params *ConsensusParams) Validate() error { + // ensure some values are greater than 0 if params.BlockSizeParams.MaxBytes <= 0 { return errors.Errorf("BlockSizeParams.MaxBytes must be greater than 0. Got %d", params.BlockSizeParams.MaxBytes) } if params.BlockGossipParams.BlockPartSizeBytes <= 0 { return errors.Errorf("BlockGossipParams.BlockPartSizeBytes must be greater than 0. Got %d", params.BlockGossipParams.BlockPartSizeBytes) } + + // ensure blocks aren't too big + if cp.BlockSizeParams.MaxBytes > maxBlockSizeBytes { + return errors.Errorf("BlockSizeParams.MaxBytes is too big. %d > %d", + cp.BlockSizeParams.MaxBytes, maxBlockSizeBytes) + } return nil } From 5feeb65cf0df559a3467a6a9d2f7271a94328178 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 21 Sep 2017 14:34:36 -0400 Subject: [PATCH 59/92] dont use pointers for ConsensusParams --- consensus/replay_test.go | 4 ++-- state/state.go | 4 +++- types/genesis.go | 5 +++-- types/genesis_test.go | 10 +++++----- types/params.go | 26 +++++++++++++------------- 5 files changed, 26 insertions(+), 23 deletions(-) diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 3fa885176..03ca6c8d7 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -570,13 +570,13 @@ func stateAndStore(config *cfg.Config, pubKey crypto.PubKey) (*sm.State, *mockBl type mockBlockStore struct { config *cfg.Config - params *types.ConsensusParams + params types.ConsensusParams chain []*types.Block commits []*types.Commit } // TODO: NewBlockStore(db.NewMemDB) ... -func NewMockBlockStore(config *cfg.Config, params *types.ConsensusParams) *mockBlockStore { +func NewMockBlockStore(config *cfg.Config, params types.ConsensusParams) *mockBlockStore { return &mockBlockStore{config, params, nil, nil} } diff --git a/state/state.go b/state/state.go index 0ea15acaf..1374fcf46 100644 --- a/state/state.go +++ b/state/state.go @@ -263,7 +263,9 @@ func (s *State) GetValidators() (*types.ValidatorSet, *types.ValidatorSet) { // Params returns the consensus parameters used for // validating blocks -func (s *State) Params() *types.ConsensusParams { +func (s *State) Params() types.ConsensusParams { + // TODO: this should move into the State proper + // when we allow the app to change it return s.GenesisDoc.ConsensusParams } diff --git a/types/genesis.go b/types/genesis.go index 822a4e166..06b3d124c 100644 --- a/types/genesis.go +++ b/types/genesis.go @@ -26,7 +26,7 @@ type GenesisValidator struct { type GenesisDoc struct { GenesisTime time.Time `json:"genesis_time"` ChainID string `json:"chain_id"` - ConsensusParams *ConsensusParams `json:"consensus_params"` + ConsensusParams ConsensusParams `json:"consensus_params"` Validators []GenesisValidator `json:"validators"` AppHash data.Bytes `json:"app_hash"` } @@ -58,7 +58,8 @@ func (genDoc *GenesisDoc) ValidateAndComplete() error { return errors.Errorf("Genesis doc must include non-empty chain_id") } - if genDoc.ConsensusParams == nil { + var emptyParams ConsensusParams + if genDoc.ConsensusParams == emptyParams { genDoc.ConsensusParams = DefaultConsensusParams() } else { if err := genDoc.ConsensusParams.Validate(); err != nil { diff --git a/types/genesis_test.go b/types/genesis_test.go index 75184e5a6..d06a70dab 100644 --- a/types/genesis_test.go +++ b/types/genesis_test.go @@ -60,10 +60,10 @@ func TestGenesis(t *testing.T) { assert.Error(t, err, "expected error for genDoc json with block size of 0") } -func newConsensusParams(blockSize, partSize int) *ConsensusParams { - return &ConsensusParams{ - BlockSizeParams: &BlockSizeParams{MaxBytes: blockSize}, - BlockGossipParams: &BlockGossipParams{BlockPartSizeBytes: partSize}, +func newConsensusParams(blockSize, partSize int) ConsensusParams { + return ConsensusParams{ + BlockSizeParams: BlockSizeParams{MaxBytes: blockSize}, + BlockGossipParams: BlockGossipParams{BlockPartSizeBytes: partSize}, } } @@ -71,7 +71,7 @@ func newConsensusParams(blockSize, partSize int) *ConsensusParams { func TestConsensusParams(t *testing.T) { testCases := []struct { - params *ConsensusParams + params ConsensusParams valid bool }{ {newConsensusParams(1, 1), true}, diff --git a/types/params.go b/types/params.go index b55ceb8c3..cd3242192 100644 --- a/types/params.go +++ b/types/params.go @@ -11,9 +11,9 @@ const ( // ConsensusParams contains consensus critical parameters // that determine the validity of blocks. type ConsensusParams struct { - *BlockSizeParams `json:"block_size_params"` - *TxSizeParams `json:"tx_size_params"` - *BlockGossipParams `json:"block_gossip_params"` + BlockSizeParams `json:"block_size_params"` + TxSizeParams `json:"tx_size_params"` + BlockGossipParams `json:"block_gossip_params"` } // BlockSizeParams contain limits on the block size. @@ -35,8 +35,8 @@ type BlockGossipParams struct { } // DefaultConsensusParams returns a default ConsensusParams. -func DefaultConsensusParams() *ConsensusParams { - return &ConsensusParams{ +func DefaultConsensusParams() ConsensusParams { + return ConsensusParams{ DefaultBlockSizeParams(), DefaultTxSizeParams(), DefaultBlockGossipParams(), @@ -44,8 +44,8 @@ func DefaultConsensusParams() *ConsensusParams { } // DefaultBlockSizeParams returns a default BlockSizeParams. -func DefaultBlockSizeParams() *BlockSizeParams { - return &BlockSizeParams{ +func DefaultBlockSizeParams() BlockSizeParams { + return BlockSizeParams{ MaxBytes: 22020096, // 21MB MaxTxs: 100000, MaxGas: -1, @@ -53,16 +53,16 @@ func DefaultBlockSizeParams() *BlockSizeParams { } // DefaultTxSizeParams returns a default TxSizeParams. -func DefaultTxSizeParams() *TxSizeParams { - return &TxSizeParams{ +func DefaultTxSizeParams() TxSizeParams { + return TxSizeParams{ MaxBytes: 10240, // 10kB MaxGas: -1, } } // DefaultBlockGossipParams returns a default BlockGossipParams. -func DefaultBlockGossipParams() *BlockGossipParams { - return &BlockGossipParams{ +func DefaultBlockGossipParams() BlockGossipParams { + return BlockGossipParams{ BlockPartSizeBytes: 65536, // 64kB, } } @@ -79,9 +79,9 @@ func (params *ConsensusParams) Validate() error { } // ensure blocks aren't too big - if cp.BlockSizeParams.MaxBytes > maxBlockSizeBytes { + if params.BlockSizeParams.MaxBytes > maxBlockSizeBytes { return errors.Errorf("BlockSizeParams.MaxBytes is too big. %d > %d", - cp.BlockSizeParams.MaxBytes, maxBlockSizeBytes) + params.BlockSizeParams.MaxBytes, maxBlockSizeBytes) } return nil } From 3089bbf2b8c011d04e2c5a53d3444d3fbc46e6f2 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 21 Sep 2017 14:37:34 -0400 Subject: [PATCH 60/92] Amount -> Power. Closes #166 --- cmd/tendermint/commands/init.go | 2 +- cmd/tendermint/commands/testnet.go | 4 ++-- config/toml.go | 2 +- consensus/common_test.go | 2 +- docs/specification/genesis.rst | 10 +++++----- docs/using-tendermint.rst | 6 +++--- rpc/client/rpc_test.go | 2 +- rpc/core/net.go | 2 +- state/state.go | 2 +- test/p2p/data/mach1/core/genesis.json | 8 ++++---- test/p2p/data/mach2/core/genesis.json | 8 ++++---- test/p2p/data/mach3/core/genesis.json | 8 ++++---- test/p2p/data/mach4/core/genesis.json | 8 ++++---- types/genesis.go | 10 ++++++++-- types/genesis_test.go | 4 ++-- 15 files changed, 42 insertions(+), 36 deletions(-) diff --git a/cmd/tendermint/commands/init.go b/cmd/tendermint/commands/init.go index cd16707cc..ce900defa 100644 --- a/cmd/tendermint/commands/init.go +++ b/cmd/tendermint/commands/init.go @@ -34,7 +34,7 @@ func initFiles(cmd *cobra.Command, args []string) { } genDoc.Validators = []types.GenesisValidator{types.GenesisValidator{ PubKey: privValidator.PubKey, - Amount: 10, + Power: 10, }} genDoc.SaveAs(genFile) diff --git a/cmd/tendermint/commands/testnet.go b/cmd/tendermint/commands/testnet.go index 58767eb05..315f3b498 100644 --- a/cmd/tendermint/commands/testnet.go +++ b/cmd/tendermint/commands/testnet.go @@ -7,8 +7,8 @@ import ( "github.com/spf13/cobra" - cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tendermint/types" + cmn "github.com/tendermint/tmlibs/common" ) var testnetFilesCmd = &cobra.Command{ @@ -48,7 +48,7 @@ func testnetFiles(cmd *cobra.Command, args []string) { privVal := types.LoadPrivValidator(privValFile) genVals[i] = types.GenesisValidator{ PubKey: privVal.PubKey, - Amount: 1, + Power: 1, Name: mach, } } diff --git a/config/toml.go b/config/toml.go index 999a05946..5dcbe5332 100644 --- a/config/toml.go +++ b/config/toml.go @@ -119,7 +119,7 @@ var testGenesis = `{ "type": "ed25519", "data":"3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8" }, - "amount": 10, + "power": 10, "name": "" } ], diff --git a/consensus/common_test.go b/consensus/common_test.go index 84f47d021..6a05b74e3 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -400,7 +400,7 @@ func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.G val, privVal := types.RandValidator(randPower, minPower) validators[i] = types.GenesisValidator{ PubKey: val.PubKey, - Amount: val.VotingPower, + Power: val.VotingPower, } privValidators[i] = privVal } diff --git a/docs/specification/genesis.rst b/docs/specification/genesis.rst index 32dd19632..2187eb572 100644 --- a/docs/specification/genesis.rst +++ b/docs/specification/genesis.rst @@ -21,7 +21,7 @@ Fields - ``validators``: - ``pub_key``: The first element specifies the pub\_key type. 1 == Ed25519. The second element are the pubkey bytes. -- ``amount``: The validator's voting power. +- ``power``: The validator's voting power. - ``name``: Name of the validator (optional). - ``app_hash``: The expected application hash (as returned by the ``Commit`` ABCI message) upon genesis. If the app's hash does not @@ -41,7 +41,7 @@ Sample genesis.json 1, "9BC5112CB9614D91CE423FA8744885126CD9D08D9FC9D1F42E552D662BAA411E" ], - "amount": 1, + "power": 1, "name": "mach1" }, { @@ -49,7 +49,7 @@ Sample genesis.json 1, "F46A5543D51F31660D9F59653B4F96061A740FF7433E0DC1ECBC30BE8494DE06" ], - "amount": 1, + "power": 1, "name": "mach2" }, { @@ -57,7 +57,7 @@ Sample genesis.json 1, "0E7B423C1635FD07C0FC3603B736D5D27953C1C6CA865BB9392CD79DE1A682BB" ], - "amount": 1, + "power": 1, "name": "mach3" }, { @@ -65,7 +65,7 @@ Sample genesis.json 1, "4F49237B9A32EB50682EDD83C48CE9CDB1D02A7CFDADCFF6EC8C1FAADB358879" ], - "amount": 1, + "power": 1, "name": "mach4" } ], diff --git a/docs/using-tendermint.rst b/docs/using-tendermint.rst index fb9882f6b..1cb3ad86e 100644 --- a/docs/using-tendermint.rst +++ b/docs/using-tendermint.rst @@ -177,7 +177,7 @@ When ``tendermint init`` is run, both a ``genesis.json`` and "genesis_time": "0001-01-01T00:00:00.000Z", "validators": [ { - "amount": 10, + "power": 10, "name": "", "pub_key": [ 1, @@ -310,7 +310,7 @@ then the new ``genesis.json`` will be: "genesis_time": "0001-01-01T00:00:00.000Z", "validators": [ { - "amount": 10, + "power": 10, "name": "", "pub_key": [ 1, @@ -318,7 +318,7 @@ then the new ``genesis.json`` will be: ] }, { - "amount": 10, + "power": 10, "name": "", "pub_key": [ 1, diff --git a/rpc/client/rpc_test.go b/rpc/client/rpc_test.go index fee92b4a9..33cd4b432 100644 --- a/rpc/client/rpc_test.go +++ b/rpc/client/rpc_test.go @@ -93,7 +93,7 @@ func TestGenesisAndValidators(t *testing.T) { val := vals.Validators[0] // make sure the current set is also the genesis set - assert.Equal(t, gval.Amount, val.VotingPower) + assert.Equal(t, gval.Power, val.VotingPower) assert.Equal(t, gval.PubKey, val.PubKey) } } diff --git a/rpc/core/net.go b/rpc/core/net.go index c6749941c..b3f1c7ce5 100644 --- a/rpc/core/net.go +++ b/rpc/core/net.go @@ -90,7 +90,7 @@ func UnsafeDialSeeds(seeds []string) (*ctypes.ResultDialSeeds, error) { // "validators": [ // { // "name": "", -// "amount": 10, +// "power": 10, // "pub_key": { // "data": "68DFDA7E50F82946E7E8546BED37944A422CD1B831E70DF66BA3B8430593944D", // "type": "ed25519" diff --git a/state/state.go b/state/state.go index 1374fcf46..3a8b98d08 100644 --- a/state/state.go +++ b/state/state.go @@ -347,7 +347,7 @@ func MakeGenesisState(db dbm.DB, genDoc *types.GenesisDoc) *State { validators[i] = &types.Validator{ Address: address, PubKey: pubKey, - VotingPower: val.Amount, + VotingPower: val.Power, } } diff --git a/test/p2p/data/mach1/core/genesis.json b/test/p2p/data/mach1/core/genesis.json index 522f9831a..8d5c6c7b9 100644 --- a/test/p2p/data/mach1/core/genesis.json +++ b/test/p2p/data/mach1/core/genesis.json @@ -4,7 +4,7 @@ "genesis_time": "2016-06-24T20:01:19.322Z", "validators": [ { - "amount": 1, + "power": 1, "name": "mach1", "pub_key": { "type": "ed25519", @@ -12,7 +12,7 @@ } }, { - "amount": 1, + "power": 1, "name": "mach2", "pub_key": { "type": "ed25519", @@ -20,7 +20,7 @@ } }, { - "amount": 1, + "power": 1, "name": "mach3", "pub_key": { "type": "ed25519", @@ -28,7 +28,7 @@ } }, { - "amount": 1, + "power": 1, "name": "mach4", "pub_key": { "type": "ed25519", diff --git a/test/p2p/data/mach2/core/genesis.json b/test/p2p/data/mach2/core/genesis.json index 522f9831a..8d5c6c7b9 100644 --- a/test/p2p/data/mach2/core/genesis.json +++ b/test/p2p/data/mach2/core/genesis.json @@ -4,7 +4,7 @@ "genesis_time": "2016-06-24T20:01:19.322Z", "validators": [ { - "amount": 1, + "power": 1, "name": "mach1", "pub_key": { "type": "ed25519", @@ -12,7 +12,7 @@ } }, { - "amount": 1, + "power": 1, "name": "mach2", "pub_key": { "type": "ed25519", @@ -20,7 +20,7 @@ } }, { - "amount": 1, + "power": 1, "name": "mach3", "pub_key": { "type": "ed25519", @@ -28,7 +28,7 @@ } }, { - "amount": 1, + "power": 1, "name": "mach4", "pub_key": { "type": "ed25519", diff --git a/test/p2p/data/mach3/core/genesis.json b/test/p2p/data/mach3/core/genesis.json index 522f9831a..8d5c6c7b9 100644 --- a/test/p2p/data/mach3/core/genesis.json +++ b/test/p2p/data/mach3/core/genesis.json @@ -4,7 +4,7 @@ "genesis_time": "2016-06-24T20:01:19.322Z", "validators": [ { - "amount": 1, + "power": 1, "name": "mach1", "pub_key": { "type": "ed25519", @@ -12,7 +12,7 @@ } }, { - "amount": 1, + "power": 1, "name": "mach2", "pub_key": { "type": "ed25519", @@ -20,7 +20,7 @@ } }, { - "amount": 1, + "power": 1, "name": "mach3", "pub_key": { "type": "ed25519", @@ -28,7 +28,7 @@ } }, { - "amount": 1, + "power": 1, "name": "mach4", "pub_key": { "type": "ed25519", diff --git a/test/p2p/data/mach4/core/genesis.json b/test/p2p/data/mach4/core/genesis.json index 522f9831a..8d5c6c7b9 100644 --- a/test/p2p/data/mach4/core/genesis.json +++ b/test/p2p/data/mach4/core/genesis.json @@ -4,7 +4,7 @@ "genesis_time": "2016-06-24T20:01:19.322Z", "validators": [ { - "amount": 1, + "power": 1, "name": "mach1", "pub_key": { "type": "ed25519", @@ -12,7 +12,7 @@ } }, { - "amount": 1, + "power": 1, "name": "mach2", "pub_key": { "type": "ed25519", @@ -20,7 +20,7 @@ } }, { - "amount": 1, + "power": 1, "name": "mach3", "pub_key": { "type": "ed25519", @@ -28,7 +28,7 @@ } }, { - "amount": 1, + "power": 1, "name": "mach4", "pub_key": { "type": "ed25519", diff --git a/types/genesis.go b/types/genesis.go index 06b3d124c..e57787c42 100644 --- a/types/genesis.go +++ b/types/genesis.go @@ -18,7 +18,7 @@ import ( // GenesisValidator is an initial validator. type GenesisValidator struct { PubKey crypto.PubKey `json:"pub_key"` - Amount int64 `json:"amount"` + Power int64 `json:"power"` Name string `json:"name"` } @@ -44,7 +44,7 @@ func (genDoc *GenesisDoc) SaveAs(file string) error { func (genDoc *GenesisDoc) ValidatorHash() []byte { vals := make([]*Validator, len(genDoc.Validators)) for i, v := range genDoc.Validators { - vals[i] = NewValidator(v.PubKey, v.Amount) + vals[i] = NewValidator(v.PubKey, v.Power) } vset := NewValidatorSet(vals) return vset.Hash() @@ -71,6 +71,12 @@ func (genDoc *GenesisDoc) ValidateAndComplete() error { return errors.Errorf("The genesis file must have at least one validator") } + for _, v := range genDoc.Validators { + if v.Power == 0 { + return errors.Errorf("The genesis file cannot contain validators with no voting power: %v", v) + } + } + if genDoc.GenesisTime.IsZero() { genDoc.GenesisTime = time.Now() } diff --git a/types/genesis_test.go b/types/genesis_test.go index d06a70dab..fe997b690 100644 --- a/types/genesis_test.go +++ b/types/genesis_test.go @@ -20,7 +20,7 @@ func TestGenesis(t *testing.T) { []byte(`{"chain_id": "mychain", "validators": [{}]`), // missing validators []byte(`{"validators":[{"pub_key": {"type":"ed25519","data":"961EAB8752E51A03618502F55C2B6E09C38C65635C64CCF3173ED452CF86C957"}, - "amount":10,"name":""}]}`), // missing chain_id + "power":10,"name":""}]}`), // missing chain_id } for _, testCase := range testCases { @@ -29,7 +29,7 @@ func TestGenesis(t *testing.T) { } // test a good one by raw json - genDocBytes := []byte(`{"genesis_time":"0001-01-01T00:00:00Z","chain_id":"test-chain-QDKdJr","consensus_params":null,"validators":[{"pub_key":{"type":"ed25519","data":"961EAB8752E51A03618502F55C2B6E09C38C65635C64CCF3173ED452CF86C957"},"amount":10,"name":""}],"app_hash":""}`) + genDocBytes := []byte(`{"genesis_time":"0001-01-01T00:00:00Z","chain_id":"test-chain-QDKdJr","consensus_params":null,"validators":[{"pub_key":{"type":"ed25519","data":"961EAB8752E51A03618502F55C2B6E09C38C65635C64CCF3173ED452CF86C957"},"power":10,"name":""}],"app_hash":""}`) _, err := GenesisDocFromJSON(genDocBytes) assert.NoError(t, err, "expected no error for good genDoc json") From 1f0985689dbc4a0f29f1ee297d37f08df6042a28 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 21 Sep 2017 15:22:58 -0400 Subject: [PATCH 61/92] ConsensusParams ptr in GenesisDoc for json --- state/state.go | 2 +- types/genesis.go | 5 ++--- types/params.go | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/state/state.go b/state/state.go index 3a8b98d08..2ebbec10e 100644 --- a/state/state.go +++ b/state/state.go @@ -266,7 +266,7 @@ func (s *State) GetValidators() (*types.ValidatorSet, *types.ValidatorSet) { func (s *State) Params() types.ConsensusParams { // TODO: this should move into the State proper // when we allow the app to change it - return s.GenesisDoc.ConsensusParams + return *s.GenesisDoc.ConsensusParams } //------------------------------------------------------------------------ diff --git a/types/genesis.go b/types/genesis.go index e57787c42..f1b2736f8 100644 --- a/types/genesis.go +++ b/types/genesis.go @@ -26,7 +26,7 @@ type GenesisValidator struct { type GenesisDoc struct { GenesisTime time.Time `json:"genesis_time"` ChainID string `json:"chain_id"` - ConsensusParams ConsensusParams `json:"consensus_params"` + ConsensusParams *ConsensusParams `json:"consensus_params,omitempty"` Validators []GenesisValidator `json:"validators"` AppHash data.Bytes `json:"app_hash"` } @@ -58,8 +58,7 @@ func (genDoc *GenesisDoc) ValidateAndComplete() error { return errors.Errorf("Genesis doc must include non-empty chain_id") } - var emptyParams ConsensusParams - if genDoc.ConsensusParams == emptyParams { + if genDoc.ConsensusParams == nil { genDoc.ConsensusParams = DefaultConsensusParams() } else { if err := genDoc.ConsensusParams.Validate(); err != nil { diff --git a/types/params.go b/types/params.go index cd3242192..495d1fd48 100644 --- a/types/params.go +++ b/types/params.go @@ -35,8 +35,8 @@ type BlockGossipParams struct { } // DefaultConsensusParams returns a default ConsensusParams. -func DefaultConsensusParams() ConsensusParams { - return ConsensusParams{ +func DefaultConsensusParams() *ConsensusParams { + return &ConsensusParams{ DefaultBlockSizeParams(), DefaultTxSizeParams(), DefaultBlockGossipParams(), From b50339e8e754ef5dfac751730a77a16f807de47f Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 21 Sep 2017 15:33:22 -0400 Subject: [PATCH 62/92] p2p: sw.AddPeer -> sw.addPeer --- p2p/switch.go | 12 ++++++------ p2p/switch_test.go | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/p2p/switch.go b/p2p/switch.go index d1d615398..9ede8c108 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -209,11 +209,11 @@ func (sw *Switch) OnStop() { } } -// AddPeer checks the given peer's validity, performs a handshake, and adds the peer to the switch +// addPeer checks the given peer's validity, performs a handshake, and adds the peer to the switch // and to all registered reactors. // NOTE: This performs a blocking handshake before the peer is added. // CONTRACT: If error is returned, peer is nil, and conn is immediately closed. -func (sw *Switch) AddPeer(peer *peer) error { +func (sw *Switch) addPeer(peer *peer) error { if err := sw.FilterConnByAddr(peer.Addr()); err != nil { return err @@ -336,7 +336,7 @@ func (sw *Switch) dialSeed(addr *NetAddress) { } } -// DialPeerWithAddress dials the given peer and runs sw.AddPeer if it connects successfully. +// DialPeerWithAddress dials the given peer and runs sw.addPeer if it connects successfully. // If `persistent == true`, the switch will always try to reconnect to this peer if the connection ever fails. func (sw *Switch) DialPeerWithAddress(addr *NetAddress, persistent bool) (Peer, error) { sw.dialing.Set(addr.IP.String(), addr) @@ -352,7 +352,7 @@ func (sw *Switch) DialPeerWithAddress(addr *NetAddress, persistent bool) (Peer, if persistent { peer.makePersistent() } - err = sw.AddPeer(peer) + err = sw.addPeer(peer) if err != nil { sw.Logger.Error("Failed to add peer", "address", addr, "err", err) peer.CloseConn() @@ -582,7 +582,7 @@ func (sw *Switch) addPeerWithConnection(conn net.Conn) error { return err } peer.SetLogger(sw.Logger.With("peer", conn.RemoteAddr())) - if err = sw.AddPeer(peer); err != nil { + if err = sw.addPeer(peer); err != nil { conn.Close() return err } @@ -597,7 +597,7 @@ func (sw *Switch) addPeerWithConnectionAndConfig(conn net.Conn, config *PeerConf return err } peer.SetLogger(sw.Logger.With("peer", conn.RemoteAddr())) - if err = sw.AddPeer(peer); err != nil { + if err = sw.addPeer(peer); err != nil { conn.Close() return err } diff --git a/p2p/switch_test.go b/p2p/switch_test.go index d2435dfff..115811b04 100644 --- a/p2p/switch_test.go +++ b/p2p/switch_test.go @@ -246,7 +246,7 @@ func TestSwitchStopsNonPersistentPeerOnError(t *testing.T) { peer, err := newOutboundPeer(rp.Addr(), sw.reactorsByCh, sw.chDescs, sw.StopPeerForError, sw.nodePrivKey, DefaultPeerConfig()) require.Nil(err) - err = sw.AddPeer(peer) + err = sw.addPeer(peer) require.Nil(err) // simulate failure by closing connection @@ -273,7 +273,7 @@ func TestSwitchReconnectsToPersistentPeer(t *testing.T) { peer, err := newOutboundPeer(rp.Addr(), sw.reactorsByCh, sw.chDescs, sw.StopPeerForError, sw.nodePrivKey, DefaultPeerConfig()) peer.makePersistent() require.Nil(err) - err = sw.AddPeer(peer) + err = sw.addPeer(peer) require.Nil(err) // simulate failure by closing connection From 2c129447fdadd42d397611b68941b5073f985094 Mon Sep 17 00:00:00 2001 From: Adrian Brink Date: Wed, 30 Aug 2017 22:21:32 +0200 Subject: [PATCH 63/92] Example that showcases how to build your own tendermint node This example shows how a user of the tendermint library can build their own node and supply it with its own commands. It includes two todos in order to make it easier for library users to use tendermint. --- Makefile | 4 + cmd/hsm/commands/run_node.go | 107 ++++++++++++++++++ cmd/hsm/main.go | 35 ++++++ cmd/tendermint/commands/gen_validator.go | 6 +- cmd/tendermint/commands/init.go | 6 +- cmd/tendermint/commands/probe_upnp.go | 6 +- cmd/tendermint/commands/replay.go | 9 +- .../commands/reset_priv_validator.go | 9 +- cmd/tendermint/commands/run_node.go | 5 +- cmd/tendermint/commands/show_validator.go | 6 +- cmd/tendermint/commands/testnet.go | 8 +- cmd/tendermint/commands/version.go | 8 +- 12 files changed, 161 insertions(+), 48 deletions(-) create mode 100644 cmd/hsm/commands/run_node.go create mode 100644 cmd/hsm/main.go diff --git a/Makefile b/Makefile index 8c9c5214d..328e65f8a 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,10 @@ build: go build \ --ldflags "-X github.com/tendermint/tendermint/version.GitCommit=`git rev-parse HEAD`" -o build/tendermint ./cmd/tendermint/ +build_hsm: + go build \ + --ldflags "-X github.com/tendermint/tendermint/version.GitCommit=`git rev-parse HEAD`" -o build/tendermint ./cmd/hsm/ + build_race: go build -race -o build/tendermint ./cmd/tendermint diff --git a/cmd/hsm/commands/run_node.go b/cmd/hsm/commands/run_node.go new file mode 100644 index 000000000..94a1c548e --- /dev/null +++ b/cmd/hsm/commands/run_node.go @@ -0,0 +1,107 @@ +package commands + +import ( + "fmt" + "os" + "time" + + "github.com/spf13/cobra" + + cfg "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/node" + "github.com/tendermint/tendermint/proxy" + "github.com/tendermint/tendermint/types" + + cmn "github.com/tendermint/tmlibs/common" + "github.com/tendermint/tmlibs/log" +) + +var ( + config = cfg.DefaultConfig() + logger = log.NewTMLogger(log.NewSyncWriter(os.Stdout)).With("module", "main") +) + +var RunNodeCmd = &cobra.Command{ + Use: "node", + Short: "Run the tendermint node", + RunE: runNode, +} + +func init() { + AddNodeFlags(RunNodeCmd) +} + +// AddNodeFlags exposes some common configuration options on the command-line +// These are exposed for convenience of commands embedding a tendermint node +func AddNodeFlags(cmd *cobra.Command) { + // bind flags + cmd.Flags().String("moniker", config.Moniker, "Node Name") + + // node flags + cmd.Flags().Bool("fast_sync", config.FastSync, "Fast blockchain syncing") + + // abci flags + cmd.Flags().String("proxy_app", config.ProxyApp, "Proxy app address, or 'nilapp' or 'dummy' for local testing.") + cmd.Flags().String("abci", config.ABCI, "Specify abci transport (socket | grpc)") + + // rpc flags + cmd.Flags().String("rpc.laddr", config.RPC.ListenAddress, "RPC listen address. Port required") + cmd.Flags().String("rpc.grpc_laddr", config.RPC.GRPCListenAddress, "GRPC listen address (BroadcastTx only). Port required") + cmd.Flags().Bool("rpc.unsafe", config.RPC.Unsafe, "Enabled unsafe rpc methods") + + // p2p flags + cmd.Flags().String("p2p.laddr", config.P2P.ListenAddress, "Node listen address. (0.0.0.0:0 means any interface, any port)") + cmd.Flags().String("p2p.seeds", config.P2P.Seeds, "Comma delimited host:port seed nodes") + cmd.Flags().Bool("p2p.skip_upnp", config.P2P.SkipUPNP, "Skip UPNP configuration") + cmd.Flags().Bool("p2p.pex", config.P2P.PexReactor, "Enable Peer-Exchange (dev feature)") + + // consensus flags + cmd.Flags().Bool("consensus.create_empty_blocks", config.Consensus.CreateEmptyBlocks, "Set this to false to only produce blocks when there are txs or when the AppHash changes") +} + +// Users wishing to: +// * Use an external signer for their validators +// * Supply an in-proc abci app +// should import github.com/tendermint/tendermint/node and implement +// their own run_node to call node.NewNode (instead of node.NewNodeDefault) +// with their custom priv validator and/or custom proxy.ClientCreator +func runNode(cmd *cobra.Command, args []string) error { + + // Wait until the genesis doc becomes available + // This is for Mintnet compatibility. + // TODO: If Mintnet gets deprecated or genesis_file is + // always available, remove. + genDocFile := config.GenesisFile() + for !cmn.FileExists(genDocFile) { + logger.Info(cmn.Fmt("Waiting for genesis file %v...", genDocFile)) + time.Sleep(time.Second) + } + + genDoc, err := types.GenesisDocFromFile(genDocFile) + if err != nil { + return err + } + config.ChainID = genDoc.ChainID + + // Create & start node + // n := node.NewNodeDefault(config, logger.With("module", "node")) + + // TODO: Make types.PrivValidator an interface so that it can be provided + // by a hardware wallet or any other wallet provider. + + // The next two lines show how a private validator is setup. + privValidator := types.LoadOrGenPrivValidator(config.PrivValidatorFile(), logger) + privValidator.SetSigner(types.NewDefaultSigner(privValidator.PrivKey)) + + n := node.NewNode(config, privValidator, proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()), logger) + if _, err := n.Start(); err != nil { + return fmt.Errorf("Failed to start node: %v", err) + } else { + logger.Info("Started node", "nodeInfo", n.Switch().NodeInfo()) + } + + // Trap signal, run forever. + n.RunForever() + + return nil +} diff --git a/cmd/hsm/main.go b/cmd/hsm/main.go new file mode 100644 index 000000000..743a8fcd9 --- /dev/null +++ b/cmd/hsm/main.go @@ -0,0 +1,35 @@ +package main + +import ( + "os" + + tc "github.com/tendermint/tendermint/cmd/tendermint/commands" + "github.com/tendermint/tmlibs/cli" + + "github.com/tendermint/tendermint/cmd/hsm/commands" +) + +func main() { + // TODO: Make it easier to build a tendermint instance from scratch. + // All commands should be exported and it should be easy to override + // certain aspects of a single command. + // Probably every command should have a constructor that allows a user + // to vary the configuration. This is at least true for run_node.go + + rootCmd := tc.RootCmd + rootCmd.AddCommand(tc.GenValidatorCmd) + rootCmd.AddCommand(tc.InitFilesCmd) + rootCmd.AddCommand(tc.ProbeUpnpCmd) + rootCmd.AddCommand(tc.ReplayCmd) + rootCmd.AddCommand(tc.ReplayConsoleCmd) + rootCmd.AddCommand(tc.ResetAllCmd) + rootCmd.AddCommand(tc.ResetPrivValidatorCmd) + rootCmd.AddCommand(tc.ShowValidatorCmd) + rootCmd.AddCommand(tc.TestnetFilesCmd) + rootCmd.AddCommand(tc.VersionCmd) + + rootCmd.AddCommand(commands.RunNodeCmd) + + cmd := cli.PrepareBaseCmd(rootCmd, "TM", os.ExpandEnv("$HOME/.tendermint")) + cmd.Execute() +} diff --git a/cmd/tendermint/commands/gen_validator.go b/cmd/tendermint/commands/gen_validator.go index 97c583c22..df82554d8 100644 --- a/cmd/tendermint/commands/gen_validator.go +++ b/cmd/tendermint/commands/gen_validator.go @@ -9,16 +9,12 @@ import ( "github.com/tendermint/tendermint/types" ) -var genValidatorCmd = &cobra.Command{ +var GenValidatorCmd = &cobra.Command{ Use: "gen_validator", Short: "Generate new validator keypair", Run: genValidator, } -func init() { - RootCmd.AddCommand(genValidatorCmd) -} - func genValidator(cmd *cobra.Command, args []string) { privValidator := types.GenPrivValidator() privValidatorJSONBytes, _ := json.MarshalIndent(privValidator, "", "\t") diff --git a/cmd/tendermint/commands/init.go b/cmd/tendermint/commands/init.go index ce900defa..b844386c5 100644 --- a/cmd/tendermint/commands/init.go +++ b/cmd/tendermint/commands/init.go @@ -9,16 +9,12 @@ import ( cmn "github.com/tendermint/tmlibs/common" ) -var initFilesCmd = &cobra.Command{ +var InitFilesCmd = &cobra.Command{ Use: "init", Short: "Initialize Tendermint", Run: initFiles, } -func init() { - RootCmd.AddCommand(initFilesCmd) -} - func initFiles(cmd *cobra.Command, args []string) { privValFile := config.PrivValidatorFile() if _, err := os.Stat(privValFile); os.IsNotExist(err) { diff --git a/cmd/tendermint/commands/probe_upnp.go b/cmd/tendermint/commands/probe_upnp.go index e23e48973..a4c71d59e 100644 --- a/cmd/tendermint/commands/probe_upnp.go +++ b/cmd/tendermint/commands/probe_upnp.go @@ -9,16 +9,12 @@ import ( "github.com/tendermint/tendermint/p2p/upnp" ) -var probeUpnpCmd = &cobra.Command{ +var ProbeUpnpCmd = &cobra.Command{ Use: "probe_upnp", Short: "Test UPnP functionality", RunE: probeUpnp, } -func init() { - RootCmd.AddCommand(probeUpnpCmd) -} - func probeUpnp(cmd *cobra.Command, args []string) error { capabilities, err := upnp.Probe(logger) if err != nil { diff --git a/cmd/tendermint/commands/replay.go b/cmd/tendermint/commands/replay.go index 0c88b2443..422d44159 100644 --- a/cmd/tendermint/commands/replay.go +++ b/cmd/tendermint/commands/replay.go @@ -6,7 +6,7 @@ import ( "github.com/tendermint/tendermint/consensus" ) -var replayCmd = &cobra.Command{ +var ReplayCmd = &cobra.Command{ Use: "replay", Short: "Replay messages from WAL", Run: func(cmd *cobra.Command, args []string) { @@ -14,15 +14,10 @@ var replayCmd = &cobra.Command{ }, } -var replayConsoleCmd = &cobra.Command{ +var ReplayConsoleCmd = &cobra.Command{ Use: "replay_console", Short: "Replay messages from WAL in a console", Run: func(cmd *cobra.Command, args []string) { consensus.RunReplayFile(config.BaseConfig, config.Consensus, true) }, } - -func init() { - RootCmd.AddCommand(replayCmd) - RootCmd.AddCommand(replayConsoleCmd) -} diff --git a/cmd/tendermint/commands/reset_priv_validator.go b/cmd/tendermint/commands/reset_priv_validator.go index fa12be4ea..389e5b79b 100644 --- a/cmd/tendermint/commands/reset_priv_validator.go +++ b/cmd/tendermint/commands/reset_priv_validator.go @@ -9,23 +9,18 @@ import ( "github.com/tendermint/tmlibs/log" ) -var resetAllCmd = &cobra.Command{ +var ResetAllCmd = &cobra.Command{ Use: "unsafe_reset_all", Short: "(unsafe) Remove all the data and WAL, reset this node's validator", Run: resetAll, } -var resetPrivValidatorCmd = &cobra.Command{ +var ResetPrivValidatorCmd = &cobra.Command{ Use: "unsafe_reset_priv_validator", Short: "(unsafe) Reset this node's validator", Run: resetPrivValidator, } -func init() { - RootCmd.AddCommand(resetAllCmd) - RootCmd.AddCommand(resetPrivValidatorCmd) -} - // XXX: this is totally unsafe. // it's only suitable for testnets. func resetAll(cmd *cobra.Command, args []string) { diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index 6740df289..413e2cc7a 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -9,15 +9,14 @@ import ( "github.com/tendermint/tendermint/types" ) -var runNodeCmd = &cobra.Command{ +var RunNodeCmd = &cobra.Command{ Use: "node", Short: "Run the tendermint node", RunE: runNode, } func init() { - AddNodeFlags(runNodeCmd) - RootCmd.AddCommand(runNodeCmd) + AddNodeFlags(RunNodeCmd) } // AddNodeFlags exposes some common configuration options on the command-line diff --git a/cmd/tendermint/commands/show_validator.go b/cmd/tendermint/commands/show_validator.go index 53a687c6d..21cc90270 100644 --- a/cmd/tendermint/commands/show_validator.go +++ b/cmd/tendermint/commands/show_validator.go @@ -9,16 +9,12 @@ import ( "github.com/tendermint/tendermint/types" ) -var showValidatorCmd = &cobra.Command{ +var ShowValidatorCmd = &cobra.Command{ Use: "show_validator", Short: "Show this node's validator info", Run: showValidator, } -func init() { - RootCmd.AddCommand(showValidatorCmd) -} - func showValidator(cmd *cobra.Command, args []string) { privValidator := types.LoadOrGenPrivValidator(config.PrivValidatorFile(), logger) pubKeyJSONBytes, _ := data.ToJSON(privValidator.PubKey) diff --git a/cmd/tendermint/commands/testnet.go b/cmd/tendermint/commands/testnet.go index 315f3b498..75e016c9d 100644 --- a/cmd/tendermint/commands/testnet.go +++ b/cmd/tendermint/commands/testnet.go @@ -11,7 +11,7 @@ import ( cmn "github.com/tendermint/tmlibs/common" ) -var testnetFilesCmd = &cobra.Command{ +var TestnetFilesCmd = &cobra.Command{ Use: "testnet", Short: "Initialize files for a Tendermint testnet", Run: testnetFiles, @@ -24,12 +24,10 @@ var ( ) func init() { - testnetFilesCmd.Flags().IntVar(&nValidators, "n", 4, + TestnetFilesCmd.Flags().IntVar(&nValidators, "n", 4, "Number of validators to initialize the testnet with") - testnetFilesCmd.Flags().StringVar(&dataDir, "dir", "mytestnet", + TestnetFilesCmd.Flags().StringVar(&dataDir, "dir", "mytestnet", "Directory to store initialization data for the testnet") - - RootCmd.AddCommand(testnetFilesCmd) } func testnetFiles(cmd *cobra.Command, args []string) { diff --git a/cmd/tendermint/commands/version.go b/cmd/tendermint/commands/version.go index 5c92160ea..692cba4ad 100644 --- a/cmd/tendermint/commands/version.go +++ b/cmd/tendermint/commands/version.go @@ -8,14 +8,10 @@ import ( "github.com/tendermint/tendermint/version" ) -var versionCmd = &cobra.Command{ +var VersionCmd = &cobra.Command{ Use: "version", - Short: "Show version info", + Short: "Show version infoooooooo", Run: func(cmd *cobra.Command, args []string) { fmt.Println(version.Version) }, } - -func init() { - RootCmd.AddCommand(versionCmd) -} From 83f7d5c95ad2fca459ae605265eba37869eae6e9 Mon Sep 17 00:00:00 2001 From: Adrian Brink Date: Thu, 31 Aug 2017 10:34:45 +0200 Subject: [PATCH 64/92] Setup custom tendermint node By exporting all of the commands, we allow users to setup their own tendermint node cli. This enables users to provide a different pivValidator without the need to fork tendermint. --- cmd/hsm/main.go | 15 ++++++++-- cmd/tendermint/commands/run_node.go | 43 +++++++++++++++++++++++++++++ types/priv_validator.go | 10 +++++-- 3 files changed, 62 insertions(+), 6 deletions(-) diff --git a/cmd/hsm/main.go b/cmd/hsm/main.go index 743a8fcd9..657c946a1 100644 --- a/cmd/hsm/main.go +++ b/cmd/hsm/main.go @@ -3,10 +3,17 @@ package main import ( "os" - tc "github.com/tendermint/tendermint/cmd/tendermint/commands" "github.com/tendermint/tmlibs/cli" + "github.com/tendermint/tmlibs/log" + + tc "github.com/tendermint/tendermint/cmd/tendermint/commands" + cfg "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/types" +) - "github.com/tendermint/tendermint/cmd/hsm/commands" +var ( + config = cfg.DefaultConfig() + logger = log.NewTMLogger(log.NewSyncWriter(os.Stdout)).With("module", "main") ) func main() { @@ -28,7 +35,9 @@ func main() { rootCmd.AddCommand(tc.TestnetFilesCmd) rootCmd.AddCommand(tc.VersionCmd) - rootCmd.AddCommand(commands.RunNodeCmd) + privValidator := types.LoadOrGenPrivValidator(config.PrivValidatorFile(), logger) + privValidator.SetSigner(types.NewDefaultSigner(privValidator.PrivKey)) + rootCmd.AddCommand(tc.NewRunNodeCmd(privValidator)) cmd := cli.PrepareBaseCmd(rootCmd, "TM", os.ExpandEnv("$HOME/.tendermint")) cmd.Execute() diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index 413e2cc7a..51b370300 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -5,16 +5,59 @@ import ( "github.com/spf13/cobra" + cmn "github.com/tendermint/tmlibs/common" + "github.com/tendermint/tendermint/node" + "github.com/tendermint/tendermint/proxy" "github.com/tendermint/tendermint/types" ) +// RunNodeCmd creates and starts a tendermint node. var RunNodeCmd = &cobra.Command{ Use: "node", Short: "Run the tendermint node", RunE: runNode, } +// NewRunNodeCmd creates and starts a tendermint node. It allows the user to +// use a custom PrivValidator. +func NewRunNodeCmd(privVal *types.PrivValidator) *cobra.Command { + return &cobra.Command{ + Use: "node", + Short: "Run the tendermint node", + RunE: func(cmd *cobra.Command, args []string) error { + // Wait until the genesis doc becomes available + // This is for Mintnet compatibility. + // TODO: If Mintnet gets deprecated or genesis_file is + // always available, remove. + genDocFile := config.GenesisFile() + for !cmn.FileExists(genDocFile) { + logger.Info(cmn.Fmt("Waiting for genesis file %v...", genDocFile)) + time.Sleep(time.Second) + } + + genDoc, err := types.GenesisDocFromFile(genDocFile) + if err != nil { + return err + } + config.ChainID = genDoc.ChainID + + // Create & start node + n := node.NewNode(config, privVal, proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()), logger) + if _, err := n.Start(); err != nil { + return fmt.Errorf("Failed to start node: %v", err) + } else { + logger.Info("Started node", "nodeInfo", n.Switch().NodeInfo()) + } + + // Trap signal, run forever. + n.RunForever() + + return nil + }, + } +} + func init() { AddNodeFlags(RunNodeCmd) } diff --git a/types/priv_validator.go b/types/priv_validator.go index 3e84e7f36..072b394ca 100644 --- a/types/priv_validator.go +++ b/types/priv_validator.go @@ -35,6 +35,7 @@ func voteToStep(vote *Vote) int8 { } } +// PrivValidator implements the functionality for signing blocks. type PrivValidator struct { Address data.Bytes `json:"address"` PubKey crypto.PubKey `json:"pub_key"` @@ -58,26 +59,29 @@ type PrivValidator struct { // It is the caller's duty to verify the msg before calling Sign, // eg. to avoid double signing. // Currently, the only callers are SignVote and SignProposal +// Signer is an interface that describes how to sign votes. type Signer interface { PubKey() crypto.PubKey Sign(msg []byte) (crypto.Signature, error) } -// Implements Signer +// DefaultSigner implements Signer. type DefaultSigner struct { priv crypto.PrivKey } +// NewDefaultSigner returns an instance of DefaultSigner. func NewDefaultSigner(priv crypto.PrivKey) *DefaultSigner { return &DefaultSigner{priv: priv} } -// Implements Signer +// Sign implements Signer. It signs the byte slice with a private key. func (ds *DefaultSigner) Sign(msg []byte) (crypto.Signature, error) { return ds.priv.Sign(msg), nil } -// Implements Signer +// PubKey implements Signer. It should return the public key that corresponds +// to the private key used for signing. func (ds *DefaultSigner) PubKey() crypto.PubKey { return ds.priv.PubKey() } From 2ccc3326ec14bce6df4e5448f97936e215a80738 Mon Sep 17 00:00:00 2001 From: Adrian Brink Date: Thu, 31 Aug 2017 10:42:10 +0200 Subject: [PATCH 65/92] Remove redundant file --- cmd/hsm/commands/run_node.go | 107 ----------------------------------- 1 file changed, 107 deletions(-) delete mode 100644 cmd/hsm/commands/run_node.go diff --git a/cmd/hsm/commands/run_node.go b/cmd/hsm/commands/run_node.go deleted file mode 100644 index 94a1c548e..000000000 --- a/cmd/hsm/commands/run_node.go +++ /dev/null @@ -1,107 +0,0 @@ -package commands - -import ( - "fmt" - "os" - "time" - - "github.com/spf13/cobra" - - cfg "github.com/tendermint/tendermint/config" - "github.com/tendermint/tendermint/node" - "github.com/tendermint/tendermint/proxy" - "github.com/tendermint/tendermint/types" - - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/log" -) - -var ( - config = cfg.DefaultConfig() - logger = log.NewTMLogger(log.NewSyncWriter(os.Stdout)).With("module", "main") -) - -var RunNodeCmd = &cobra.Command{ - Use: "node", - Short: "Run the tendermint node", - RunE: runNode, -} - -func init() { - AddNodeFlags(RunNodeCmd) -} - -// AddNodeFlags exposes some common configuration options on the command-line -// These are exposed for convenience of commands embedding a tendermint node -func AddNodeFlags(cmd *cobra.Command) { - // bind flags - cmd.Flags().String("moniker", config.Moniker, "Node Name") - - // node flags - cmd.Flags().Bool("fast_sync", config.FastSync, "Fast blockchain syncing") - - // abci flags - cmd.Flags().String("proxy_app", config.ProxyApp, "Proxy app address, or 'nilapp' or 'dummy' for local testing.") - cmd.Flags().String("abci", config.ABCI, "Specify abci transport (socket | grpc)") - - // rpc flags - cmd.Flags().String("rpc.laddr", config.RPC.ListenAddress, "RPC listen address. Port required") - cmd.Flags().String("rpc.grpc_laddr", config.RPC.GRPCListenAddress, "GRPC listen address (BroadcastTx only). Port required") - cmd.Flags().Bool("rpc.unsafe", config.RPC.Unsafe, "Enabled unsafe rpc methods") - - // p2p flags - cmd.Flags().String("p2p.laddr", config.P2P.ListenAddress, "Node listen address. (0.0.0.0:0 means any interface, any port)") - cmd.Flags().String("p2p.seeds", config.P2P.Seeds, "Comma delimited host:port seed nodes") - cmd.Flags().Bool("p2p.skip_upnp", config.P2P.SkipUPNP, "Skip UPNP configuration") - cmd.Flags().Bool("p2p.pex", config.P2P.PexReactor, "Enable Peer-Exchange (dev feature)") - - // consensus flags - cmd.Flags().Bool("consensus.create_empty_blocks", config.Consensus.CreateEmptyBlocks, "Set this to false to only produce blocks when there are txs or when the AppHash changes") -} - -// Users wishing to: -// * Use an external signer for their validators -// * Supply an in-proc abci app -// should import github.com/tendermint/tendermint/node and implement -// their own run_node to call node.NewNode (instead of node.NewNodeDefault) -// with their custom priv validator and/or custom proxy.ClientCreator -func runNode(cmd *cobra.Command, args []string) error { - - // Wait until the genesis doc becomes available - // This is for Mintnet compatibility. - // TODO: If Mintnet gets deprecated or genesis_file is - // always available, remove. - genDocFile := config.GenesisFile() - for !cmn.FileExists(genDocFile) { - logger.Info(cmn.Fmt("Waiting for genesis file %v...", genDocFile)) - time.Sleep(time.Second) - } - - genDoc, err := types.GenesisDocFromFile(genDocFile) - if err != nil { - return err - } - config.ChainID = genDoc.ChainID - - // Create & start node - // n := node.NewNodeDefault(config, logger.With("module", "node")) - - // TODO: Make types.PrivValidator an interface so that it can be provided - // by a hardware wallet or any other wallet provider. - - // The next two lines show how a private validator is setup. - privValidator := types.LoadOrGenPrivValidator(config.PrivValidatorFile(), logger) - privValidator.SetSigner(types.NewDefaultSigner(privValidator.PrivKey)) - - n := node.NewNode(config, privValidator, proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()), logger) - if _, err := n.Start(); err != nil { - return fmt.Errorf("Failed to start node: %v", err) - } else { - logger.Info("Started node", "nodeInfo", n.Switch().NodeInfo()) - } - - // Trap signal, run forever. - n.RunForever() - - return nil -} From bf5e95608756bc59f65442e4d9a654dd586f7598 Mon Sep 17 00:00:00 2001 From: Adrian Brink Date: Thu, 31 Aug 2017 10:43:11 +0200 Subject: [PATCH 66/92] Change capitalisation --- cmd/tendermint/commands/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/tendermint/commands/root.go b/cmd/tendermint/commands/root.go index 6cef5daa9..4488eae61 100644 --- a/cmd/tendermint/commands/root.go +++ b/cmd/tendermint/commands/root.go @@ -38,7 +38,7 @@ var RootCmd = &cobra.Command{ Use: "tendermint", Short: "Tendermint Core (BFT Consensus) in Go", PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) { - if cmd.Name() == versionCmd.Name() { + if cmd.Name() == VersionCmd.Name() { return nil } config, err = ParseConfig() From 7e4a704bd1f986c5b12b795a8948164a030ae5c5 Mon Sep 17 00:00:00 2001 From: Duncan Jones Date: Fri, 1 Sep 2017 10:45:39 +0100 Subject: [PATCH 67/92] Remove reliance on default Signer This change allows the default privValidator to use a custom Signer implementation with no reliance on the default Signer implementation. --- cmd/hsm/main.go | 8 ++++++-- types/priv_validator.go | 10 +++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/cmd/hsm/main.go b/cmd/hsm/main.go index 657c946a1..d2144745c 100644 --- a/cmd/hsm/main.go +++ b/cmd/hsm/main.go @@ -35,8 +35,12 @@ func main() { rootCmd.AddCommand(tc.TestnetFilesCmd) rootCmd.AddCommand(tc.VersionCmd) - privValidator := types.LoadOrGenPrivValidator(config.PrivValidatorFile(), logger) - privValidator.SetSigner(types.NewDefaultSigner(privValidator.PrivKey)) + // Override with HSM implementation, otherwise nil will trigger default + // software signer: + var signer types.Signer = nil + + privValidator := types.LoadPrivValidatorWithSigner(config.PrivValidatorFile(), + signer) rootCmd.AddCommand(tc.NewRunNodeCmd(privValidator)) cmd := cli.PrepareBaseCmd(rootCmd, "TM", os.ExpandEnv("$HOME/.tendermint")) diff --git a/types/priv_validator.go b/types/priv_validator.go index 072b394ca..fe2f2e730 100644 --- a/types/priv_validator.go +++ b/types/priv_validator.go @@ -112,6 +112,10 @@ func GenPrivValidator() *PrivValidator { } func LoadPrivValidator(filePath string) *PrivValidator { + return LoadPrivValidatorWithSigner(filePath, nil) +} + +func LoadPrivValidatorWithSigner(filePath string, signer Signer) *PrivValidator { privValJSONBytes, err := ioutil.ReadFile(filePath) if err != nil { Exit(err.Error()) @@ -123,7 +127,11 @@ func LoadPrivValidator(filePath string) *PrivValidator { } privVal.filePath = filePath - privVal.Signer = NewDefaultSigner(privVal.PrivKey) + if signer == nil { + privVal.Signer = NewDefaultSigner(privVal.PrivKey) + } else { + privVal.Signer = signer + } privVal.setPubKeyAndAddress() return &privVal } From 0d392a04425f62f7ed82fa7061807fcf320f3b28 Mon Sep 17 00:00:00 2001 From: Duncan Jones Date: Mon, 11 Sep 2017 18:56:41 +0100 Subject: [PATCH 68/92] Allow Signer to be generated with priv key Prior to this change, a custom Signer would have no knowledge of the private key stored in the configuration file. This changes introduces a generator function, which creates a Signer based on the private key. This provides an opportunity for customer Signers to adjust behaviour based on the key contents. (E.g. imagine key contents are a key label, rather than the key itself). --- cmd/hsm/main.go | 19 +++++++++---------- types/priv_validator.go | 17 ++++++++++------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/cmd/hsm/main.go b/cmd/hsm/main.go index d2144745c..7c12f03c8 100644 --- a/cmd/hsm/main.go +++ b/cmd/hsm/main.go @@ -1,14 +1,13 @@ package main import ( - "os" - - "github.com/tendermint/tmlibs/cli" - "github.com/tendermint/tmlibs/log" - + tcrypto "github.com/tendermint/go-crypto" tc "github.com/tendermint/tendermint/cmd/tendermint/commands" cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/types" + "github.com/tendermint/tmlibs/cli" + "github.com/tendermint/tmlibs/log" + "os" ) var ( @@ -35,12 +34,12 @@ func main() { rootCmd.AddCommand(tc.TestnetFilesCmd) rootCmd.AddCommand(tc.VersionCmd) - // Override with HSM implementation, otherwise nil will trigger default - // software signer: - var signer types.Signer = nil + signerGenerator := func(pk tcrypto.PrivKey) types.Signer { + // Return your own signer implementation here + return types.NewDefaultSigner(pk) + } - privValidator := types.LoadPrivValidatorWithSigner(config.PrivValidatorFile(), - signer) + privValidator := types.LoadPrivValidatorWithSigner(config.PrivValidatorFile(), signerGenerator) rootCmd.AddCommand(tc.NewRunNodeCmd(privValidator)) cmd := cli.PrepareBaseCmd(rootCmd, "TM", os.ExpandEnv("$HOME/.tendermint")) diff --git a/types/priv_validator.go b/types/priv_validator.go index fe2f2e730..9dfd95b5f 100644 --- a/types/priv_validator.go +++ b/types/priv_validator.go @@ -55,6 +55,10 @@ type PrivValidator struct { mtx sync.Mutex } + +type SignerGenerator func(pk crypto.PrivKey) (Signer) + + // This is used to sign votes. // It is the caller's duty to verify the msg before calling Sign, // eg. to avoid double signing. @@ -112,10 +116,12 @@ func GenPrivValidator() *PrivValidator { } func LoadPrivValidator(filePath string) *PrivValidator { - return LoadPrivValidatorWithSigner(filePath, nil) + return LoadPrivValidatorWithSigner(filePath, func(pk crypto.PrivKey) Signer { + return NewDefaultSigner(pk) + }) } -func LoadPrivValidatorWithSigner(filePath string, signer Signer) *PrivValidator { +func LoadPrivValidatorWithSigner(filePath string, generator SignerGenerator) *PrivValidator { privValJSONBytes, err := ioutil.ReadFile(filePath) if err != nil { Exit(err.Error()) @@ -127,11 +133,8 @@ func LoadPrivValidatorWithSigner(filePath string, signer Signer) *PrivValidator } privVal.filePath = filePath - if signer == nil { - privVal.Signer = NewDefaultSigner(privVal.PrivKey) - } else { - privVal.Signer = signer - } + privVal.Signer = generator(privVal.PrivKey) + privVal.setPubKeyAndAddress() return &privVal } From 7dd3c007c7c965987b7e5a635ff5345400b5dad5 Mon Sep 17 00:00:00 2001 From: Adrian Brink Date: Wed, 13 Sep 2017 00:18:07 +0200 Subject: [PATCH 69/92] Refactor priv_validator Users can now just pass an object that implements the Signer interface. --- cmd/hsm/main.go | 47 ------- cmd/tendermint/commands/gen_validator.go | 2 + cmd/tendermint/commands/init.go | 1 + cmd/tendermint/commands/probe_upnp.go | 1 + cmd/tendermint/commands/replay.go | 3 + .../commands/reset_priv_validator.go | 18 ++- cmd/tendermint/commands/root.go | 1 + cmd/tendermint/commands/run_node.go | 75 ++++++----- cmd/tendermint/commands/show_validator.go | 3 +- cmd/tendermint/commands/testnet.go | 14 +- cmd/tendermint/commands/version.go | 1 + cmd/tendermint/main.go | 22 +++- consensus/common_test.go | 2 +- node/node.go | 2 +- rpc/test/helpers.go | 2 +- types/priv_validator.go | 123 +++++++++--------- 16 files changed, 152 insertions(+), 165 deletions(-) delete mode 100644 cmd/hsm/main.go diff --git a/cmd/hsm/main.go b/cmd/hsm/main.go deleted file mode 100644 index 7c12f03c8..000000000 --- a/cmd/hsm/main.go +++ /dev/null @@ -1,47 +0,0 @@ -package main - -import ( - tcrypto "github.com/tendermint/go-crypto" - tc "github.com/tendermint/tendermint/cmd/tendermint/commands" - cfg "github.com/tendermint/tendermint/config" - "github.com/tendermint/tendermint/types" - "github.com/tendermint/tmlibs/cli" - "github.com/tendermint/tmlibs/log" - "os" -) - -var ( - config = cfg.DefaultConfig() - logger = log.NewTMLogger(log.NewSyncWriter(os.Stdout)).With("module", "main") -) - -func main() { - // TODO: Make it easier to build a tendermint instance from scratch. - // All commands should be exported and it should be easy to override - // certain aspects of a single command. - // Probably every command should have a constructor that allows a user - // to vary the configuration. This is at least true for run_node.go - - rootCmd := tc.RootCmd - rootCmd.AddCommand(tc.GenValidatorCmd) - rootCmd.AddCommand(tc.InitFilesCmd) - rootCmd.AddCommand(tc.ProbeUpnpCmd) - rootCmd.AddCommand(tc.ReplayCmd) - rootCmd.AddCommand(tc.ReplayConsoleCmd) - rootCmd.AddCommand(tc.ResetAllCmd) - rootCmd.AddCommand(tc.ResetPrivValidatorCmd) - rootCmd.AddCommand(tc.ShowValidatorCmd) - rootCmd.AddCommand(tc.TestnetFilesCmd) - rootCmd.AddCommand(tc.VersionCmd) - - signerGenerator := func(pk tcrypto.PrivKey) types.Signer { - // Return your own signer implementation here - return types.NewDefaultSigner(pk) - } - - privValidator := types.LoadPrivValidatorWithSigner(config.PrivValidatorFile(), signerGenerator) - rootCmd.AddCommand(tc.NewRunNodeCmd(privValidator)) - - cmd := cli.PrepareBaseCmd(rootCmd, "TM", os.ExpandEnv("$HOME/.tendermint")) - cmd.Execute() -} diff --git a/cmd/tendermint/commands/gen_validator.go b/cmd/tendermint/commands/gen_validator.go index df82554d8..e4dfb0bd1 100644 --- a/cmd/tendermint/commands/gen_validator.go +++ b/cmd/tendermint/commands/gen_validator.go @@ -9,6 +9,8 @@ import ( "github.com/tendermint/tendermint/types" ) +// GenValidatorCmd allows the generation of a keypair for a +// validator. var GenValidatorCmd = &cobra.Command{ Use: "gen_validator", Short: "Generate new validator keypair", diff --git a/cmd/tendermint/commands/init.go b/cmd/tendermint/commands/init.go index b844386c5..5e973335d 100644 --- a/cmd/tendermint/commands/init.go +++ b/cmd/tendermint/commands/init.go @@ -9,6 +9,7 @@ import ( cmn "github.com/tendermint/tmlibs/common" ) +// InitFilesCmd initialises a fresh Tendermint Core instance. var InitFilesCmd = &cobra.Command{ Use: "init", Short: "Initialize Tendermint", diff --git a/cmd/tendermint/commands/probe_upnp.go b/cmd/tendermint/commands/probe_upnp.go index a4c71d59e..643b7713f 100644 --- a/cmd/tendermint/commands/probe_upnp.go +++ b/cmd/tendermint/commands/probe_upnp.go @@ -9,6 +9,7 @@ import ( "github.com/tendermint/tendermint/p2p/upnp" ) +// ProbeUpnpCmd adds capabilities to test the UPnP functionality. var ProbeUpnpCmd = &cobra.Command{ Use: "probe_upnp", Short: "Test UPnP functionality", diff --git a/cmd/tendermint/commands/replay.go b/cmd/tendermint/commands/replay.go index 422d44159..303ccba6b 100644 --- a/cmd/tendermint/commands/replay.go +++ b/cmd/tendermint/commands/replay.go @@ -6,6 +6,7 @@ import ( "github.com/tendermint/tendermint/consensus" ) +// ReplayCmd allows replaying of messages from the WAL. var ReplayCmd = &cobra.Command{ Use: "replay", Short: "Replay messages from WAL", @@ -14,6 +15,8 @@ var ReplayCmd = &cobra.Command{ }, } +// ReplayConsoleCmd allows replaying of messages from the WAL in a +// console. var ReplayConsoleCmd = &cobra.Command{ Use: "replay_console", Short: "Replay messages from WAL in a console", diff --git a/cmd/tendermint/commands/reset_priv_validator.go b/cmd/tendermint/commands/reset_priv_validator.go index 389e5b79b..7f01c2313 100644 --- a/cmd/tendermint/commands/reset_priv_validator.go +++ b/cmd/tendermint/commands/reset_priv_validator.go @@ -9,18 +9,29 @@ import ( "github.com/tendermint/tmlibs/log" ) +// ResetAllCmd removes the database of this Tendermint core +// instance. var ResetAllCmd = &cobra.Command{ Use: "unsafe_reset_all", Short: "(unsafe) Remove all the data and WAL, reset this node's validator", Run: resetAll, } +// ResetPrivValidatorCmd resets the private validator files. var ResetPrivValidatorCmd = &cobra.Command{ Use: "unsafe_reset_priv_validator", Short: "(unsafe) Reset this node's validator", Run: resetPrivValidator, } +// ResetAll removes the privValidator files. +// Exported so other CLI tools can use it +func ResetAll(dbDir, privValFile string, logger log.Logger) { + resetPrivValidatorLocal(privValFile, logger) + os.RemoveAll(dbDir) + logger.Info("Removed all data", "dir", dbDir) +} + // XXX: this is totally unsafe. // it's only suitable for testnets. func resetAll(cmd *cobra.Command, args []string) { @@ -33,13 +44,6 @@ func resetPrivValidator(cmd *cobra.Command, args []string) { resetPrivValidatorLocal(config.PrivValidatorFile(), logger) } -// Exported so other CLI tools can use it -func ResetAll(dbDir, privValFile string, logger log.Logger) { - resetPrivValidatorLocal(privValFile, logger) - os.RemoveAll(dbDir) - logger.Info("Removed all data", "dir", dbDir) -} - func resetPrivValidatorLocal(privValFile string, logger log.Logger) { // Get PrivValidator var privValidator *types.PrivValidator diff --git a/cmd/tendermint/commands/root.go b/cmd/tendermint/commands/root.go index 4488eae61..a54b50069 100644 --- a/cmd/tendermint/commands/root.go +++ b/cmd/tendermint/commands/root.go @@ -34,6 +34,7 @@ func ParseConfig() (*cfg.Config, error) { return conf, err } +// RootCmd is the root command for Tendermint core. var RootCmd = &cobra.Command{ Use: "tendermint", Short: "Tendermint Core (BFT Consensus) in Go", diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index 51b370300..fd33faefe 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -12,6 +12,38 @@ import ( "github.com/tendermint/tendermint/types" ) +func init() { + AddNodeFlags(RunNodeCmd) +} + +// AddNodeFlags exposes some common configuration options on the command-line +// These are exposed for convenience of commands embedding a tendermint node +func AddNodeFlags(cmd *cobra.Command) { + // bind flags + cmd.Flags().String("moniker", config.Moniker, "Node Name") + + // node flags + cmd.Flags().Bool("fast_sync", config.FastSync, "Fast blockchain syncing") + + // abci flags + cmd.Flags().String("proxy_app", config.ProxyApp, "Proxy app address, or 'nilapp' or 'dummy' for local testing.") + cmd.Flags().String("abci", config.ABCI, "Specify abci transport (socket | grpc)") + + // rpc flags + cmd.Flags().String("rpc.laddr", config.RPC.ListenAddress, "RPC listen address. Port required") + cmd.Flags().String("rpc.grpc_laddr", config.RPC.GRPCListenAddress, "GRPC listen address (BroadcastTx only). Port required") + cmd.Flags().Bool("rpc.unsafe", config.RPC.Unsafe, "Enabled unsafe rpc methods") + + // p2p flags + cmd.Flags().String("p2p.laddr", config.P2P.ListenAddress, "Node listen address. (0.0.0.0:0 means any interface, any port)") + cmd.Flags().String("p2p.seeds", config.P2P.Seeds, "Comma delimited host:port seed nodes") + cmd.Flags().Bool("p2p.skip_upnp", config.P2P.SkipUPNP, "Skip UPNP configuration") + cmd.Flags().Bool("p2p.pex", config.P2P.PexReactor, "Enable Peer-Exchange (dev feature)") + + // consensus flags + cmd.Flags().Bool("consensus.create_empty_blocks", config.Consensus.CreateEmptyBlocks, "Set this to false to only produce blocks when there are txs or when the AppHash changes") +} + // RunNodeCmd creates and starts a tendermint node. var RunNodeCmd = &cobra.Command{ Use: "node", @@ -19,8 +51,8 @@ var RunNodeCmd = &cobra.Command{ RunE: runNode, } -// NewRunNodeCmd creates and starts a tendermint node. It allows the user to -// use a custom PrivValidator. +// NewRunNodeCmd returns the command that allows the CLI to start a +// node. It can be used with a custom PrivValidator. func NewRunNodeCmd(privVal *types.PrivValidator) *cobra.Command { return &cobra.Command{ Use: "node", @@ -43,7 +75,12 @@ func NewRunNodeCmd(privVal *types.PrivValidator) *cobra.Command { config.ChainID = genDoc.ChainID // Create & start node - n := node.NewNode(config, privVal, proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()), logger) + var n *node.Node + if privVal == nil { + n = node.NewNodeDefault(config, logger.With("module", "node")) + } + n = node.NewNode(config, privVal, proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()), logger) + if _, err := n.Start(); err != nil { return fmt.Errorf("Failed to start node: %v", err) } else { @@ -58,38 +95,6 @@ func NewRunNodeCmd(privVal *types.PrivValidator) *cobra.Command { } } -func init() { - AddNodeFlags(RunNodeCmd) -} - -// AddNodeFlags exposes some common configuration options on the command-line -// These are exposed for convenience of commands embedding a tendermint node -func AddNodeFlags(cmd *cobra.Command) { - // bind flags - cmd.Flags().String("moniker", config.Moniker, "Node Name") - - // node flags - cmd.Flags().Bool("fast_sync", config.FastSync, "Fast blockchain syncing") - - // abci flags - cmd.Flags().String("proxy_app", config.ProxyApp, "Proxy app address, or 'nilapp' or 'dummy' for local testing.") - cmd.Flags().String("abci", config.ABCI, "Specify abci transport (socket | grpc)") - - // rpc flags - cmd.Flags().String("rpc.laddr", config.RPC.ListenAddress, "RPC listen address. Port required") - cmd.Flags().String("rpc.grpc_laddr", config.RPC.GRPCListenAddress, "GRPC listen address (BroadcastTx only). Port required") - cmd.Flags().Bool("rpc.unsafe", config.RPC.Unsafe, "Enabled unsafe rpc methods") - - // p2p flags - cmd.Flags().String("p2p.laddr", config.P2P.ListenAddress, "Node listen address. (0.0.0.0:0 means any interface, any port)") - cmd.Flags().String("p2p.seeds", config.P2P.Seeds, "Comma delimited host:port seed nodes") - cmd.Flags().Bool("p2p.skip_upnp", config.P2P.SkipUPNP, "Skip UPNP configuration") - cmd.Flags().Bool("p2p.pex", config.P2P.PexReactor, "Enable Peer-Exchange (dev feature)") - - // consensus flags - cmd.Flags().Bool("consensus.create_empty_blocks", config.Consensus.CreateEmptyBlocks, "Set this to false to only produce blocks when there are txs or when the AppHash changes") -} - // Users wishing to: // * Use an external signer for their validators // * Supply an in-proc abci app diff --git a/cmd/tendermint/commands/show_validator.go b/cmd/tendermint/commands/show_validator.go index 21cc90270..86a665f0c 100644 --- a/cmd/tendermint/commands/show_validator.go +++ b/cmd/tendermint/commands/show_validator.go @@ -9,6 +9,7 @@ import ( "github.com/tendermint/tendermint/types" ) +// ShowValidatorCmd adds capabilities for showing the validator info. var ShowValidatorCmd = &cobra.Command{ Use: "show_validator", Short: "Show this node's validator info", @@ -16,7 +17,7 @@ var ShowValidatorCmd = &cobra.Command{ } func showValidator(cmd *cobra.Command, args []string) { - privValidator := types.LoadOrGenPrivValidator(config.PrivValidatorFile(), logger) + privValidator := types.LoadOrGenPrivValidator(config.PrivValidatorFile()) pubKeyJSONBytes, _ := data.ToJSON(privValidator.PubKey) fmt.Println(string(pubKeyJSONBytes)) } diff --git a/cmd/tendermint/commands/testnet.go b/cmd/tendermint/commands/testnet.go index 75e016c9d..617b66597 100644 --- a/cmd/tendermint/commands/testnet.go +++ b/cmd/tendermint/commands/testnet.go @@ -11,12 +11,6 @@ import ( cmn "github.com/tendermint/tmlibs/common" ) -var TestnetFilesCmd = &cobra.Command{ - Use: "testnet", - Short: "Initialize files for a Tendermint testnet", - Run: testnetFiles, -} - //flags var ( nValidators int @@ -30,6 +24,14 @@ func init() { "Directory to store initialization data for the testnet") } +// TestnetFilesCmd allows initialisation of files for a +// Tendermint testnet. +var TestnetFilesCmd = &cobra.Command{ + Use: "testnet", + Short: "Initialize files for a Tendermint testnet", + Run: testnetFiles, +} + func testnetFiles(cmd *cobra.Command, args []string) { genVals := make([]types.GenesisValidator, nValidators) diff --git a/cmd/tendermint/commands/version.go b/cmd/tendermint/commands/version.go index 692cba4ad..b212adc70 100644 --- a/cmd/tendermint/commands/version.go +++ b/cmd/tendermint/commands/version.go @@ -8,6 +8,7 @@ import ( "github.com/tendermint/tendermint/version" ) +// VersionCmd ... var VersionCmd = &cobra.Command{ Use: "version", Short: "Show version infoooooooo", diff --git a/cmd/tendermint/main.go b/cmd/tendermint/main.go index 5493e4f2f..d70df634d 100644 --- a/cmd/tendermint/main.go +++ b/cmd/tendermint/main.go @@ -3,11 +3,29 @@ package main import ( "os" - "github.com/tendermint/tendermint/cmd/tendermint/commands" + // crypto "github.com/tendermint/go-crypto" + "github.com/tendermint/tmlibs/cli" + + . "github.com/tendermint/tendermint/cmd/tendermint/commands" + // "github.com/tendermint/tendermint/types" ) func main() { - cmd := cli.PrepareBaseCmd(commands.RootCmd, "TM", os.ExpandEnv("$HOME/.tendermint")) + rootCmd := RootCmd + rootCmd.AddCommand(GenValidatorCmd, InitFilesCmd, ProbeUpnpCmd, + ReplayCmd, ReplayConsoleCmd, ResetAllCmd, ResetPrivValidatorCmd, + ShowValidatorCmd, TestnetFilesCmd, VersionCmd) + + // NOTE: Implement your own type that implements the Signer interface + // and then instantiate it here. + // signer := types.NewDefaultSigner(pk) + // privValidator := types.LoadPrivValidatorWithSigner(signer) + // rootCmd.AddCommand(NewRunNodeCmd(privValidator)) + + // Create & start node + rootCmd.AddCommand(RunNodeCmd) + + cmd := cli.PrepareBaseCmd(rootCmd, "TM", os.ExpandEnv("$HOME/.tendermint")) cmd.Execute() } diff --git a/consensus/common_test.go b/consensus/common_test.go index 6a05b74e3..cde56a3db 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -261,7 +261,7 @@ func newConsensusStateWithConfig(thisConfig *cfg.Config, state *sm.State, pv *ty func loadPrivValidator(config *cfg.Config) *types.PrivValidator { privValidatorFile := config.PrivValidatorFile() ensureDir(path.Dir(privValidatorFile), 0700) - privValidator := types.LoadOrGenPrivValidator(privValidatorFile, log.TestingLogger()) + privValidator := types.LoadOrGenPrivValidator(privValidatorFile) privValidator.Reset() return privValidator } diff --git a/node/node.go b/node/node.go index e0ddeb5d4..e831ba1d0 100644 --- a/node/node.go +++ b/node/node.go @@ -60,7 +60,7 @@ type Node struct { func NewNodeDefault(config *cfg.Config, logger log.Logger) *Node { // Get PrivValidator - privValidator := types.LoadOrGenPrivValidator(config.PrivValidatorFile(), logger) + privValidator := types.LoadOrGenPrivValidator(config.PrivValidatorFile()) return NewNode(config, privValidator, proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()), logger) } diff --git a/rpc/test/helpers.go b/rpc/test/helpers.go index 14da15c94..132c4d4d9 100644 --- a/rpc/test/helpers.go +++ b/rpc/test/helpers.go @@ -79,7 +79,7 @@ func NewTendermint(app abci.Application) *nm.Node { logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout)) logger = log.NewFilter(logger, log.AllowError()) privValidatorFile := config.PrivValidatorFile() - privValidator := types.LoadOrGenPrivValidator(privValidatorFile, logger) + privValidator := types.LoadOrGenPrivValidator(privValidatorFile) papp := proxy.NewLocalClientCreator(app) node := nm.NewNode(config, privValidator, papp, logger) return node diff --git a/types/priv_validator.go b/types/priv_validator.go index 9dfd95b5f..ed0cde070 100644 --- a/types/priv_validator.go +++ b/types/priv_validator.go @@ -12,7 +12,6 @@ import ( crypto "github.com/tendermint/go-crypto" data "github.com/tendermint/go-wire/data" . "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/log" ) // TODO: type ? @@ -35,30 +34,6 @@ func voteToStep(vote *Vote) int8 { } } -// PrivValidator implements the functionality for signing blocks. -type PrivValidator struct { - Address data.Bytes `json:"address"` - PubKey crypto.PubKey `json:"pub_key"` - LastHeight int `json:"last_height"` - LastRound int `json:"last_round"` - LastStep int8 `json:"last_step"` - LastSignature crypto.Signature `json:"last_signature,omitempty"` // so we dont lose signatures - LastSignBytes data.Bytes `json:"last_signbytes,omitempty"` // so we dont lose signatures - - // PrivKey should be empty if a Signer other than the default is being used. - PrivKey crypto.PrivKey `json:"priv_key"` - Signer `json:"-"` - - // For persistence. - // Overloaded for testing. - filePath string - mtx sync.Mutex -} - - -type SignerGenerator func(pk crypto.PrivKey) (Signer) - - // This is used to sign votes. // It is the caller's duty to verify the msg before calling Sign, // eg. to avoid double signing. @@ -90,38 +65,39 @@ func (ds *DefaultSigner) PubKey() crypto.PubKey { return ds.priv.PubKey() } -func (privVal *PrivValidator) SetSigner(s Signer) { - privVal.Signer = s - privVal.setPubKeyAndAddress() -} +// PrivValidator implements the functionality for signing blocks. +type PrivValidator struct { + Address data.Bytes `json:"address"` + PubKey crypto.PubKey `json:"pub_key"` + LastHeight int `json:"last_height"` + LastRound int `json:"last_round"` + LastStep int8 `json:"last_step"` + LastSignature crypto.Signature `json:"last_signature,omitempty"` // so we dont lose signatures + LastSignBytes data.Bytes `json:"last_signbytes,omitempty"` // so we dont lose signatures -// Overwrite address and pubkey for convenience -func (privVal *PrivValidator) setPubKeyAndAddress() { - privVal.PubKey = privVal.Signer.PubKey() - privVal.Address = privVal.PubKey.Address() + // PrivKey should be empty if a Signer other than the default is being used. + PrivKey crypto.PrivKey `json:"priv_key"` + Signer `json:"-"` + + // For persistence. + // Overloaded for testing. + filePath string + mtx sync.Mutex } -// Generates a new validator with private key. -func GenPrivValidator() *PrivValidator { - privKey := crypto.GenPrivKeyEd25519().Wrap() - pubKey := privKey.PubKey() - return &PrivValidator{ - Address: pubKey.Address(), - PubKey: pubKey, - PrivKey: privKey, - LastStep: stepNone, - filePath: "", - Signer: NewDefaultSigner(privKey), +func LoadOrGenPrivValidator(filePath string) *PrivValidator { + var privValidator *PrivValidator + if _, err := os.Stat(filePath); err == nil { + privValidator = LoadPrivValidator(filePath) + } else { + privValidator = GenPrivValidator() + privValidator.SetFile(filePath) + privValidator.Save() } + return privValidator } func LoadPrivValidator(filePath string) *PrivValidator { - return LoadPrivValidatorWithSigner(filePath, func(pk crypto.PrivKey) Signer { - return NewDefaultSigner(pk) - }) -} - -func LoadPrivValidatorWithSigner(filePath string, generator SignerGenerator) *PrivValidator { privValJSONBytes, err := ioutil.ReadFile(filePath) if err != nil { Exit(err.Error()) @@ -133,25 +109,44 @@ func LoadPrivValidatorWithSigner(filePath string, generator SignerGenerator) *Pr } privVal.filePath = filePath - privVal.Signer = generator(privVal.PrivKey) - + privVal.Signer = NewDefaultSigner(privVal.PrivKey) privVal.setPubKeyAndAddress() return &privVal } -func LoadOrGenPrivValidator(filePath string, logger log.Logger) *PrivValidator { - var privValidator *PrivValidator - if _, err := os.Stat(filePath); err == nil { - privValidator = LoadPrivValidator(filePath) - logger.Info("Loaded PrivValidator", - "file", filePath, "privValidator", privValidator) - } else { - privValidator = GenPrivValidator() - privValidator.SetFile(filePath) - privValidator.Save() - logger.Info("Generated PrivValidator", "file", filePath) +// Generates a new validator with private key. +func GenPrivValidator() *PrivValidator { + privKey := crypto.GenPrivKeyEd25519().Wrap() + pubKey := privKey.PubKey() + return &PrivValidator{ + Address: pubKey.Address(), + PubKey: pubKey, + PrivKey: privKey, + LastStep: stepNone, + filePath: "", + Signer: NewDefaultSigner(privKey), } - return privValidator +} + +func LoadPrivValidatorWithSigner(signer Signer) *PrivValidator { + return &PrivValidator{ + Address: signer.PubKey().Address(), + PubKey: signer.PubKey(), + LastStep: stepNone, + filePath: "", + Signer: signer, + } +} + +func (privVal *PrivValidator) SetSigner(s Signer) { + privVal.Signer = s + privVal.setPubKeyAndAddress() +} + +// Overwrite address and pubkey for convenience +func (privVal *PrivValidator) setPubKeyAndAddress() { + privVal.PubKey = privVal.Signer.PubKey() + privVal.Address = privVal.PubKey.Address() } func (privVal *PrivValidator) SetFile(filePath string) { From 4e13a19339a033c405435b4180447d9f4cbbd147 Mon Sep 17 00:00:00 2001 From: Adrian Brink Date: Wed, 13 Sep 2017 08:32:53 +0200 Subject: [PATCH 70/92] Add ability to construct new instance of Tendermint core from scratch --- cmd/tendermint/commands/run_node.go | 46 +++++------------------------ cmd/tendermint/main.go | 10 ++++--- types/priv_validator.go | 8 +++-- 3 files changed, 19 insertions(+), 45 deletions(-) diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index fd33faefe..d52f2977c 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -2,6 +2,7 @@ package commands import ( "fmt" + "time" "github.com/spf13/cobra" @@ -12,10 +13,6 @@ import ( "github.com/tendermint/tendermint/types" ) -func init() { - AddNodeFlags(RunNodeCmd) -} - // AddNodeFlags exposes some common configuration options on the command-line // These are exposed for convenience of commands embedding a tendermint node func AddNodeFlags(cmd *cobra.Command) { @@ -44,24 +41,13 @@ func AddNodeFlags(cmd *cobra.Command) { cmd.Flags().Bool("consensus.create_empty_blocks", config.Consensus.CreateEmptyBlocks, "Set this to false to only produce blocks when there are txs or when the AppHash changes") } -// RunNodeCmd creates and starts a tendermint node. -var RunNodeCmd = &cobra.Command{ - Use: "node", - Short: "Run the tendermint node", - RunE: runNode, -} - // NewRunNodeCmd returns the command that allows the CLI to start a // node. It can be used with a custom PrivValidator. func NewRunNodeCmd(privVal *types.PrivValidator) *cobra.Command { - return &cobra.Command{ + cmd := &cobra.Command{ Use: "node", Short: "Run the tendermint node", RunE: func(cmd *cobra.Command, args []string) error { - // Wait until the genesis doc becomes available - // This is for Mintnet compatibility. - // TODO: If Mintnet gets deprecated or genesis_file is - // always available, remove. genDocFile := config.GenesisFile() for !cmn.FileExists(genDocFile) { logger.Info(cmn.Fmt("Waiting for genesis file %v...", genDocFile)) @@ -79,7 +65,8 @@ func NewRunNodeCmd(privVal *types.PrivValidator) *cobra.Command { if privVal == nil { n = node.NewNodeDefault(config, logger.With("module", "node")) } - n = node.NewNode(config, privVal, proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()), logger) + n = node.NewNode(config, privVal, proxy.DefaultClientCreator(config.ProxyApp, + config.ABCI, config.DBDir()), logger.With("module", "node")) if _, err := n.Start(); err != nil { return fmt.Errorf("Failed to start node: %v", err) @@ -93,6 +80,9 @@ func NewRunNodeCmd(privVal *types.PrivValidator) *cobra.Command { return nil }, } + + AddNodeFlags(cmd) + return cmd } // Users wishing to: @@ -101,25 +91,3 @@ func NewRunNodeCmd(privVal *types.PrivValidator) *cobra.Command { // should import github.com/tendermint/tendermint/node and implement // their own run_node to call node.NewNode (instead of node.NewNodeDefault) // with their custom priv validator and/or custom proxy.ClientCreator -func runNode(cmd *cobra.Command, args []string) error { - - genDocFile := config.GenesisFile() - genDoc, err := types.GenesisDocFromFile(genDocFile) - if err != nil { - return err - } - config.ChainID = genDoc.ChainID - - // Create & start node - n := node.NewNodeDefault(config, logger.With("module", "node")) - if _, err := n.Start(); err != nil { - return fmt.Errorf("Failed to start node: %v", err) - } else { - logger.Info("Started node", "nodeInfo", n.Switch().NodeInfo()) - } - - // Trap signal, run forever. - n.RunForever() - - return nil -} diff --git a/cmd/tendermint/main.go b/cmd/tendermint/main.go index d70df634d..e231f6c67 100644 --- a/cmd/tendermint/main.go +++ b/cmd/tendermint/main.go @@ -19,12 +19,14 @@ func main() { // NOTE: Implement your own type that implements the Signer interface // and then instantiate it here. - // signer := types.NewDefaultSigner(pk) - // privValidator := types.LoadPrivValidatorWithSigner(signer) - // rootCmd.AddCommand(NewRunNodeCmd(privValidator)) + /* + signer := types.NewDefaultSigner(pk) + privValidator := types.LoadPrivValidatorWithSigner(signer) + rootCmd.AddCommand(NewRunNodeCmd(privValidator)) + */ // Create & start node - rootCmd.AddCommand(RunNodeCmd) + rootCmd.AddCommand(NewRunNodeCmd(nil)) cmd := cli.PrepareBaseCmd(rootCmd, "TM", os.ExpandEnv("$HOME/.tendermint")) cmd.Execute() diff --git a/types/priv_validator.go b/types/priv_validator.go index ed0cde070..3944a9cdf 100644 --- a/types/priv_validator.go +++ b/types/priv_validator.go @@ -128,12 +128,16 @@ func GenPrivValidator() *PrivValidator { } } -func LoadPrivValidatorWithSigner(signer Signer) *PrivValidator { +// LoadPrivValidatorWithSigner instantiates a private validator with a custom +// signer object. Tendermint tracks state in the PrivValidator that might be +// saved to disk. Please supply a filepath where Tendermint can save the +// private validator. +func LoadPrivValidatorWithSigner(signer Signer, filePath string) *PrivValidator { return &PrivValidator{ Address: signer.PubKey().Address(), PubKey: signer.PubKey(), LastStep: stepNone, - filePath: "", + filePath: filePath, Signer: signer, } } From 66fcdf7c7a2f4cb8e0bb123412a218191d61d618 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 18 Sep 2017 17:11:35 -0400 Subject: [PATCH 71/92] minor fixes --- Makefile | 4 ---- cmd/tendermint/commands/run_node.go | 8 -------- cmd/tendermint/commands/version.go | 2 +- 3 files changed, 1 insertion(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 328e65f8a..8c9c5214d 100644 --- a/Makefile +++ b/Makefile @@ -17,10 +17,6 @@ build: go build \ --ldflags "-X github.com/tendermint/tendermint/version.GitCommit=`git rev-parse HEAD`" -o build/tendermint ./cmd/tendermint/ -build_hsm: - go build \ - --ldflags "-X github.com/tendermint/tendermint/version.GitCommit=`git rev-parse HEAD`" -o build/tendermint ./cmd/hsm/ - build_race: go build -race -o build/tendermint ./cmd/tendermint diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index d52f2977c..3eac5d981 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -2,12 +2,9 @@ package commands import ( "fmt" - "time" "github.com/spf13/cobra" - cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tendermint/node" "github.com/tendermint/tendermint/proxy" "github.com/tendermint/tendermint/types" @@ -49,11 +46,6 @@ func NewRunNodeCmd(privVal *types.PrivValidator) *cobra.Command { Short: "Run the tendermint node", RunE: func(cmd *cobra.Command, args []string) error { genDocFile := config.GenesisFile() - for !cmn.FileExists(genDocFile) { - logger.Info(cmn.Fmt("Waiting for genesis file %v...", genDocFile)) - time.Sleep(time.Second) - } - genDoc, err := types.GenesisDocFromFile(genDocFile) if err != nil { return err diff --git a/cmd/tendermint/commands/version.go b/cmd/tendermint/commands/version.go index b212adc70..f9f545e59 100644 --- a/cmd/tendermint/commands/version.go +++ b/cmd/tendermint/commands/version.go @@ -11,7 +11,7 @@ import ( // VersionCmd ... var VersionCmd = &cobra.Command{ Use: "version", - Short: "Show version infoooooooo", + Short: "Show version info", Run: func(cmd *cobra.Command, args []string) { fmt.Println(version.Version) }, From abe912c61095615a92f4c14056815734609cf718 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 18 Sep 2017 17:40:24 -0400 Subject: [PATCH 72/92] FuncSignerAndApp allows custom signer and abci app --- cmd/tendermint/commands/run_node.go | 30 +++++++++++----------- cmd/tendermint/main.go | 39 ++++++++++++++++------------- types/priv_validator.go | 6 ++--- 3 files changed, 40 insertions(+), 35 deletions(-) diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index 3eac5d981..b9d23a27a 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -5,6 +5,7 @@ import ( "github.com/spf13/cobra" + cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/node" "github.com/tendermint/tendermint/proxy" "github.com/tendermint/tendermint/types" @@ -38,9 +39,19 @@ func AddNodeFlags(cmd *cobra.Command) { cmd.Flags().Bool("consensus.create_empty_blocks", config.Consensus.CreateEmptyBlocks, "Set this to false to only produce blocks when there are txs or when the AppHash changes") } +// FuncSignerAndApp takes a config and returns a PrivValidator and ClientCreator. +// It allows other projects to make Tendermint binaries with custom signers and applications. +type FuncSignerAndApp func(*cfg.Config) (*types.PrivValidator, proxy.ClientCreator) + +func DefaultSignerAndApp(config *cfg.Config) (*types.PrivValidator, proxy.ClientCreator) { + privValidator := types.LoadOrGenPrivValidator(config.PrivValidatorFile()) + clientCreator := proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()) + return privValidator, clientCreator +} + // NewRunNodeCmd returns the command that allows the CLI to start a -// node. It can be used with a custom PrivValidator. -func NewRunNodeCmd(privVal *types.PrivValidator) *cobra.Command { +// node. It can be used with a custom PrivValidator and in-process ABCI application. +func NewRunNodeCmd(signerAndApp FuncSignerAndApp) *cobra.Command { cmd := &cobra.Command{ Use: "node", Short: "Run the tendermint node", @@ -53,12 +64,8 @@ func NewRunNodeCmd(privVal *types.PrivValidator) *cobra.Command { config.ChainID = genDoc.ChainID // Create & start node - var n *node.Node - if privVal == nil { - n = node.NewNodeDefault(config, logger.With("module", "node")) - } - n = node.NewNode(config, privVal, proxy.DefaultClientCreator(config.ProxyApp, - config.ABCI, config.DBDir()), logger.With("module", "node")) + privVal, clientCreator := signerAndApp(config) + n := node.NewNode(config, privVal, clientCreator, logger.With("module", "node")) if _, err := n.Start(); err != nil { return fmt.Errorf("Failed to start node: %v", err) @@ -76,10 +83,3 @@ func NewRunNodeCmd(privVal *types.PrivValidator) *cobra.Command { AddNodeFlags(cmd) return cmd } - -// Users wishing to: -// * Use an external signer for their validators -// * Supply an in-proc abci app -// should import github.com/tendermint/tendermint/node and implement -// their own run_node to call node.NewNode (instead of node.NewNodeDefault) -// with their custom priv validator and/or custom proxy.ClientCreator diff --git a/cmd/tendermint/main.go b/cmd/tendermint/main.go index e231f6c67..7cc1b2183 100644 --- a/cmd/tendermint/main.go +++ b/cmd/tendermint/main.go @@ -3,30 +3,35 @@ package main import ( "os" - // crypto "github.com/tendermint/go-crypto" - "github.com/tendermint/tmlibs/cli" - . "github.com/tendermint/tendermint/cmd/tendermint/commands" - // "github.com/tendermint/tendermint/types" + cmd "github.com/tendermint/tendermint/cmd/tendermint/commands" ) func main() { - rootCmd := RootCmd - rootCmd.AddCommand(GenValidatorCmd, InitFilesCmd, ProbeUpnpCmd, - ReplayCmd, ReplayConsoleCmd, ResetAllCmd, ResetPrivValidatorCmd, - ShowValidatorCmd, TestnetFilesCmd, VersionCmd) - - // NOTE: Implement your own type that implements the Signer interface - // and then instantiate it here. - /* - signer := types.NewDefaultSigner(pk) - privValidator := types.LoadPrivValidatorWithSigner(signer) - rootCmd.AddCommand(NewRunNodeCmd(privValidator)) - */ + rootCmd := cmd.RootCmd + rootCmd.AddCommand( + cmd.GenValidatorCmd, + cmd.InitFilesCmd, + cmd.ProbeUpnpCmd, + cmd.ReplayCmd, + cmd.ReplayConsoleCmd, + cmd.ResetAllCmd, + cmd.ResetPrivValidatorCmd, + cmd.ShowValidatorCmd, + cmd.TestnetFilesCmd, + cmd.VersionCmd) + + // NOTE: + // Users wishing to: + // * Use an external signer for their validators + // * Supply an in-proc abci app + // can copy this file and use something other than the + // default SignerAndApp function + signerAndApp := cmd.DefaultSignerAndApp // Create & start node - rootCmd.AddCommand(NewRunNodeCmd(nil)) + rootCmd.AddCommand(cmd.NewRunNodeCmd(signerAndApp)) cmd := cli.PrepareBaseCmd(rootCmd, "TM", os.ExpandEnv("$HOME/.tendermint")) cmd.Execute() diff --git a/types/priv_validator.go b/types/priv_validator.go index 3944a9cdf..72d05c49e 100644 --- a/types/priv_validator.go +++ b/types/priv_validator.go @@ -34,17 +34,17 @@ func voteToStep(vote *Vote) int8 { } } -// This is used to sign votes. +// Signer is an interface that defines how to sign votes. // It is the caller's duty to verify the msg before calling Sign, // eg. to avoid double signing. -// Currently, the only callers are SignVote and SignProposal -// Signer is an interface that describes how to sign votes. +// Currently, the only callers are SignVote and SignProposal. type Signer interface { PubKey() crypto.PubKey Sign(msg []byte) (crypto.Signature, error) } // DefaultSigner implements Signer. +// It uses a standard crypto.PrivKey. type DefaultSigner struct { priv crypto.PrivKey } From fd1b0b997a38e764345d8abc2b3c748c6bf9c688 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 18 Sep 2017 18:12:31 -0400 Subject: [PATCH 73/92] PrivValidator interface --- cmd/tendermint/commands/init.go | 2 +- .../commands/reset_priv_validator.go | 2 +- cmd/tendermint/commands/run_node.go | 4 +- cmd/tendermint/commands/testnet.go | 2 +- consensus/state.go | 24 +-- node/node.go | 14 +- types/priv_validator.go | 177 ++++++++++-------- types/priv_validator_test.go | 37 ++-- types/proposal_test.go | 6 +- types/validator.go | 4 +- types/validator_set.go | 4 +- types/vote_set_test.go | 60 +++--- 12 files changed, 177 insertions(+), 159 deletions(-) diff --git a/cmd/tendermint/commands/init.go b/cmd/tendermint/commands/init.go index 5e973335d..259645044 100644 --- a/cmd/tendermint/commands/init.go +++ b/cmd/tendermint/commands/init.go @@ -30,7 +30,7 @@ func initFiles(cmd *cobra.Command, args []string) { ChainID: cmn.Fmt("test-chain-%v", cmn.RandStr(6)), } genDoc.Validators = []types.GenesisValidator{types.GenesisValidator{ - PubKey: privValidator.PubKey, + PubKey: privValidator.PubKey(), Power: 10, }} diff --git a/cmd/tendermint/commands/reset_priv_validator.go b/cmd/tendermint/commands/reset_priv_validator.go index 7f01c2313..bedd38926 100644 --- a/cmd/tendermint/commands/reset_priv_validator.go +++ b/cmd/tendermint/commands/reset_priv_validator.go @@ -46,7 +46,7 @@ func resetPrivValidator(cmd *cobra.Command, args []string) { func resetPrivValidatorLocal(privValFile string, logger log.Logger) { // Get PrivValidator - var privValidator *types.PrivValidator + var privValidator types.PrivValidator if _, err := os.Stat(privValFile); err == nil { privValidator = types.LoadPrivValidator(privValFile) privValidator.Reset() diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index b9d23a27a..883ec6ec4 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -41,9 +41,9 @@ func AddNodeFlags(cmd *cobra.Command) { // FuncSignerAndApp takes a config and returns a PrivValidator and ClientCreator. // It allows other projects to make Tendermint binaries with custom signers and applications. -type FuncSignerAndApp func(*cfg.Config) (*types.PrivValidator, proxy.ClientCreator) +type FuncSignerAndApp func(*cfg.Config) (types.PrivValidator, proxy.ClientCreator) -func DefaultSignerAndApp(config *cfg.Config) (*types.PrivValidator, proxy.ClientCreator) { +func DefaultSignerAndApp(config *cfg.Config) (types.PrivValidator, proxy.ClientCreator) { privValidator := types.LoadOrGenPrivValidator(config.PrivValidatorFile()) clientCreator := proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()) return privValidator, clientCreator diff --git a/cmd/tendermint/commands/testnet.go b/cmd/tendermint/commands/testnet.go index 617b66597..9412379a5 100644 --- a/cmd/tendermint/commands/testnet.go +++ b/cmd/tendermint/commands/testnet.go @@ -47,7 +47,7 @@ func testnetFiles(cmd *cobra.Command, args []string) { privValFile := path.Join(dataDir, mach, "priv_validator.json") privVal := types.LoadPrivValidator(privValFile) genVals[i] = types.GenesisValidator{ - PubKey: privVal.PubKey, + PubKey: privVal.PubKey(), Power: 1, Name: mach, } diff --git a/consensus/state.go b/consensus/state.go index ac1656209..f141cad58 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -180,14 +180,6 @@ func (ti *timeoutInfo) String() string { return fmt.Sprintf("%v ; %d/%d %v", ti.Duration, ti.Height, ti.Round, ti.Step) } -// PrivValidator is a validator that can sign votes and proposals. -type PrivValidator interface { - GetAddress() []byte - SignVote(chainID string, vote *types.Vote) error - SignProposal(chainID string, proposal *types.Proposal) error - SignHeartbeat(chainID string, heartbeat *types.Heartbeat) error -} - // ConsensusState handles execution of the consensus algorithm. // It processes votes and proposals, and upon reaching agreement, // commits blocks to the chain and executes them against the application. @@ -197,7 +189,7 @@ type ConsensusState struct { // config details config *cfg.ConsensusConfig - privValidator PrivValidator // for signing votes + privValidator types.PrivValidator // for signing votes // services for creating and executing blocks proxyAppConn proxy.AppConnConsensus @@ -308,7 +300,7 @@ func (cs *ConsensusState) GetValidators() (int, []*types.Validator) { } // SetPrivValidator sets the private validator account for signing votes. -func (cs *ConsensusState) SetPrivValidator(priv PrivValidator) { +func (cs *ConsensusState) SetPrivValidator(priv types.PrivValidator) { cs.mtx.Lock() defer cs.mtx.Unlock() cs.privValidator = priv @@ -825,7 +817,7 @@ func (cs *ConsensusState) needProofBlock(height int) bool { func (cs *ConsensusState) proposalHeartbeat(height, round int) { counter := 0 - addr := cs.privValidator.GetAddress() + addr := cs.privValidator.Address() valIndex, v := cs.Validators.GetByAddress(addr) if v == nil { // not a validator @@ -886,7 +878,7 @@ func (cs *ConsensusState) enterPropose(height int, round int) { if !cs.isProposer() { cs.Logger.Info("enterPropose: Not our turn to propose", "proposer", cs.Validators.GetProposer().Address, "privValidator", cs.privValidator) - if cs.Validators.HasAddress(cs.privValidator.GetAddress()) { + if cs.Validators.HasAddress(cs.privValidator.Address()) { cs.Logger.Debug("This node is a validator") } else { cs.Logger.Debug("This node is not a validator") @@ -899,7 +891,7 @@ func (cs *ConsensusState) enterPropose(height int, round int) { } func (cs *ConsensusState) isProposer() bool { - return bytes.Equal(cs.Validators.GetProposer().Address, cs.privValidator.GetAddress()) + return bytes.Equal(cs.Validators.GetProposer().Address, cs.privValidator.Address()) } func (cs *ConsensusState) defaultDecideProposal(height, round int) { @@ -1444,7 +1436,7 @@ func (cs *ConsensusState) tryAddVote(vote *types.Vote, peerKey string) error { if err == ErrVoteHeightMismatch { return err } else if _, ok := err.(*types.ErrVoteConflictingVotes); ok { - if bytes.Equal(vote.ValidatorAddress, cs.privValidator.GetAddress()) { + if bytes.Equal(vote.ValidatorAddress, cs.privValidator.Address()) { cs.Logger.Error("Found conflicting vote from ourselves. Did you unsafe_reset a validator?", "height", vote.Height, "round", vote.Round, "type", vote.Type) return err } @@ -1573,7 +1565,7 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool, } func (cs *ConsensusState) signVote(type_ byte, hash []byte, header types.PartSetHeader) (*types.Vote, error) { - addr := cs.privValidator.GetAddress() + addr := cs.privValidator.Address() valIndex, _ := cs.Validators.GetByAddress(addr) vote := &types.Vote{ ValidatorAddress: addr, @@ -1590,7 +1582,7 @@ func (cs *ConsensusState) signVote(type_ byte, hash []byte, header types.PartSet // sign the vote and publish on internalMsgQueue func (cs *ConsensusState) signAddVote(type_ byte, hash []byte, header types.PartSetHeader) *types.Vote { // if we don't have a key or we're not in the validator set, do nothing - if cs.privValidator == nil || !cs.Validators.HasAddress(cs.privValidator.GetAddress()) { + if cs.privValidator == nil || !cs.Validators.HasAddress(cs.privValidator.Address()) { return nil } vote, err := cs.signVote(type_, hash, header) diff --git a/node/node.go b/node/node.go index e831ba1d0..a0a8c529a 100644 --- a/node/node.go +++ b/node/node.go @@ -38,8 +38,8 @@ type Node struct { // config config *cfg.Config - genesisDoc *types.GenesisDoc // initial validator set - privValidator *types.PrivValidator // local node's validator key + genesisDoc *types.GenesisDoc // initial validator set + privValidator types.PrivValidator // local node's validator key // network privKey crypto.PrivKeyEd25519 // local node's p2p key @@ -65,7 +65,7 @@ func NewNodeDefault(config *cfg.Config, logger log.Logger) *Node { proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()), logger) } -func NewNode(config *cfg.Config, privValidator *types.PrivValidator, clientCreator proxy.ClientCreator, logger log.Logger) *Node { +func NewNode(config *cfg.Config, privValidator types.PrivValidator, clientCreator proxy.ClientCreator, logger log.Logger) *Node { // Get BlockStore blockStoreDB := dbm.NewDB("blockstore", config.DBBackend, config.DBDir()) blockStore := bc.NewBlockStore(blockStoreDB) @@ -119,13 +119,13 @@ func NewNode(config *cfg.Config, privValidator *types.PrivValidator, clientCreat fastSync := config.FastSync if state.Validators.Size() == 1 { addr, _ := state.Validators.GetByIndex(0) - if bytes.Equal(privValidator.Address, addr) { + if bytes.Equal(privValidator.Address(), addr) { fastSync = false } } // Log whether this node is a validator or an observer - if state.Validators.HasAddress(privValidator.Address) { + if state.Validators.HasAddress(privValidator.PubKey().Address()) { consensusLogger.Info("This node is a validator") } else { consensusLogger.Info("This node is not a validator") @@ -314,7 +314,7 @@ func (n *Node) ConfigureRPC() { rpccore.SetConsensusState(n.consensusState) rpccore.SetMempool(n.mempoolReactor.Mempool) rpccore.SetSwitch(n.sw) - rpccore.SetPubKey(n.privValidator.PubKey) + rpccore.SetPubKey(n.privValidator.PubKey()) rpccore.SetGenesisDoc(n.genesisDoc) rpccore.SetAddrBook(n.addrBook) rpccore.SetProxyAppQuery(n.proxyApp.Query()) @@ -385,7 +385,7 @@ func (n *Node) EventSwitch() types.EventSwitch { } // XXX: for convenience -func (n *Node) PrivValidator() *types.PrivValidator { +func (n *Node) PrivValidator() types.PrivValidator { return n.privValidator } diff --git a/types/priv_validator.go b/types/priv_validator.go index 72d05c49e..e8fb3e240 100644 --- a/types/priv_validator.go +++ b/types/priv_validator.go @@ -46,27 +46,59 @@ type Signer interface { // DefaultSigner implements Signer. // It uses a standard crypto.PrivKey. type DefaultSigner struct { - priv crypto.PrivKey + PrivKey crypto.PrivKey `json:"priv_key"` } // NewDefaultSigner returns an instance of DefaultSigner. func NewDefaultSigner(priv crypto.PrivKey) *DefaultSigner { - return &DefaultSigner{priv: priv} + return &DefaultSigner{PrivKey: priv} } // Sign implements Signer. It signs the byte slice with a private key. func (ds *DefaultSigner) Sign(msg []byte) (crypto.Signature, error) { - return ds.priv.Sign(msg), nil + return ds.PrivKey.Sign(msg), nil } // PubKey implements Signer. It should return the public key that corresponds // to the private key used for signing. func (ds *DefaultSigner) PubKey() crypto.PubKey { - return ds.priv.PubKey() + return ds.PrivKey.PubKey() +} + +type PrivValidator interface { + Address() data.Bytes // redundant since .PubKey().Address() + PubKey() crypto.PubKey + + SignVote(chainID string, vote *Vote) error + SignProposal(chainID string, proposal *Proposal) error + SignHeartbeat(chainID string, heartbeat *Heartbeat) error + + Reset() + + SetFile(file string) + Save() +} + +// DefaultPrivValidator implements the functionality for signing blocks. +type DefaultPrivValidator struct { + Info PrivValidatorInfo `json:"info"` + Signer *DefaultSigner `json:"signer"` + + // For persistence. + // Overloaded for testing. + filePath string + mtx sync.Mutex +} + +func (pv *DefaultPrivValidator) Address() data.Bytes { + return pv.Info.Address } -// PrivValidator implements the functionality for signing blocks. -type PrivValidator struct { +func (pv *DefaultPrivValidator) PubKey() crypto.PubKey { + return pv.Info.PubKey +} + +type PrivValidatorInfo struct { Address data.Bytes `json:"address"` PubKey crypto.PubKey `json:"pub_key"` LastHeight int `json:"last_height"` @@ -74,19 +106,10 @@ type PrivValidator struct { LastStep int8 `json:"last_step"` LastSignature crypto.Signature `json:"last_signature,omitempty"` // so we dont lose signatures LastSignBytes data.Bytes `json:"last_signbytes,omitempty"` // so we dont lose signatures - - // PrivKey should be empty if a Signer other than the default is being used. - PrivKey crypto.PrivKey `json:"priv_key"` - Signer `json:"-"` - - // For persistence. - // Overloaded for testing. - filePath string - mtx sync.Mutex } -func LoadOrGenPrivValidator(filePath string) *PrivValidator { - var privValidator *PrivValidator +func LoadOrGenPrivValidator(filePath string) *DefaultPrivValidator { + var privValidator *DefaultPrivValidator if _, err := os.Stat(filePath); err == nil { privValidator = LoadPrivValidator(filePath) } else { @@ -97,34 +120,33 @@ func LoadOrGenPrivValidator(filePath string) *PrivValidator { return privValidator } -func LoadPrivValidator(filePath string) *PrivValidator { +func LoadPrivValidator(filePath string) *DefaultPrivValidator { privValJSONBytes, err := ioutil.ReadFile(filePath) if err != nil { Exit(err.Error()) } - privVal := PrivValidator{} + privVal := DefaultPrivValidator{} err = json.Unmarshal(privValJSONBytes, &privVal) if err != nil { Exit(Fmt("Error reading PrivValidator from %v: %v\n", filePath, err)) } privVal.filePath = filePath - privVal.Signer = NewDefaultSigner(privVal.PrivKey) - privVal.setPubKeyAndAddress() return &privVal } // Generates a new validator with private key. -func GenPrivValidator() *PrivValidator { +func GenPrivValidator() *DefaultPrivValidator { privKey := crypto.GenPrivKeyEd25519().Wrap() pubKey := privKey.PubKey() - return &PrivValidator{ - Address: pubKey.Address(), - PubKey: pubKey, - PrivKey: privKey, - LastStep: stepNone, - filePath: "", + return &DefaultPrivValidator{ + Info: PrivValidatorInfo{ + Address: pubKey.Address(), + PubKey: pubKey, + LastStep: stepNone, + }, Signer: NewDefaultSigner(privKey), + filePath: "", } } @@ -132,40 +154,37 @@ func GenPrivValidator() *PrivValidator { // signer object. Tendermint tracks state in the PrivValidator that might be // saved to disk. Please supply a filepath where Tendermint can save the // private validator. -func LoadPrivValidatorWithSigner(signer Signer, filePath string) *PrivValidator { - return &PrivValidator{ - Address: signer.PubKey().Address(), - PubKey: signer.PubKey(), - LastStep: stepNone, - filePath: filePath, +func LoadPrivValidatorWithSigner(signer *DefaultSigner, filePath string) *DefaultPrivValidator { + return &DefaultPrivValidator{ + Info: PrivValidatorInfo{ + Address: signer.PubKey().Address(), + PubKey: signer.PubKey(), + LastStep: stepNone, + }, Signer: signer, + filePath: filePath, } } -func (privVal *PrivValidator) SetSigner(s Signer) { - privVal.Signer = s - privVal.setPubKeyAndAddress() -} - // Overwrite address and pubkey for convenience -func (privVal *PrivValidator) setPubKeyAndAddress() { +/*func (privVal *DefaultPrivValidator) setPubKeyAndAddress() { privVal.PubKey = privVal.Signer.PubKey() privVal.Address = privVal.PubKey.Address() -} +}*/ -func (privVal *PrivValidator) SetFile(filePath string) { +func (privVal *DefaultPrivValidator) SetFile(filePath string) { privVal.mtx.Lock() defer privVal.mtx.Unlock() privVal.filePath = filePath } -func (privVal *PrivValidator) Save() { +func (privVal *DefaultPrivValidator) Save() { privVal.mtx.Lock() defer privVal.mtx.Unlock() privVal.save() } -func (privVal *PrivValidator) save() { +func (privVal *DefaultPrivValidator) save() { if privVal.filePath == "" { PanicSanity("Cannot save PrivValidator: filePath not set") } @@ -182,20 +201,20 @@ func (privVal *PrivValidator) save() { } // NOTE: Unsafe! -func (privVal *PrivValidator) Reset() { - privVal.LastHeight = 0 - privVal.LastRound = 0 - privVal.LastStep = 0 - privVal.LastSignature = crypto.Signature{} - privVal.LastSignBytes = nil +func (privVal *DefaultPrivValidator) Reset() { + privVal.Info.LastHeight = 0 + privVal.Info.LastRound = 0 + privVal.Info.LastStep = 0 + privVal.Info.LastSignature = crypto.Signature{} + privVal.Info.LastSignBytes = nil privVal.Save() } -func (privVal *PrivValidator) GetAddress() []byte { - return privVal.Address +func (privVal *DefaultPrivValidator) GetAddress() []byte { + return privVal.Address() } -func (privVal *PrivValidator) SignVote(chainID string, vote *Vote) error { +func (privVal *DefaultPrivValidator) SignVote(chainID string, vote *Vote) error { privVal.mtx.Lock() defer privVal.mtx.Unlock() signature, err := privVal.signBytesHRS(vote.Height, vote.Round, voteToStep(vote), SignBytes(chainID, vote)) @@ -206,7 +225,7 @@ func (privVal *PrivValidator) SignVote(chainID string, vote *Vote) error { return nil } -func (privVal *PrivValidator) SignProposal(chainID string, proposal *Proposal) error { +func (privVal *DefaultPrivValidator) SignProposal(chainID string, proposal *Proposal) error { privVal.mtx.Lock() defer privVal.mtx.Unlock() signature, err := privVal.signBytesHRS(proposal.Height, proposal.Round, stepPropose, SignBytes(chainID, proposal)) @@ -218,33 +237,34 @@ func (privVal *PrivValidator) SignProposal(chainID string, proposal *Proposal) e } // check if there's a regression. Else sign and write the hrs+signature to disk -func (privVal *PrivValidator) signBytesHRS(height, round int, step int8, signBytes []byte) (crypto.Signature, error) { +func (privVal *DefaultPrivValidator) signBytesHRS(height, round int, step int8, signBytes []byte) (crypto.Signature, error) { sig := crypto.Signature{} + info := privVal.Info // If height regression, err - if privVal.LastHeight > height { + if info.LastHeight > height { return sig, errors.New("Height regression") } // More cases for when the height matches - if privVal.LastHeight == height { + if info.LastHeight == height { // If round regression, err - if privVal.LastRound > round { + if info.LastRound > round { return sig, errors.New("Round regression") } // If step regression, err - if privVal.LastRound == round { - if privVal.LastStep > step { + if info.LastRound == round { + if info.LastStep > step { return sig, errors.New("Step regression") - } else if privVal.LastStep == step { - if privVal.LastSignBytes != nil { - if privVal.LastSignature.Empty() { + } else if info.LastStep == step { + if info.LastSignBytes != nil { + if info.LastSignature.Empty() { PanicSanity("privVal: LastSignature is nil but LastSignBytes is not!") } // so we dont sign a conflicting vote or proposal // NOTE: proposals are non-deterministic (include time), // so we can actually lose them, but will still never sign conflicting ones - if bytes.Equal(privVal.LastSignBytes, signBytes) { - // log.Notice("Using privVal.LastSignature", "sig", privVal.LastSignature) - return privVal.LastSignature, nil + if bytes.Equal(info.LastSignBytes, signBytes) { + // log.Notice("Using info.LastSignature", "sig", info.LastSignature) + return info.LastSignature, nil } } return sig, errors.New("Step regression") @@ -253,45 +273,46 @@ func (privVal *PrivValidator) signBytesHRS(height, round int, step int8, signByt } // Sign - sig, err := privVal.Sign(signBytes) + sig, err := privVal.Signer.Sign(signBytes) if err != nil { return sig, err } // Persist height/round/step - privVal.LastHeight = height - privVal.LastRound = round - privVal.LastStep = step - privVal.LastSignature = sig - privVal.LastSignBytes = signBytes + privVal.Info.LastHeight = height + privVal.Info.LastRound = round + privVal.Info.LastStep = step + privVal.Info.LastSignature = sig + privVal.Info.LastSignBytes = signBytes privVal.save() return sig, nil } -func (privVal *PrivValidator) SignHeartbeat(chainID string, heartbeat *Heartbeat) error { +func (privVal *DefaultPrivValidator) SignHeartbeat(chainID string, heartbeat *Heartbeat) error { privVal.mtx.Lock() defer privVal.mtx.Unlock() var err error - heartbeat.Signature, err = privVal.Sign(SignBytes(chainID, heartbeat)) + heartbeat.Signature, err = privVal.Signer.Sign(SignBytes(chainID, heartbeat)) return err } -func (privVal *PrivValidator) String() string { - return fmt.Sprintf("PrivValidator{%v LH:%v, LR:%v, LS:%v}", privVal.Address, privVal.LastHeight, privVal.LastRound, privVal.LastStep) +func (privVal *DefaultPrivValidator) String() string { + info := privVal.Info + return fmt.Sprintf("PrivValidator{%v LH:%v, LR:%v, LS:%v}", info.Address, info.LastHeight, info.LastRound, info.LastStep) } //------------------------------------- -type PrivValidatorsByAddress []*PrivValidator +type PrivValidatorsByAddress []*DefaultPrivValidator func (pvs PrivValidatorsByAddress) Len() int { return len(pvs) } func (pvs PrivValidatorsByAddress) Less(i, j int) bool { - return bytes.Compare(pvs[i].Address, pvs[j].Address) == -1 + return bytes.Compare(pvs[i].Info.Address, pvs[j].Info.Address) == -1 } func (pvs PrivValidatorsByAddress) Swap(i, j int) { diff --git a/types/priv_validator_test.go b/types/priv_validator_test.go index 1eb0b57db..d0fcfd185 100644 --- a/types/priv_validator_test.go +++ b/types/priv_validator_test.go @@ -29,29 +29,34 @@ func TestLoadValidator(t *testing.T) { require.Nil(err, "%+v", err) serialized := fmt.Sprintf(`{ - "address": "%s", - "pub_key": { - "type": "ed25519", - "data": "%s" + "info": { + "address": "%s", + "pub_key": { + "type": "ed25519", + "data": "%s" + }, + "last_height": 0, + "last_round": 0, + "last_step": 0, + "last_signature": null }, - "priv_key": { - "type": "ed25519", - "data": "%s" - }, - "last_height": 0, - "last_round": 0, - "last_step": 0, - "last_signature": null + "signer": { + "priv_key": { + "type": "ed25519", + "data": "%s" + } + } }`, addrStr, pubStr, privStr) - val := PrivValidator{} + val := DefaultPrivValidator{} err = json.Unmarshal([]byte(serialized), &val) require.Nil(err, "%+v", err) // make sure the values match - assert.EqualValues(addrBytes, val.Address) - assert.EqualValues(pubKey, val.PubKey) - assert.EqualValues(privKey, val.PrivKey) + assert.EqualValues(addrBytes, val.Address()) + assert.EqualValues(pubKey, val.PubKey()) + valPrivKey := val.Signer.PrivKey + assert.EqualValues(privKey, valPrivKey) // export it and make sure it is the same out, err := json.Marshal(val) diff --git a/types/proposal_test.go b/types/proposal_test.go index 6f6189595..ce1bbc8d5 100644 --- a/types/proposal_test.go +++ b/types/proposal_test.go @@ -30,15 +30,15 @@ func BenchmarkProposalWriteSignBytes(b *testing.B) { func BenchmarkProposalSign(b *testing.B) { privVal := GenPrivValidator() for i := 0; i < b.N; i++ { - privVal.Sign(SignBytes("test_chain_id", testProposal)) + privVal.Signer.Sign(SignBytes("test_chain_id", testProposal)) } } func BenchmarkProposalVerifySignature(b *testing.B) { signBytes := SignBytes("test_chain_id", testProposal) privVal := GenPrivValidator() - signature, _ := privVal.Sign(signBytes) - pubKey := privVal.PubKey + signature, _ := privVal.Signer.Sign(signBytes) + pubKey := privVal.PubKey() for i := 0; i < b.N; i++ { pubKey.VerifyBytes(SignBytes("test_chain_id", testProposal), signature) diff --git a/types/validator.go b/types/validator.go index 24f8974f3..29cbfea27 100644 --- a/types/validator.go +++ b/types/validator.go @@ -106,7 +106,7 @@ func (vc validatorCodec) Compare(o1 interface{}, o2 interface{}) int { //-------------------------------------------------------------------------------- // For testing... -func RandValidator(randPower bool, minPower int64) (*Validator, *PrivValidator) { +func RandValidator(randPower bool, minPower int64) (*Validator, *DefaultPrivValidator) { privVal := GenPrivValidator() _, tempFilePath := cmn.Tempfile("priv_validator_") privVal.SetFile(tempFilePath) @@ -114,6 +114,6 @@ func RandValidator(randPower bool, minPower int64) (*Validator, *PrivValidator) if randPower { votePower += int64(cmn.RandUint32()) } - val := NewValidator(privVal.PubKey, votePower) + val := NewValidator(privVal.PubKey(), votePower) return val, privVal } diff --git a/types/validator_set.go b/types/validator_set.go index e334ed9f3..5531f4af3 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -369,9 +369,9 @@ func (ac accumComparable) Less(o interface{}) bool { // For testing // NOTE: PrivValidator are in order. -func RandValidatorSet(numValidators int, votingPower int64) (*ValidatorSet, []*PrivValidator) { +func RandValidatorSet(numValidators int, votingPower int64) (*ValidatorSet, []*DefaultPrivValidator) { vals := make([]*Validator, numValidators) - privValidators := make([]*PrivValidator, numValidators) + privValidators := make([]*DefaultPrivValidator, numValidators) for i := 0; i < numValidators; i++ { val, privValidator := RandValidator(false, votingPower) vals[i] = val diff --git a/types/vote_set_test.go b/types/vote_set_test.go index ddc055489..716becf52 100644 --- a/types/vote_set_test.go +++ b/types/vote_set_test.go @@ -11,7 +11,7 @@ import ( ) // NOTE: privValidators are in order -func randVoteSet(height int, round int, type_ byte, numValidators int, votingPower int64) (*VoteSet, *ValidatorSet, []*PrivValidator) { +func randVoteSet(height int, round int, type_ byte, numValidators int, votingPower int64) (*VoteSet, *ValidatorSet, []*DefaultPrivValidator) { valSet, privValidators := RandValidatorSet(numValidators, votingPower) return NewVoteSet("test_chain_id", height, round, type_, valSet), valSet, privValidators } @@ -59,9 +59,9 @@ func withBlockPartsHeader(vote *Vote, blockPartsHeader PartSetHeader) *Vote { return vote } -func signAddVote(privVal *PrivValidator, vote *Vote, voteSet *VoteSet) (bool, error) { +func signAddVote(privVal *DefaultPrivValidator, vote *Vote, voteSet *VoteSet) (bool, error) { var err error - vote.Signature, err = privVal.Sign(SignBytes(voteSet.ChainID(), vote)) + vote.Signature, err = privVal.Signer.Sign(SignBytes(voteSet.ChainID(), vote)) if err != nil { return false, err } @@ -76,7 +76,7 @@ func TestAddVote(t *testing.T) { // t.Logf(">> %v", voteSet) - if voteSet.GetByAddress(val0.Address) != nil { + if voteSet.GetByAddress(val0.Address()) != nil { t.Errorf("Expected GetByAddress(val0.Address) to be nil") } if voteSet.BitArray().GetIndex(0) { @@ -88,7 +88,7 @@ func TestAddVote(t *testing.T) { } vote := &Vote{ - ValidatorAddress: val0.Address, + ValidatorAddress: val0.Address(), ValidatorIndex: 0, // since privValidators are in order Height: height, Round: round, @@ -100,7 +100,7 @@ func TestAddVote(t *testing.T) { t.Error(err) } - if voteSet.GetByAddress(val0.Address) == nil { + if voteSet.GetByAddress(val0.Address()) == nil { t.Errorf("Expected GetByAddress(val0.Address) to be present") } if !voteSet.BitArray().GetIndex(0) { @@ -126,7 +126,7 @@ func Test2_3Majority(t *testing.T) { } // 6 out of 10 voted for nil. for i := 0; i < 6; i++ { - vote := withValidator(voteProto, privValidators[i].Address, i) + vote := withValidator(voteProto, privValidators[i].Address(), i) signAddVote(privValidators[i], vote, voteSet) } blockID, ok := voteSet.TwoThirdsMajority() @@ -136,7 +136,7 @@ func Test2_3Majority(t *testing.T) { // 7th validator voted for some blockhash { - vote := withValidator(voteProto, privValidators[6].Address, 6) + vote := withValidator(voteProto, privValidators[6].Address(), 6) signAddVote(privValidators[6], withBlockHash(vote, RandBytes(32)), voteSet) blockID, ok = voteSet.TwoThirdsMajority() if ok || !blockID.IsZero() { @@ -146,7 +146,7 @@ func Test2_3Majority(t *testing.T) { // 8th validator voted for nil. { - vote := withValidator(voteProto, privValidators[7].Address, 7) + vote := withValidator(voteProto, privValidators[7].Address(), 7) signAddVote(privValidators[7], vote, voteSet) blockID, ok = voteSet.TwoThirdsMajority() if !ok || !blockID.IsZero() { @@ -174,7 +174,7 @@ func Test2_3MajorityRedux(t *testing.T) { // 66 out of 100 voted for nil. for i := 0; i < 66; i++ { - vote := withValidator(voteProto, privValidators[i].Address, i) + vote := withValidator(voteProto, privValidators[i].Address(), i) signAddVote(privValidators[i], vote, voteSet) } blockID, ok := voteSet.TwoThirdsMajority() @@ -184,7 +184,7 @@ func Test2_3MajorityRedux(t *testing.T) { // 67th validator voted for nil { - vote := withValidator(voteProto, privValidators[66].Address, 66) + vote := withValidator(voteProto, privValidators[66].Address(), 66) signAddVote(privValidators[66], withBlockHash(vote, nil), voteSet) blockID, ok = voteSet.TwoThirdsMajority() if ok || !blockID.IsZero() { @@ -194,7 +194,7 @@ func Test2_3MajorityRedux(t *testing.T) { // 68th validator voted for a different BlockParts PartSetHeader { - vote := withValidator(voteProto, privValidators[67].Address, 67) + vote := withValidator(voteProto, privValidators[67].Address(), 67) blockPartsHeader := PartSetHeader{blockPartsTotal, crypto.CRandBytes(32)} signAddVote(privValidators[67], withBlockPartsHeader(vote, blockPartsHeader), voteSet) blockID, ok = voteSet.TwoThirdsMajority() @@ -205,7 +205,7 @@ func Test2_3MajorityRedux(t *testing.T) { // 69th validator voted for different BlockParts Total { - vote := withValidator(voteProto, privValidators[68].Address, 68) + vote := withValidator(voteProto, privValidators[68].Address(), 68) blockPartsHeader := PartSetHeader{blockPartsTotal + 1, blockPartsHeader.Hash} signAddVote(privValidators[68], withBlockPartsHeader(vote, blockPartsHeader), voteSet) blockID, ok = voteSet.TwoThirdsMajority() @@ -216,7 +216,7 @@ func Test2_3MajorityRedux(t *testing.T) { // 70th validator voted for different BlockHash { - vote := withValidator(voteProto, privValidators[69].Address, 69) + vote := withValidator(voteProto, privValidators[69].Address(), 69) signAddVote(privValidators[69], withBlockHash(vote, RandBytes(32)), voteSet) blockID, ok = voteSet.TwoThirdsMajority() if ok || !blockID.IsZero() { @@ -226,7 +226,7 @@ func Test2_3MajorityRedux(t *testing.T) { // 71st validator voted for the right BlockHash & BlockPartsHeader { - vote := withValidator(voteProto, privValidators[70].Address, 70) + vote := withValidator(voteProto, privValidators[70].Address(), 70) signAddVote(privValidators[70], vote, voteSet) blockID, ok = voteSet.TwoThirdsMajority() if !ok || !blockID.Equals(BlockID{blockHash, blockPartsHeader}) { @@ -250,7 +250,7 @@ func TestBadVotes(t *testing.T) { // val0 votes for nil. { - vote := withValidator(voteProto, privValidators[0].Address, 0) + vote := withValidator(voteProto, privValidators[0].Address(), 0) added, err := signAddVote(privValidators[0], vote, voteSet) if !added || err != nil { t.Errorf("Expected VoteSet.Add to succeed") @@ -259,7 +259,7 @@ func TestBadVotes(t *testing.T) { // val0 votes again for some block. { - vote := withValidator(voteProto, privValidators[0].Address, 0) + vote := withValidator(voteProto, privValidators[0].Address(), 0) added, err := signAddVote(privValidators[0], withBlockHash(vote, RandBytes(32)), voteSet) if added || err == nil { t.Errorf("Expected VoteSet.Add to fail, conflicting vote.") @@ -268,7 +268,7 @@ func TestBadVotes(t *testing.T) { // val1 votes on another height { - vote := withValidator(voteProto, privValidators[1].Address, 1) + vote := withValidator(voteProto, privValidators[1].Address(), 1) added, err := signAddVote(privValidators[1], withHeight(vote, height+1), voteSet) if added || err == nil { t.Errorf("Expected VoteSet.Add to fail, wrong height") @@ -277,7 +277,7 @@ func TestBadVotes(t *testing.T) { // val2 votes on another round { - vote := withValidator(voteProto, privValidators[2].Address, 2) + vote := withValidator(voteProto, privValidators[2].Address(), 2) added, err := signAddVote(privValidators[2], withRound(vote, round+1), voteSet) if added || err == nil { t.Errorf("Expected VoteSet.Add to fail, wrong round") @@ -286,7 +286,7 @@ func TestBadVotes(t *testing.T) { // val3 votes of another type. { - vote := withValidator(voteProto, privValidators[3].Address, 3) + vote := withValidator(voteProto, privValidators[3].Address(), 3) added, err := signAddVote(privValidators[3], withType(vote, VoteTypePrecommit), voteSet) if added || err == nil { t.Errorf("Expected VoteSet.Add to fail, wrong type") @@ -311,7 +311,7 @@ func TestConflicts(t *testing.T) { // val0 votes for nil. { - vote := withValidator(voteProto, privValidators[0].Address, 0) + vote := withValidator(voteProto, privValidators[0].Address(), 0) added, err := signAddVote(privValidators[0], vote, voteSet) if !added || err != nil { t.Errorf("Expected VoteSet.Add to succeed") @@ -320,7 +320,7 @@ func TestConflicts(t *testing.T) { // val0 votes again for blockHash1. { - vote := withValidator(voteProto, privValidators[0].Address, 0) + vote := withValidator(voteProto, privValidators[0].Address(), 0) added, err := signAddVote(privValidators[0], withBlockHash(vote, blockHash1), voteSet) if added { t.Errorf("Expected VoteSet.Add to fail, conflicting vote.") @@ -335,7 +335,7 @@ func TestConflicts(t *testing.T) { // val0 votes again for blockHash1. { - vote := withValidator(voteProto, privValidators[0].Address, 0) + vote := withValidator(voteProto, privValidators[0].Address(), 0) added, err := signAddVote(privValidators[0], withBlockHash(vote, blockHash1), voteSet) if !added { t.Errorf("Expected VoteSet.Add to succeed, called SetPeerMaj23().") @@ -350,7 +350,7 @@ func TestConflicts(t *testing.T) { // val0 votes again for blockHash1. { - vote := withValidator(voteProto, privValidators[0].Address, 0) + vote := withValidator(voteProto, privValidators[0].Address(), 0) added, err := signAddVote(privValidators[0], withBlockHash(vote, blockHash2), voteSet) if added { t.Errorf("Expected VoteSet.Add to fail, duplicate SetPeerMaj23() from peerA") @@ -362,7 +362,7 @@ func TestConflicts(t *testing.T) { // val1 votes for blockHash1. { - vote := withValidator(voteProto, privValidators[1].Address, 1) + vote := withValidator(voteProto, privValidators[1].Address(), 1) added, err := signAddVote(privValidators[1], withBlockHash(vote, blockHash1), voteSet) if !added || err != nil { t.Errorf("Expected VoteSet.Add to succeed") @@ -379,7 +379,7 @@ func TestConflicts(t *testing.T) { // val2 votes for blockHash2. { - vote := withValidator(voteProto, privValidators[2].Address, 2) + vote := withValidator(voteProto, privValidators[2].Address(), 2) added, err := signAddVote(privValidators[2], withBlockHash(vote, blockHash2), voteSet) if !added || err != nil { t.Errorf("Expected VoteSet.Add to succeed") @@ -399,7 +399,7 @@ func TestConflicts(t *testing.T) { // val2 votes for blockHash1. { - vote := withValidator(voteProto, privValidators[2].Address, 2) + vote := withValidator(voteProto, privValidators[2].Address(), 2) added, err := signAddVote(privValidators[2], withBlockHash(vote, blockHash1), voteSet) if !added { t.Errorf("Expected VoteSet.Add to succeed") @@ -439,7 +439,7 @@ func TestMakeCommit(t *testing.T) { // 6 out of 10 voted for some block. for i := 0; i < 6; i++ { - vote := withValidator(voteProto, privValidators[i].Address, i) + vote := withValidator(voteProto, privValidators[i].Address(), i) signAddVote(privValidators[i], vote, voteSet) } @@ -448,7 +448,7 @@ func TestMakeCommit(t *testing.T) { // 7th voted for some other block. { - vote := withValidator(voteProto, privValidators[6].Address, 6) + vote := withValidator(voteProto, privValidators[6].Address(), 6) vote = withBlockHash(vote, RandBytes(32)) vote = withBlockPartsHeader(vote, PartSetHeader{123, RandBytes(32)}) signAddVote(privValidators[6], vote, voteSet) @@ -456,7 +456,7 @@ func TestMakeCommit(t *testing.T) { // The 8th voted like everyone else. { - vote := withValidator(voteProto, privValidators[7].Address, 7) + vote := withValidator(voteProto, privValidators[7].Address(), 7) signAddVote(privValidators[7], vote, voteSet) } From 944ebccfe9063550923276f2b3fd0c9d3a8bd953 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 18 Sep 2017 22:05:33 -0400 Subject: [PATCH 74/92] more PrivValidator interface --- cmd/tendermint/commands/gen_validator.go | 2 +- cmd/tendermint/commands/init.go | 3 +- .../commands/reset_priv_validator.go | 6 +- cmd/tendermint/commands/run_node.go | 2 +- cmd/tendermint/commands/show_validator.go | 2 +- cmd/tendermint/commands/testnet.go | 5 +- consensus/common_test.go | 3 +- node/node.go | 4 +- types/priv_validator.go | 239 +++++++++--------- types/validator.go | 5 +- types/validator_set.go | 4 +- 11 files changed, 135 insertions(+), 140 deletions(-) diff --git a/cmd/tendermint/commands/gen_validator.go b/cmd/tendermint/commands/gen_validator.go index e4dfb0bd1..984176d2a 100644 --- a/cmd/tendermint/commands/gen_validator.go +++ b/cmd/tendermint/commands/gen_validator.go @@ -18,7 +18,7 @@ var GenValidatorCmd = &cobra.Command{ } func genValidator(cmd *cobra.Command, args []string) { - privValidator := types.GenPrivValidator() + privValidator := types.GenPrivValidatorFS("") privValidatorJSONBytes, _ := json.MarshalIndent(privValidator, "", "\t") fmt.Printf(`%v `, string(privValidatorJSONBytes)) diff --git a/cmd/tendermint/commands/init.go b/cmd/tendermint/commands/init.go index 259645044..d8e8bde40 100644 --- a/cmd/tendermint/commands/init.go +++ b/cmd/tendermint/commands/init.go @@ -19,8 +19,7 @@ var InitFilesCmd = &cobra.Command{ func initFiles(cmd *cobra.Command, args []string) { privValFile := config.PrivValidatorFile() if _, err := os.Stat(privValFile); os.IsNotExist(err) { - privValidator := types.GenPrivValidator() - privValidator.SetFile(privValFile) + privValidator := types.GenPrivValidatorFS(privValFile) privValidator.Save() genFile := config.GenesisFile() diff --git a/cmd/tendermint/commands/reset_priv_validator.go b/cmd/tendermint/commands/reset_priv_validator.go index bedd38926..6255f9613 100644 --- a/cmd/tendermint/commands/reset_priv_validator.go +++ b/cmd/tendermint/commands/reset_priv_validator.go @@ -46,14 +46,12 @@ func resetPrivValidator(cmd *cobra.Command, args []string) { func resetPrivValidatorLocal(privValFile string, logger log.Logger) { // Get PrivValidator - var privValidator types.PrivValidator if _, err := os.Stat(privValFile); err == nil { - privValidator = types.LoadPrivValidator(privValFile) + privValidator := types.LoadPrivValidatorFS(privValFile) privValidator.Reset() logger.Info("Reset PrivValidator", "file", privValFile) } else { - privValidator = types.GenPrivValidator() - privValidator.SetFile(privValFile) + privValidator := types.GenPrivValidatorFS(privValFile) privValidator.Save() logger.Info("Generated PrivValidator", "file", privValFile) } diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index 883ec6ec4..d4acd98df 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -44,7 +44,7 @@ func AddNodeFlags(cmd *cobra.Command) { type FuncSignerAndApp func(*cfg.Config) (types.PrivValidator, proxy.ClientCreator) func DefaultSignerAndApp(config *cfg.Config) (types.PrivValidator, proxy.ClientCreator) { - privValidator := types.LoadOrGenPrivValidator(config.PrivValidatorFile()) + privValidator := types.LoadOrGenPrivValidatorFS(config.PrivValidatorFile()) clientCreator := proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()) return privValidator, clientCreator } diff --git a/cmd/tendermint/commands/show_validator.go b/cmd/tendermint/commands/show_validator.go index 86a665f0c..458f11c2d 100644 --- a/cmd/tendermint/commands/show_validator.go +++ b/cmd/tendermint/commands/show_validator.go @@ -17,7 +17,7 @@ var ShowValidatorCmd = &cobra.Command{ } func showValidator(cmd *cobra.Command, args []string) { - privValidator := types.LoadOrGenPrivValidator(config.PrivValidatorFile()) + privValidator := types.LoadOrGenPrivValidatorFS(config.PrivValidatorFile()) pubKeyJSONBytes, _ := data.ToJSON(privValidator.PubKey) fmt.Println(string(pubKeyJSONBytes)) } diff --git a/cmd/tendermint/commands/testnet.go b/cmd/tendermint/commands/testnet.go index 9412379a5..7fc4f94fe 100644 --- a/cmd/tendermint/commands/testnet.go +++ b/cmd/tendermint/commands/testnet.go @@ -45,7 +45,7 @@ func testnetFiles(cmd *cobra.Command, args []string) { } // Read priv_validator.json to populate vals privValFile := path.Join(dataDir, mach, "priv_validator.json") - privVal := types.LoadPrivValidator(privValFile) + privVal := types.LoadPrivValidatorFS(privValFile) genVals[i] = types.GenesisValidator{ PubKey: privVal.PubKey(), Power: 1, @@ -87,7 +87,6 @@ func ensurePrivValidator(file string) { if cmn.FileExists(file) { return } - privValidator := types.GenPrivValidator() - privValidator.SetFile(file) + privValidator := types.GenPrivValidatorFS(file) privValidator.Save() } diff --git a/consensus/common_test.go b/consensus/common_test.go index cde56a3db..09b113553 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -368,9 +368,8 @@ func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerF if i < nValidators { privVal = privVals[i] } else { - privVal = types.GenPrivValidator() _, tempFilePath := Tempfile("priv_validator_") - privVal.SetFile(tempFilePath) + privVal = types.GenPrivValidator(tempFilePath) } css[i] = newConsensusStateWithConfig(thisConfig, state, privVal, appFunc()) diff --git a/node/node.go b/node/node.go index a0a8c529a..4f22ee923 100644 --- a/node/node.go +++ b/node/node.go @@ -60,7 +60,7 @@ type Node struct { func NewNodeDefault(config *cfg.Config, logger log.Logger) *Node { // Get PrivValidator - privValidator := types.LoadOrGenPrivValidator(config.PrivValidatorFile()) + privValidator := types.LoadOrGenPrivValidatorFS(config.PrivValidatorFile()) return NewNode(config, privValidator, proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()), logger) } @@ -125,7 +125,7 @@ func NewNode(config *cfg.Config, privValidator types.PrivValidator, clientCreato } // Log whether this node is a validator or an observer - if state.Validators.HasAddress(privValidator.PubKey().Address()) { + if state.Validators.HasAddress(privValidator.Address()) { consensusLogger.Info("This node is a validator") } else { consensusLogger.Info("This node is not a validator") diff --git a/types/priv_validator.go b/types/priv_validator.go index e8fb3e240..a90f620b6 100644 --- a/types/priv_validator.go +++ b/types/priv_validator.go @@ -34,37 +34,7 @@ func voteToStep(vote *Vote) int8 { } } -// Signer is an interface that defines how to sign votes. -// It is the caller's duty to verify the msg before calling Sign, -// eg. to avoid double signing. -// Currently, the only callers are SignVote and SignProposal. -type Signer interface { - PubKey() crypto.PubKey - Sign(msg []byte) (crypto.Signature, error) -} - -// DefaultSigner implements Signer. -// It uses a standard crypto.PrivKey. -type DefaultSigner struct { - PrivKey crypto.PrivKey `json:"priv_key"` -} - -// NewDefaultSigner returns an instance of DefaultSigner. -func NewDefaultSigner(priv crypto.PrivKey) *DefaultSigner { - return &DefaultSigner{PrivKey: priv} -} - -// Sign implements Signer. It signs the byte slice with a private key. -func (ds *DefaultSigner) Sign(msg []byte) (crypto.Signature, error) { - return ds.PrivKey.Sign(msg), nil -} - -// PubKey implements Signer. It should return the public key that corresponds -// to the private key used for signing. -func (ds *DefaultSigner) PubKey() crypto.PubKey { - return ds.PrivKey.PubKey() -} - +// PrivValidator defines the functionality of a local Tendermint validator. type PrivValidator interface { Address() data.Bytes // redundant since .PubKey().Address() PubKey() crypto.PubKey @@ -72,60 +42,45 @@ type PrivValidator interface { SignVote(chainID string, vote *Vote) error SignProposal(chainID string, proposal *Proposal) error SignHeartbeat(chainID string, heartbeat *Heartbeat) error - - Reset() - - SetFile(file string) - Save() } -// DefaultPrivValidator implements the functionality for signing blocks. -type DefaultPrivValidator struct { - Info PrivValidatorInfo `json:"info"` - Signer *DefaultSigner `json:"signer"` +// PrivValidatorFS implements PrivValidator using data persisted to disk +// to prevent double signing. The Signer itself can be mutated to use +// something besides the default, for instance a hardware signer. +type PrivValidatorFS struct { + ID ValidatorID `json:"id"` + Signer Signer `json:"signer"` + + // mutable state to be persisted to disk + // after each signature to prevent double signing + mtx sync.Mutex + Info LastSignedInfo `json:"info"` // For persistence. // Overloaded for testing. filePath string - mtx sync.Mutex -} - -func (pv *DefaultPrivValidator) Address() data.Bytes { - return pv.Info.Address } -func (pv *DefaultPrivValidator) PubKey() crypto.PubKey { - return pv.Info.PubKey -} - -type PrivValidatorInfo struct { - Address data.Bytes `json:"address"` - PubKey crypto.PubKey `json:"pub_key"` - LastHeight int `json:"last_height"` - LastRound int `json:"last_round"` - LastStep int8 `json:"last_step"` - LastSignature crypto.Signature `json:"last_signature,omitempty"` // so we dont lose signatures - LastSignBytes data.Bytes `json:"last_signbytes,omitempty"` // so we dont lose signatures -} - -func LoadOrGenPrivValidator(filePath string) *DefaultPrivValidator { - var privValidator *DefaultPrivValidator +// LoadOrGenPrivValidatorFS loads a PrivValidatorFS from the given filePath +// or else generates a new one and saves it to the filePath. +func LoadOrGenPrivValidatorFS(filePath string) *PrivValidatorFS { + var PrivValidatorFS *PrivValidatorFS if _, err := os.Stat(filePath); err == nil { - privValidator = LoadPrivValidator(filePath) + PrivValidatorFS = LoadPrivValidatorFS(filePath) } else { - privValidator = GenPrivValidator() - privValidator.SetFile(filePath) - privValidator.Save() + PrivValidatorFS = GenPrivValidatorFS(filePath) + PrivValidatorFS.Save() } - return privValidator + return PrivValidatorFS } -func LoadPrivValidator(filePath string) *DefaultPrivValidator { +// LoadPrivValidatorFS loads a PrivValidatorFS from the filePath. +func LoadPrivValidatorFS(filePath string) *PrivValidatorFS { privValJSONBytes, err := ioutil.ReadFile(filePath) if err != nil { Exit(err.Error()) } - privVal := DefaultPrivValidator{} + privVal := PrivValidatorFS{} err = json.Unmarshal(privValJSONBytes, &privVal) if err != nil { Exit(Fmt("Error reading PrivValidator from %v: %v\n", filePath, err)) @@ -135,56 +90,58 @@ func LoadPrivValidator(filePath string) *DefaultPrivValidator { return &privVal } -// Generates a new validator with private key. -func GenPrivValidator() *DefaultPrivValidator { +// GenPrivValidatorFS generates a new validator with randomly generated private key +// and sets the filePath, but does not call Save(). +func GenPrivValidatorFS(filePath string) *PrivValidatorFS { privKey := crypto.GenPrivKeyEd25519().Wrap() - pubKey := privKey.PubKey() - return &DefaultPrivValidator{ - Info: PrivValidatorInfo{ - Address: pubKey.Address(), - PubKey: pubKey, + return &PrivValidatorFS{ + ID: ValidatorID{privKey.PubKey().Address(), privKey.PubKey()}, + Info: LastSignedInfo{ LastStep: stepNone, }, Signer: NewDefaultSigner(privKey), - filePath: "", + filePath: filePath, } } -// LoadPrivValidatorWithSigner instantiates a private validator with a custom -// signer object. Tendermint tracks state in the PrivValidator that might be -// saved to disk. Please supply a filepath where Tendermint can save the -// private validator. -func LoadPrivValidatorWithSigner(signer *DefaultSigner, filePath string) *DefaultPrivValidator { - return &DefaultPrivValidator{ - Info: PrivValidatorInfo{ - Address: signer.PubKey().Address(), - PubKey: signer.PubKey(), - LastStep: stepNone, - }, - Signer: signer, - filePath: filePath, +// LoadPrivValidatorWithSigner loads a PrivValidatorFS with a custom +// signer object. The PrivValidatorFS handles double signing prevention by persisting +// data to the filePath, while the Signer handles the signing. +// If the filePath does not exist, the PrivValidatorFS must be created manually and saved. +func LoadPrivValidatorFSWithSigner(filePath string, signerFunc func(ValidatorID) Signer) *PrivValidatorFS { + privValJSONBytes, err := ioutil.ReadFile(filePath) + if err != nil { + Exit(err.Error()) } + privVal := PrivValidatorFS{} + err = json.Unmarshal(privValJSONBytes, &privVal) + if err != nil { + Exit(Fmt("Error reading PrivValidator from %v: %v\n", filePath, err)) + } + + privVal.filePath = filePath + privVal.Signer = signerFunc(privVal.ID) + return &privVal } -// Overwrite address and pubkey for convenience -/*func (privVal *DefaultPrivValidator) setPubKeyAndAddress() { - privVal.PubKey = privVal.Signer.PubKey() - privVal.Address = privVal.PubKey.Address() -}*/ +// Address returns the address of the validator. +func (pv *PrivValidatorFS) Address() data.Bytes { + return pv.ID.Address +} -func (privVal *DefaultPrivValidator) SetFile(filePath string) { - privVal.mtx.Lock() - defer privVal.mtx.Unlock() - privVal.filePath = filePath +// PubKey returns the public key of the validator. +func (pv *PrivValidatorFS) PubKey() crypto.PubKey { + return pv.ID.PubKey } -func (privVal *DefaultPrivValidator) Save() { +// Save persists the PrivValidatorFS to disk. +func (privVal *PrivValidatorFS) Save() { privVal.mtx.Lock() defer privVal.mtx.Unlock() privVal.save() } -func (privVal *DefaultPrivValidator) save() { +func (privVal *PrivValidatorFS) save() { if privVal.filePath == "" { PanicSanity("Cannot save PrivValidator: filePath not set") } @@ -200,8 +157,9 @@ func (privVal *DefaultPrivValidator) save() { } } +// Reset resets all fields in the PrivValidatorFS.Info. // NOTE: Unsafe! -func (privVal *DefaultPrivValidator) Reset() { +func (privVal *PrivValidatorFS) Reset() { privVal.Info.LastHeight = 0 privVal.Info.LastRound = 0 privVal.Info.LastStep = 0 @@ -210,11 +168,8 @@ func (privVal *DefaultPrivValidator) Reset() { privVal.Save() } -func (privVal *DefaultPrivValidator) GetAddress() []byte { - return privVal.Address() -} - -func (privVal *DefaultPrivValidator) SignVote(chainID string, vote *Vote) error { +// SignVote signs a canonical representation of the vote, along with the chainID. +func (privVal *PrivValidatorFS) SignVote(chainID string, vote *Vote) error { privVal.mtx.Lock() defer privVal.mtx.Unlock() signature, err := privVal.signBytesHRS(vote.Height, vote.Round, voteToStep(vote), SignBytes(chainID, vote)) @@ -225,7 +180,8 @@ func (privVal *DefaultPrivValidator) SignVote(chainID string, vote *Vote) error return nil } -func (privVal *DefaultPrivValidator) SignProposal(chainID string, proposal *Proposal) error { +// SignProposal signs a canonical representation of the proposal, along with the chainID. +func (privVal *PrivValidatorFS) SignProposal(chainID string, proposal *Proposal) error { privVal.mtx.Lock() defer privVal.mtx.Unlock() signature, err := privVal.signBytesHRS(proposal.Height, proposal.Round, stepPropose, SignBytes(chainID, proposal)) @@ -236,8 +192,17 @@ func (privVal *DefaultPrivValidator) SignProposal(chainID string, proposal *Prop return nil } +// SignHeartbeat signs a canonical representation of the heartbeat, along with the chainID. +func (privVal *PrivValidatorFS) SignHeartbeat(chainID string, heartbeat *Heartbeat) error { + privVal.mtx.Lock() + defer privVal.mtx.Unlock() + var err error + heartbeat.Signature, err = privVal.Signer.Sign(SignBytes(chainID, heartbeat)) + return err +} + // check if there's a regression. Else sign and write the hrs+signature to disk -func (privVal *DefaultPrivValidator) signBytesHRS(height, round int, step int8, signBytes []byte) (crypto.Signature, error) { +func (privVal *PrivValidatorFS) signBytesHRS(height, round int, step int8, signBytes []byte) (crypto.Signature, error) { sig := crypto.Signature{} info := privVal.Info // If height regression, err @@ -287,32 +252,68 @@ func (privVal *DefaultPrivValidator) signBytesHRS(height, round int, step int8, privVal.save() return sig, nil +} +// String returns a string representation of the PrivValidatorFS. +func (privVal *PrivValidatorFS) String() string { + info := privVal.Info + return fmt.Sprintf("PrivValidator{%v LH:%v, LR:%v, LS:%v}", privVal.Address(), info.LastHeight, info.LastRound, info.LastStep) } -func (privVal *DefaultPrivValidator) SignHeartbeat(chainID string, heartbeat *Heartbeat) error { - privVal.mtx.Lock() - defer privVal.mtx.Unlock() - var err error - heartbeat.Signature, err = privVal.Signer.Sign(SignBytes(chainID, heartbeat)) - return err +//------------------------------------- + +// ValidatorID contains the identity of the validator. +type ValidatorID struct { + Address data.Bytes `json:"address"` + PubKey crypto.PubKey `json:"pub_key"` } -func (privVal *DefaultPrivValidator) String() string { - info := privVal.Info - return fmt.Sprintf("PrivValidator{%v LH:%v, LR:%v, LS:%v}", info.Address, info.LastHeight, info.LastRound, info.LastStep) +// LastSignedInfo contains information about the latest +// data signed by a validator to help prevent double signing. +type LastSignedInfo struct { + LastHeight int `json:"last_height"` + LastRound int `json:"last_round"` + LastStep int8 `json:"last_step"` + LastSignature crypto.Signature `json:"last_signature,omitempty"` // so we dont lose signatures + LastSignBytes data.Bytes `json:"last_signbytes,omitempty"` // so we dont lose signatures +} + +// Signer is an interface that defines how to sign messages. +// It is the caller's duty to verify the msg before calling Sign, +// eg. to avoid double signing. +// Currently, the only callers are SignVote, SignProposal, and SignHeartbeat. +type Signer interface { + Sign(msg []byte) (crypto.Signature, error) +} + +// DefaultSigner implements Signer. +// It uses a standard, unencrypted crypto.PrivKey. +type DefaultSigner struct { + PrivKey crypto.PrivKey `json:"priv_key"` +} + +// NewDefaultSigner returns an instance of DefaultSigner. +func NewDefaultSigner(priv crypto.PrivKey) *DefaultSigner { + return &DefaultSigner{ + PrivKey: priv, + } +} + +// Sign implements Signer. It signs the byte slice with a private key. +func (ds *DefaultSigner) Sign(msg []byte) (crypto.Signature, error) { + return ds.PrivKey.Sign(msg), nil } //------------------------------------- -type PrivValidatorsByAddress []*DefaultPrivValidator +type PrivValidatorsByAddress []*PrivValidatorFS func (pvs PrivValidatorsByAddress) Len() int { return len(pvs) } func (pvs PrivValidatorsByAddress) Less(i, j int) bool { - return bytes.Compare(pvs[i].Info.Address, pvs[j].Info.Address) == -1 + return bytes.Compare(pvs[i].Address(), pvs[j].Address()) == -1 } func (pvs PrivValidatorsByAddress) Swap(i, j int) { diff --git a/types/validator.go b/types/validator.go index 29cbfea27..e52831e27 100644 --- a/types/validator.go +++ b/types/validator.go @@ -106,10 +106,9 @@ func (vc validatorCodec) Compare(o1 interface{}, o2 interface{}) int { //-------------------------------------------------------------------------------- // For testing... -func RandValidator(randPower bool, minPower int64) (*Validator, *DefaultPrivValidator) { - privVal := GenPrivValidator() +func RandValidator(randPower bool, minPower int64) (*Validator, *PrivValidatorFS) { _, tempFilePath := cmn.Tempfile("priv_validator_") - privVal.SetFile(tempFilePath) + privVal := GenPrivValidatorFS(tempFilePath) votePower := minPower if randPower { votePower += int64(cmn.RandUint32()) diff --git a/types/validator_set.go b/types/validator_set.go index 5531f4af3..1e9e3e314 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -369,9 +369,9 @@ func (ac accumComparable) Less(o interface{}) bool { // For testing // NOTE: PrivValidator are in order. -func RandValidatorSet(numValidators int, votingPower int64) (*ValidatorSet, []*DefaultPrivValidator) { +func RandValidatorSet(numValidators int, votingPower int64) (*ValidatorSet, []*PrivValidatorFS) { vals := make([]*Validator, numValidators) - privValidators := make([]*DefaultPrivValidator, numValidators) + privValidators := make([]*PrivValidatorFS, numValidators) for i := 0; i < numValidators; i++ { val, privValidator := RandValidator(false, votingPower) vals[i] = val From 4382c8d28be23f14ff5a7f33b5978a2e1eabab35 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 18 Sep 2017 23:16:14 -0400 Subject: [PATCH 75/92] fix tests --- config/toml.go | 26 ++++++++++++++--------- consensus/byzantine_test.go | 35 +++++++++++++------------------ consensus/common_test.go | 30 +++++++++++++------------- consensus/height_vote_set_test.go | 4 ++-- consensus/reactor_test.go | 14 ++++++------- consensus/replay_test.go | 24 ++++++++++----------- consensus/state_test.go | 20 ++++++++---------- rpc/test/helpers.go | 2 +- types/priv_validator.go | 24 +++++++++++++++++++++ types/priv_validator_test.go | 10 +++++---- types/proposal_test.go | 4 ++-- types/vote_set_test.go | 4 ++-- 12 files changed, 110 insertions(+), 87 deletions(-) diff --git a/config/toml.go b/config/toml.go index 5dcbe5332..02ae3fc61 100644 --- a/config/toml.go +++ b/config/toml.go @@ -127,16 +127,22 @@ var testGenesis = `{ }` var testPrivValidator = `{ - "address": "D028C9981F7A87F3093672BF0D5B0E2A1B3ED456", - "pub_key": { - "type": "ed25519", - "data": "3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8" + "id": { + "address": "D028C9981F7A87F3093672BF0D5B0E2A1B3ED456", + "pub_key": { + "type": "ed25519", + "data": "3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8" + } }, - "priv_key": { - "type": "ed25519", - "data": "27F82582AEFAE7AB151CFB01C48BB6C1A0DA78F9BDDA979A9F70A84D074EB07D3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8" + "signer": { + "priv_key": { + "type": "ed25519", + "data": "27F82582AEFAE7AB151CFB01C48BB6C1A0DA78F9BDDA979A9F70A84D074EB07D3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8" + } }, - "last_height": 0, - "last_round": 0, - "last_step": 0 + "info": { + "last_height": 0, + "last_round": 0, + "last_step": 0 + } }` diff --git a/consensus/byzantine_test.go b/consensus/byzantine_test.go index 81a4ab42c..d8b6b0495 100644 --- a/consensus/byzantine_test.go +++ b/consensus/byzantine_test.go @@ -5,6 +5,8 @@ import ( "testing" "time" + crypto "github.com/tendermint/go-crypto" + data "github.com/tendermint/go-wire/data" "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/types" . "github.com/tendermint/tmlibs/common" @@ -53,7 +55,7 @@ func TestByzantine(t *testing.T) { eventLogger := logger.With("module", "events") for i := 0; i < N; i++ { if i == 0 { - css[i].privValidator = NewByzantinePrivValidator(css[i].privValidator.(*types.PrivValidator)) + css[i].privValidator = NewByzantinePrivValidator(css[i].privValidator) // make byzantine css[i].decideProposal = func(j int) func(int, int) { return func(height, round int) { @@ -257,47 +259,38 @@ func (br *ByzantineReactor) Receive(chID byte, peer p2p.Peer, msgBytes []byte) { // byzantine privValidator type ByzantinePrivValidator struct { - Address []byte `json:"address"` - types.Signer `json:"-"` + types.Signer - mtx sync.Mutex + pv types.PrivValidator } // Return a priv validator that will sign anything -func NewByzantinePrivValidator(pv *types.PrivValidator) *ByzantinePrivValidator { +func NewByzantinePrivValidator(pv types.PrivValidator) *ByzantinePrivValidator { return &ByzantinePrivValidator{ - Address: pv.Address, - Signer: pv.Signer, + Signer: pv.(*types.PrivValidatorFS).Signer, + pv: pv, } } -func (privVal *ByzantinePrivValidator) GetAddress() []byte { - return privVal.Address +func (privVal *ByzantinePrivValidator) Address() data.Bytes { + return privVal.pv.Address() } -func (privVal *ByzantinePrivValidator) SignVote(chainID string, vote *types.Vote) (err error) { - privVal.mtx.Lock() - defer privVal.mtx.Unlock() +func (privVal *ByzantinePrivValidator) PubKey() crypto.PubKey { + return privVal.pv.PubKey() +} - // Sign +func (privVal *ByzantinePrivValidator) SignVote(chainID string, vote *types.Vote) (err error) { vote.Signature, err = privVal.Sign(types.SignBytes(chainID, vote)) return err } func (privVal *ByzantinePrivValidator) SignProposal(chainID string, proposal *types.Proposal) (err error) { - privVal.mtx.Lock() - defer privVal.mtx.Unlock() - - // Sign proposal.Signature, err = privVal.Sign(types.SignBytes(chainID, proposal)) return nil } func (privVal *ByzantinePrivValidator) SignHeartbeat(chainID string, heartbeat *types.Heartbeat) (err error) { - privVal.mtx.Lock() - defer privVal.mtx.Unlock() - - // Sign heartbeat.Signature, err = privVal.Sign(types.SignBytes(chainID, heartbeat)) return nil } diff --git a/consensus/common_test.go b/consensus/common_test.go index 09b113553..bf56b8bff 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -50,12 +50,12 @@ type validatorStub struct { Index int // Validator index. NOTE: we don't assume validator set changes. Height int Round int - *types.PrivValidator + types.PrivValidator } var testMinPower = 10 -func NewValidatorStub(privValidator *types.PrivValidator, valIndex int) *validatorStub { +func NewValidatorStub(privValidator types.PrivValidator, valIndex int) *validatorStub { return &validatorStub{ Index: valIndex, PrivValidator: privValidator, @@ -65,7 +65,7 @@ func NewValidatorStub(privValidator *types.PrivValidator, valIndex int) *validat func (vs *validatorStub) signVote(voteType byte, hash []byte, header types.PartSetHeader) (*types.Vote, error) { vote := &types.Vote{ ValidatorIndex: vs.Index, - ValidatorAddress: vs.PrivValidator.Address, + ValidatorAddress: vs.PrivValidator.Address(), Height: vs.Height, Round: vs.Round, Type: voteType, @@ -142,7 +142,7 @@ func signAddVotes(to *ConsensusState, voteType byte, hash []byte, header types.P func validatePrevote(t *testing.T, cs *ConsensusState, round int, privVal *validatorStub, blockHash []byte) { prevotes := cs.Votes.Prevotes(round) var vote *types.Vote - if vote = prevotes.GetByAddress(privVal.Address); vote == nil { + if vote = prevotes.GetByAddress(privVal.Address()); vote == nil { panic("Failed to find prevote from validator") } if blockHash == nil { @@ -159,7 +159,7 @@ func validatePrevote(t *testing.T, cs *ConsensusState, round int, privVal *valid func validateLastPrecommit(t *testing.T, cs *ConsensusState, privVal *validatorStub, blockHash []byte) { votes := cs.LastCommit var vote *types.Vote - if vote = votes.GetByAddress(privVal.Address); vote == nil { + if vote = votes.GetByAddress(privVal.Address()); vote == nil { panic("Failed to find precommit from validator") } if !bytes.Equal(vote.BlockID.Hash, blockHash) { @@ -170,7 +170,7 @@ func validateLastPrecommit(t *testing.T, cs *ConsensusState, privVal *validatorS func validatePrecommit(t *testing.T, cs *ConsensusState, thisRound, lockRound int, privVal *validatorStub, votedBlockHash, lockedBlockHash []byte) { precommits := cs.Votes.Precommits(thisRound) var vote *types.Vote - if vote = precommits.GetByAddress(privVal.Address); vote == nil { + if vote = precommits.GetByAddress(privVal.Address()); vote == nil { panic("Failed to find precommit from validator") } @@ -225,11 +225,11 @@ func subscribeToVoter(cs *ConsensusState, addr []byte) chan interface{} { //------------------------------------------------------------------------------- // consensus states -func newConsensusState(state *sm.State, pv *types.PrivValidator, app abci.Application) *ConsensusState { +func newConsensusState(state *sm.State, pv types.PrivValidator, app abci.Application) *ConsensusState { return newConsensusStateWithConfig(config, state, pv, app) } -func newConsensusStateWithConfig(thisConfig *cfg.Config, state *sm.State, pv *types.PrivValidator, app abci.Application) *ConsensusState { +func newConsensusStateWithConfig(thisConfig *cfg.Config, state *sm.State, pv types.PrivValidator, app abci.Application) *ConsensusState { // Get BlockStore blockDB := dbm.NewMemDB() blockStore := bc.NewBlockStore(blockDB) @@ -258,10 +258,10 @@ func newConsensusStateWithConfig(thisConfig *cfg.Config, state *sm.State, pv *ty return cs } -func loadPrivValidator(config *cfg.Config) *types.PrivValidator { +func loadPrivValidator(config *cfg.Config) *types.PrivValidatorFS { privValidatorFile := config.PrivValidatorFile() ensureDir(path.Dir(privValidatorFile), 0700) - privValidator := types.LoadOrGenPrivValidator(privValidatorFile) + privValidator := types.LoadOrGenPrivValidatorFS(privValidatorFile) privValidator.Reset() return privValidator } @@ -364,12 +364,12 @@ func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerF state.Save() thisConfig := ResetConfig(Fmt("%s_%d", testName, i)) ensureDir(path.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal - var privVal *types.PrivValidator + var privVal types.PrivValidator if i < nValidators { privVal = privVals[i] } else { _, tempFilePath := Tempfile("priv_validator_") - privVal = types.GenPrivValidator(tempFilePath) + privVal = types.GenPrivValidatorFS(tempFilePath) } css[i] = newConsensusStateWithConfig(thisConfig, state, privVal, appFunc()) @@ -392,9 +392,9 @@ func getSwitchIndex(switches []*p2p.Switch, peer p2p.Peer) int { //------------------------------------------------------------------------------- // genesis -func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.GenesisDoc, []*types.PrivValidator) { +func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.GenesisDoc, []*types.PrivValidatorFS) { validators := make([]types.GenesisValidator, numValidators) - privValidators := make([]*types.PrivValidator, numValidators) + privValidators := make([]*types.PrivValidatorFS, numValidators) for i := 0; i < numValidators; i++ { val, privVal := types.RandValidator(randPower, minPower) validators[i] = types.GenesisValidator{ @@ -411,7 +411,7 @@ func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.G }, privValidators } -func randGenesisState(numValidators int, randPower bool, minPower int64) (*sm.State, []*types.PrivValidator) { +func randGenesisState(numValidators int, randPower bool, minPower int64) (*sm.State, []*types.PrivValidatorFS) { genDoc, privValidators := randGenesisDoc(numValidators, randPower, minPower) db := dbm.NewMemDB() s0 := sm.MakeGenesisState(db, genDoc) diff --git a/consensus/height_vote_set_test.go b/consensus/height_vote_set_test.go index 29751b40a..3d22a1dab 100644 --- a/consensus/height_vote_set_test.go +++ b/consensus/height_vote_set_test.go @@ -44,10 +44,10 @@ func TestPeerCatchupRounds(t *testing.T) { } -func makeVoteHR(t *testing.T, height, round int, privVals []*types.PrivValidator, valIndex int) *types.Vote { +func makeVoteHR(t *testing.T, height, round int, privVals []*types.PrivValidatorFS, valIndex int) *types.Vote { privVal := privVals[valIndex] vote := &types.Vote{ - ValidatorAddress: privVal.Address, + ValidatorAddress: privVal.Address(), ValidatorIndex: valIndex, Height: height, Round: round, diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index b1f9a0a5b..752c759f1 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -119,7 +119,7 @@ func TestVotingPowerChange(t *testing.T) { // map of active validators activeVals := make(map[string]struct{}) for i := 0; i < nVals; i++ { - activeVals[string(css[i].privValidator.GetAddress())] = struct{}{} + activeVals[string(css[i].privValidator.Address())] = struct{}{} } // wait till everyone makes block 1 @@ -132,7 +132,7 @@ func TestVotingPowerChange(t *testing.T) { //--------------------------------------------------------------------------- t.Log("---------------------------- Testing changing the voting power of one validator a few times") - val1PubKey := css[0].privValidator.(*types.PrivValidator).PubKey + val1PubKey := css[0].privValidator.PubKey() updateValidatorTx := dummy.MakeValSetChangeTx(val1PubKey.Bytes(), 25) previousTotalVotingPower := css[0].GetRoundState().LastValidators.TotalVotingPower() @@ -180,7 +180,7 @@ func TestValidatorSetChanges(t *testing.T) { // map of active validators activeVals := make(map[string]struct{}) for i := 0; i < nVals; i++ { - activeVals[string(css[i].privValidator.GetAddress())] = struct{}{} + activeVals[string(css[i].privValidator.Address())] = struct{}{} } // wait till everyone makes block 1 @@ -193,7 +193,7 @@ func TestValidatorSetChanges(t *testing.T) { //--------------------------------------------------------------------------- t.Log("---------------------------- Testing adding one validator") - newValidatorPubKey1 := css[nVals].privValidator.(*types.PrivValidator).PubKey + newValidatorPubKey1 := css[nVals].privValidator.PubKey() newValidatorTx1 := dummy.MakeValSetChangeTx(newValidatorPubKey1.Bytes(), uint64(testMinPower)) // wait till everyone makes block 2 @@ -219,7 +219,7 @@ func TestValidatorSetChanges(t *testing.T) { //--------------------------------------------------------------------------- t.Log("---------------------------- Testing changing the voting power of one validator") - updateValidatorPubKey1 := css[nVals].privValidator.(*types.PrivValidator).PubKey + updateValidatorPubKey1 := css[nVals].privValidator.PubKey() updateValidatorTx1 := dummy.MakeValSetChangeTx(updateValidatorPubKey1.Bytes(), 25) previousTotalVotingPower := css[nVals].GetRoundState().LastValidators.TotalVotingPower() @@ -235,10 +235,10 @@ func TestValidatorSetChanges(t *testing.T) { //--------------------------------------------------------------------------- t.Log("---------------------------- Testing adding two validators at once") - newValidatorPubKey2 := css[nVals+1].privValidator.(*types.PrivValidator).PubKey + newValidatorPubKey2 := css[nVals+1].privValidator.PubKey() newValidatorTx2 := dummy.MakeValSetChangeTx(newValidatorPubKey2.Bytes(), uint64(testMinPower)) - newValidatorPubKey3 := css[nVals+2].privValidator.(*types.PrivValidator).PubKey + newValidatorPubKey3 := css[nVals+2].privValidator.PubKey() newValidatorTx3 := dummy.MakeValSetChangeTx(newValidatorPubKey3.Bytes(), uint64(testMinPower)) waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, newValidatorTx2, newValidatorTx3) diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 03ca6c8d7..d2d848bc5 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -162,8 +162,8 @@ LOOP: cs.Wait() } -func toPV(pv PrivValidator) *types.PrivValidator { - return pv.(*types.PrivValidator) +func toPV(pv types.PrivValidator) *types.PrivValidatorFS { + return pv.(*types.PrivValidatorFS) } func setupReplayTest(t *testing.T, thisCase *testCase, nLines int, crashAfter bool) (*ConsensusState, chan interface{}, string, string) { @@ -184,10 +184,10 @@ func setupReplayTest(t *testing.T, thisCase *testCase, nLines int, crashAfter bo cs := fixedConsensusStateDummy() // set the last step according to when we crashed vs the wal - toPV(cs.privValidator).LastHeight = 1 // first block - toPV(cs.privValidator).LastStep = thisCase.stepMap[lineStep] + toPV(cs.privValidator).Info.LastHeight = 1 // first block + toPV(cs.privValidator).Info.LastStep = thisCase.stepMap[lineStep] - t.Logf("[WARN] setupReplayTest LastStep=%v", toPV(cs.privValidator).LastStep) + t.Logf("[WARN] setupReplayTest LastStep=%v", toPV(cs.privValidator).Info.LastStep) newBlockCh := subscribeToEvent(cs.evsw, "tester", types.EventStringNewBlock(), 1) @@ -230,8 +230,8 @@ func TestWALCrashBeforeWritePropose(t *testing.T) { msg := readTimedWALMessage(t, proposalMsg) proposal := msg.Msg.(msgInfo).Msg.(*ProposalMessage) // Set LastSig - toPV(cs.privValidator).LastSignBytes = types.SignBytes(cs.state.ChainID, proposal.Proposal) - toPV(cs.privValidator).LastSignature = proposal.Proposal.Signature + toPV(cs.privValidator).Info.LastSignBytes = types.SignBytes(cs.state.ChainID, proposal.Proposal) + toPV(cs.privValidator).Info.LastSignature = proposal.Proposal.Signature runReplayTest(t, cs, walFile, newBlockCh, thisCase, lineNum) } } @@ -255,8 +255,8 @@ func testReplayCrashBeforeWriteVote(t *testing.T, thisCase *testCase, lineNum in msg := readTimedWALMessage(t, voteMsg) vote := msg.Msg.(msgInfo).Msg.(*VoteMessage) // Set LastSig - toPV(cs.privValidator).LastSignBytes = types.SignBytes(cs.state.ChainID, vote.Vote) - toPV(cs.privValidator).LastSignature = vote.Vote.Signature + toPV(cs.privValidator).Info.LastSignBytes = types.SignBytes(cs.state.ChainID, vote.Vote) + toPV(cs.privValidator).Info.LastSignature = vote.Vote.Signature }) runReplayTest(t, cs, walFile, newBlockCh, thisCase, lineNum) } @@ -317,7 +317,7 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) { walFile := writeWAL(string(walBody)) config.Consensus.SetWalFile(walFile) - privVal := types.LoadPrivValidator(config.PrivValidatorFile()) + privVal := types.LoadPrivValidatorFS(config.PrivValidatorFile()) wal, err := NewWAL(walFile, false) if err != nil { @@ -332,7 +332,7 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) { t.Fatalf(err.Error()) } - state, store := stateAndStore(config, privVal.PubKey) + state, store := stateAndStore(config, privVal.PubKey()) store.chain = chain store.commits = commits @@ -346,7 +346,7 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) { // run nBlocks against a new client to build up the app state. // use a throwaway tendermint state proxyApp := proxy.NewAppConns(clientCreator2, nil) - state, _ := stateAndStore(config, privVal.PubKey) + state, _ := stateAndStore(config, privVal.PubKey()) buildAppStateFromChain(proxyApp, state, chain, nBlocks, mode) } diff --git a/consensus/state_test.go b/consensus/state_test.go index 9ae052033..977d6445c 100644 --- a/consensus/state_test.go +++ b/consensus/state_test.go @@ -65,7 +65,7 @@ func TestProposerSelection0(t *testing.T) { // lets commit a block and ensure proposer for the next height is correct prop := cs1.GetRoundState().Validators.GetProposer() - if !bytes.Equal(prop.Address, cs1.privValidator.GetAddress()) { + if !bytes.Equal(prop.Address, cs1.privValidator.Address()) { t.Fatalf("expected proposer to be validator %d. Got %X", 0, prop.Address) } @@ -79,7 +79,7 @@ func TestProposerSelection0(t *testing.T) { <-newRoundCh prop = cs1.GetRoundState().Validators.GetProposer() - if !bytes.Equal(prop.Address, vss[1].Address) { + if !bytes.Equal(prop.Address, vss[1].Address()) { panic(Fmt("expected proposer to be validator %d. Got %X", 1, prop.Address)) } } @@ -100,7 +100,7 @@ func TestProposerSelection2(t *testing.T) { // everyone just votes nil. we get a new proposer each round for i := 0; i < len(vss); i++ { prop := cs1.GetRoundState().Validators.GetProposer() - if !bytes.Equal(prop.Address, vss[(i+2)%len(vss)].Address) { + if !bytes.Equal(prop.Address, vss[(i+2)%len(vss)].Address()) { panic(Fmt("expected proposer to be validator %d. Got %X", (i+2)%len(vss), prop.Address)) } @@ -502,8 +502,6 @@ func TestLockPOLRelock(t *testing.T) { newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1) newBlockCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewBlockHeader(), 1) - t.Logf("vs2 last round %v", vs2.PrivValidator.LastRound) - // everything done from perspective of cs1 /* @@ -615,7 +613,7 @@ func TestLockPOLUnlock(t *testing.T) { timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1) newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1) unlockCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringUnlock(), 1) - voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress()) + voteCh := subscribeToVoter(cs1, cs1.privValidator.Address()) // everything done from perspective of cs1 @@ -709,7 +707,7 @@ func TestLockPOLSafety1(t *testing.T) { timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1) timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1) newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1) - voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress()) + voteCh := subscribeToVoter(cs1, cs1.privValidator.Address()) // start round and wait for propose and prevote startTestRound(cs1, cs1.Height, 0) @@ -831,7 +829,7 @@ func TestLockPOLSafety2(t *testing.T) { timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1) newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1) unlockCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringUnlock(), 1) - voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress()) + voteCh := subscribeToVoter(cs1, cs1.privValidator.Address()) // the block for R0: gets polkad but we miss it // (even though we signed it, shhh) @@ -923,7 +921,7 @@ func TestSlashingPrevotes(t *testing.T) { proposalCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringCompleteProposal() , 1) timeoutWaitCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringTimeoutWait() , 1) newRoundCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringNewRound() , 1) - voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress()) + voteCh := subscribeToVoter(cs1, cs1.privValidator.Address()) // start round and wait for propose and prevote startTestRound(cs1, cs1.Height, 0) @@ -958,7 +956,7 @@ func TestSlashingPrecommits(t *testing.T) { proposalCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringCompleteProposal() , 1) timeoutWaitCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringTimeoutWait() , 1) newRoundCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringNewRound() , 1) - voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress()) + voteCh := subscribeToVoter(cs1, cs1.privValidator.Address()) // start round and wait for propose and prevote startTestRound(cs1, cs1.Height, 0) @@ -1005,7 +1003,7 @@ func TestHalt1(t *testing.T) { timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1) newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1) newBlockCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewBlock(), 1) - voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress()) + voteCh := subscribeToVoter(cs1, cs1.privValidator.Address()) // start round and wait for propose and prevote startTestRound(cs1, cs1.Height, 0) diff --git a/rpc/test/helpers.go b/rpc/test/helpers.go index 132c4d4d9..82b14ce34 100644 --- a/rpc/test/helpers.go +++ b/rpc/test/helpers.go @@ -79,7 +79,7 @@ func NewTendermint(app abci.Application) *nm.Node { logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout)) logger = log.NewFilter(logger, log.AllowError()) privValidatorFile := config.PrivValidatorFile() - privValidator := types.LoadOrGenPrivValidator(privValidatorFile) + privValidator := types.LoadOrGenPrivValidatorFS(privValidatorFile) papp := proxy.NewLocalClientCreator(app) node := nm.NewNode(config, privValidator, papp, logger) return node diff --git a/types/priv_validator.go b/types/priv_validator.go index a90f620b6..e5b537789 100644 --- a/types/priv_validator.go +++ b/types/priv_validator.go @@ -157,6 +157,30 @@ func (privVal *PrivValidatorFS) save() { } } +// UnmarshalJSON unmarshals the given jsonString +// into a PrivValidatorFS using a DefaultSigner. +func (pv *PrivValidatorFS) UnmarshalJSON(jsonString []byte) error { + idAndInfo := &struct { + ID ValidatorID `json:"id"` + Info LastSignedInfo `json:"info"` + }{} + if err := json.Unmarshal(jsonString, idAndInfo); err != nil { + return err + } + + signer := &struct { + Signer *DefaultSigner `json:"signer"` + }{} + if err := json.Unmarshal(jsonString, signer); err != nil { + return err + } + + pv.ID = idAndInfo.ID + pv.Info = idAndInfo.Info + pv.Signer = signer.Signer + return nil +} + // Reset resets all fields in the PrivValidatorFS.Info. // NOTE: Unsafe! func (privVal *PrivValidatorFS) Reset() { diff --git a/types/priv_validator_test.go b/types/priv_validator_test.go index d0fcfd185..58a43d489 100644 --- a/types/priv_validator_test.go +++ b/types/priv_validator_test.go @@ -29,12 +29,14 @@ func TestLoadValidator(t *testing.T) { require.Nil(err, "%+v", err) serialized := fmt.Sprintf(`{ - "info": { + "id": { "address": "%s", "pub_key": { "type": "ed25519", "data": "%s" - }, + } + }, + "info": { "last_height": 0, "last_round": 0, "last_step": 0, @@ -48,14 +50,14 @@ func TestLoadValidator(t *testing.T) { } }`, addrStr, pubStr, privStr) - val := DefaultPrivValidator{} + val := PrivValidatorFS{} err = json.Unmarshal([]byte(serialized), &val) require.Nil(err, "%+v", err) // make sure the values match assert.EqualValues(addrBytes, val.Address()) assert.EqualValues(pubKey, val.PubKey()) - valPrivKey := val.Signer.PrivKey + valPrivKey := val.Signer.(*DefaultSigner).PrivKey assert.EqualValues(privKey, valPrivKey) // export it and make sure it is the same diff --git a/types/proposal_test.go b/types/proposal_test.go index ce1bbc8d5..2e4e7e198 100644 --- a/types/proposal_test.go +++ b/types/proposal_test.go @@ -28,7 +28,7 @@ func BenchmarkProposalWriteSignBytes(b *testing.B) { } func BenchmarkProposalSign(b *testing.B) { - privVal := GenPrivValidator() + privVal := GenPrivValidatorFS("") for i := 0; i < b.N; i++ { privVal.Signer.Sign(SignBytes("test_chain_id", testProposal)) } @@ -36,7 +36,7 @@ func BenchmarkProposalSign(b *testing.B) { func BenchmarkProposalVerifySignature(b *testing.B) { signBytes := SignBytes("test_chain_id", testProposal) - privVal := GenPrivValidator() + privVal := GenPrivValidatorFS("") signature, _ := privVal.Signer.Sign(signBytes) pubKey := privVal.PubKey() diff --git a/types/vote_set_test.go b/types/vote_set_test.go index 716becf52..ab1456ed9 100644 --- a/types/vote_set_test.go +++ b/types/vote_set_test.go @@ -11,7 +11,7 @@ import ( ) // NOTE: privValidators are in order -func randVoteSet(height int, round int, type_ byte, numValidators int, votingPower int64) (*VoteSet, *ValidatorSet, []*DefaultPrivValidator) { +func randVoteSet(height int, round int, type_ byte, numValidators int, votingPower int64) (*VoteSet, *ValidatorSet, []*PrivValidatorFS) { valSet, privValidators := RandValidatorSet(numValidators, votingPower) return NewVoteSet("test_chain_id", height, round, type_, valSet), valSet, privValidators } @@ -59,7 +59,7 @@ func withBlockPartsHeader(vote *Vote, blockPartsHeader PartSetHeader) *Vote { return vote } -func signAddVote(privVal *DefaultPrivValidator, vote *Vote, voteSet *VoteSet) (bool, error) { +func signAddVote(privVal *PrivValidatorFS, vote *Vote, voteSet *VoteSet) (bool, error) { var err error vote.Signature, err = privVal.Signer.Sign(SignBytes(voteSet.ChainID(), vote)) if err != nil { From 147a18b34a7e5d5871e777b8393c5615aac63237 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 18 Sep 2017 23:20:36 -0400 Subject: [PATCH 76/92] fix some comments --- cmd/tendermint/commands/run_node.go | 2 ++ cmd/tendermint/main.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index d4acd98df..3befae279 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -43,6 +43,8 @@ func AddNodeFlags(cmd *cobra.Command) { // It allows other projects to make Tendermint binaries with custom signers and applications. type FuncSignerAndApp func(*cfg.Config) (types.PrivValidator, proxy.ClientCreator) +// DefaultSignerAndApp is a default FuncSignerAndApp that returns a PrivValidatorFS +// and a DefaultClientCreator using the relevant fields from the config. func DefaultSignerAndApp(config *cfg.Config) (types.PrivValidator, proxy.ClientCreator) { privValidator := types.LoadOrGenPrivValidatorFS(config.PrivValidatorFile()) clientCreator := proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()) diff --git a/cmd/tendermint/main.go b/cmd/tendermint/main.go index 7cc1b2183..1e2c00051 100644 --- a/cmd/tendermint/main.go +++ b/cmd/tendermint/main.go @@ -27,7 +27,7 @@ func main() { // * Use an external signer for their validators // * Supply an in-proc abci app // can copy this file and use something other than the - // default SignerAndApp function + // DefaultSignerAndApp function signerAndApp := cmd.DefaultSignerAndApp // Create & start node From 779c2a22d0bddb4e2c7949dbc42128f0cecf217a Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 20 Sep 2017 18:29:36 -0400 Subject: [PATCH 77/92] node: NewNode takes DBProvider and GenDocProvider --- cmd/tendermint/commands/run_node.go | 10 ++- consensus/common_test.go | 8 +- consensus/replay_file.go | 7 +- consensus/replay_test.go | 2 +- node/node.go | 128 +++++++++++++++++++++++----- node/node_test.go | 8 +- rpc/test/helpers.go | 7 +- state/execution_test.go | 3 +- state/state.go | 34 +++++--- state/state_test.go | 3 +- 10 files changed, 164 insertions(+), 46 deletions(-) diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index 3befae279..6a793706f 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -67,7 +67,15 @@ func NewRunNodeCmd(signerAndApp FuncSignerAndApp) *cobra.Command { // Create & start node privVal, clientCreator := signerAndApp(config) - n := node.NewNode(config, privVal, clientCreator, logger.With("module", "node")) + n, err := node.NewNode(config, + privVal, + clientCreator, + node.DefaultGenesisDocProviderFunc(config), + node.DefaultDBProvider, + logger.With("module", "node")) + if err != nil { + return fmt.Errorf("Failed to create node: %v", err) + } if _, err := n.Start(); err != nil { return fmt.Errorf("Failed to start node: %v", err) diff --git a/consensus/common_test.go b/consensus/common_test.go index bf56b8bff..a746ec59c 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -268,7 +268,7 @@ func loadPrivValidator(config *cfg.Config) *types.PrivValidatorFS { func fixedConsensusStateDummy() *ConsensusState { stateDB := dbm.NewMemDB() - state := sm.MakeGenesisStateFromFile(stateDB, config.GenesisFile()) + state, _ := sm.MakeGenesisStateFromFile(stateDB, config.GenesisFile()) state.SetLogger(log.TestingLogger().With("module", "state")) privValidator := loadPrivValidator(config) cs := newConsensusState(state, privValidator, dummy.NewDummyApplication()) @@ -338,7 +338,7 @@ func randConsensusNet(nValidators int, testName string, tickerFunc func() Timeou logger := consensusLogger() for i := 0; i < nValidators; i++ { db := dbm.NewMemDB() // each state needs its own db - state := sm.MakeGenesisState(db, genDoc) + state, _ := sm.MakeGenesisState(db, genDoc) state.SetLogger(logger.With("module", "state", "validator", i)) state.Save() thisConfig := ResetConfig(Fmt("%s_%d", testName, i)) @@ -359,7 +359,7 @@ func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerF css := make([]*ConsensusState, nPeers) for i := 0; i < nPeers; i++ { db := dbm.NewMemDB() // each state needs its own db - state := sm.MakeGenesisState(db, genDoc) + state, _ := sm.MakeGenesisState(db, genDoc) state.SetLogger(log.TestingLogger().With("module", "state")) state.Save() thisConfig := ResetConfig(Fmt("%s_%d", testName, i)) @@ -414,7 +414,7 @@ func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.G func randGenesisState(numValidators int, randPower bool, minPower int64) (*sm.State, []*types.PrivValidatorFS) { genDoc, privValidators := randGenesisDoc(numValidators, randPower, minPower) db := dbm.NewMemDB() - s0 := sm.MakeGenesisState(db, genDoc) + s0, _ := sm.MakeGenesisState(db, genDoc) s0.SetLogger(log.TestingLogger().With("module", "state")) s0.Save() return s0, privValidators diff --git a/consensus/replay_file.go b/consensus/replay_file.go index 39ce47c5e..1182aaf04 100644 --- a/consensus/replay_file.go +++ b/consensus/replay_file.go @@ -241,12 +241,15 @@ func newConsensusStateForReplay(config cfg.BaseConfig, csConfig *cfg.ConsensusCo // Get State stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir()) - state := sm.MakeGenesisStateFromFile(stateDB, config.GenesisFile()) + state, err := sm.MakeGenesisStateFromFile(stateDB, config.GenesisFile()) + if err != nil { + cmn.Exit(err.Error()) + } // Create proxyAppConn connection (consensus, mempool, query) clientCreator := proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()) proxyApp := proxy.NewAppConns(clientCreator, NewHandshaker(state, blockStore)) - _, err := proxyApp.Start() + _, err = proxyApp.Start() if err != nil { cmn.Exit(cmn.Fmt("Error starting proxy app conns: %v", err)) } diff --git a/consensus/replay_test.go b/consensus/replay_test.go index d2d848bc5..c6504380d 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -558,7 +558,7 @@ func readPieceFromWAL(msgBytes []byte) (interface{}, error) { // fresh state and mock store func stateAndStore(config *cfg.Config, pubKey crypto.PubKey) (*sm.State, *mockBlockStore) { stateDB := dbm.NewMemDB() - state := sm.MakeGenesisStateFromFile(stateDB, config.GenesisFile()) + state, _ := sm.MakeGenesisStateFromFile(stateDB, config.GenesisFile()) state.SetLogger(log.TestingLogger().With("module", "state")) store := NewMockBlockStore(config, state.Params()) diff --git a/node/node.go b/node/node.go index 4f22ee923..761f4d5e4 100644 --- a/node/node.go +++ b/node/node.go @@ -3,6 +3,7 @@ package node import ( "bytes" "errors" + "fmt" "net" "net/http" "strings" @@ -10,11 +11,15 @@ import ( abci "github.com/tendermint/abci/types" crypto "github.com/tendermint/go-crypto" wire "github.com/tendermint/go-wire" + cmn "github.com/tendermint/tmlibs/common" + dbm "github.com/tendermint/tmlibs/db" + "github.com/tendermint/tmlibs/log" + bc "github.com/tendermint/tendermint/blockchain" cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/consensus" mempl "github.com/tendermint/tendermint/mempool" - p2p "github.com/tendermint/tendermint/p2p" + "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/proxy" rpccore "github.com/tendermint/tendermint/rpc/core" grpccore "github.com/tendermint/tendermint/rpc/grpc" @@ -26,13 +31,44 @@ import ( "github.com/tendermint/tendermint/state/txindex/null" "github.com/tendermint/tendermint/types" "github.com/tendermint/tendermint/version" - cmn "github.com/tendermint/tmlibs/common" - dbm "github.com/tendermint/tmlibs/db" - "github.com/tendermint/tmlibs/log" _ "net/http/pprof" ) +//------------------------------------------------------------------------------ + +// DBContext specifies config information for loading a new DB. +type DBContext struct { + ID string + Config *cfg.Config +} + +// DBProvider takes a DBContext and returns an instantiated DB. +type DBProvider func(*DBContext) (dbm.DB, error) + +// DefaultDBProvider returns a database using the DBBackend and DBDir +// specified in the ctx.Config. +func DefaultDBProvider(ctx *DBContext) (dbm.DB, error) { + return dbm.NewDB(ctx.ID, ctx.Config.DBBackend, ctx.Config.DBDir()), nil +} + +// GenesisDocProvider returns a GenesisDoc. +// It allows the GenesisDoc to be pulled from sources other than the +// filesystem, for instance from a distributed key-value store cluster. +type GenesisDocProvider func() (*types.GenesisDoc, error) + +// DefaultGenesisDocProviderFunc returns a GenesisDocProvider that loads +// the GenesisDoc from the config.GenesisFile() on the filesystem. +func DefaultGenesisDocProviderFunc(config *cfg.Config) GenesisDocProvider { + return func() (*types.GenesisDoc, error) { + return types.GenesisDocFromFile(config.GenesisFile()) + } +} + +//------------------------------------------------------------------------------ + +// Node is the highest level interface to a full Tendermint node. +// It includes all configuration information and running services. type Node struct { cmn.BaseService @@ -58,24 +94,55 @@ type Node struct { txIndexer txindex.TxIndexer } -func NewNodeDefault(config *cfg.Config, logger log.Logger) *Node { +// NewNodeDefault returns a Tendermint node with default settings for the +// PrivValidator, ClientCreator, GenesisDoc, and DBProvider, +func NewNodeDefault(config *cfg.Config, logger log.Logger) (*Node, error) { // Get PrivValidator privValidator := types.LoadOrGenPrivValidatorFS(config.PrivValidatorFile()) - return NewNode(config, privValidator, - proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()), logger) + return NewNode(config, + privValidator, + proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()), + DefaultGenesisDocProviderFunc(config), + DefaultDBProvider, + logger) } -func NewNode(config *cfg.Config, privValidator types.PrivValidator, clientCreator proxy.ClientCreator, logger log.Logger) *Node { +// NewNode returns a new, ready to go, Tendermint Node. +func NewNode(config *cfg.Config, + privValidator types.PrivValidator, + clientCreator proxy.ClientCreator, + genDocProvider GenesisDocProvider, + dbProvider DBProvider, + logger log.Logger) (*Node, error) { + // Get BlockStore - blockStoreDB := dbm.NewDB("blockstore", config.DBBackend, config.DBDir()) + blockStoreDB, err := dbProvider(&DBContext{"blockstore", config}) + if err != nil { + return nil, err + } blockStore := bc.NewBlockStore(blockStoreDB) consensusLogger := logger.With("module", "consensus") stateLogger := logger.With("module", "state") // Get State - stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir()) - state := sm.GetState(stateDB, config.GenesisFile()) + stateDB, err := dbProvider(&DBContext{"state", config}) + if err != nil { + return nil, err + } + state := sm.LoadState(stateDB) + if state == nil { + genDoc, err := genDocProvider() + if err != nil { + return nil, err + } + state, err = sm.MakeGenesisState(stateDB, genDoc) + if err != nil { + return nil, err + } + state.Save() + } + state.SetLogger(stateLogger) // Create the proxyApp, which manages connections (consensus, mempool, query) @@ -85,7 +152,7 @@ func NewNode(config *cfg.Config, privValidator types.PrivValidator, clientCreato proxyApp := proxy.NewAppConns(clientCreator, handshaker) proxyApp.SetLogger(logger.With("module", "proxy")) if _, err := proxyApp.Start(); err != nil { - cmn.Exit(cmn.Fmt("Error starting proxy app connections: %v", err)) + return nil, fmt.Errorf("Error starting proxy app connections: %v", err) } // reload the state (it may have been updated by the handshake) @@ -96,7 +163,10 @@ func NewNode(config *cfg.Config, privValidator types.PrivValidator, clientCreato var txIndexer txindex.TxIndexer switch config.TxIndex { case "kv": - store := dbm.NewDB("tx_index", config.DBBackend, config.DBDir()) + store, err := dbProvider(&DBContext{"tx_index", config}) + if err != nil { + return nil, err + } txIndexer = kv.NewTxIndex(store) default: txIndexer = &null.TxIndex{} @@ -109,9 +179,8 @@ func NewNode(config *cfg.Config, privValidator types.PrivValidator, clientCreato // Make event switch eventSwitch := types.NewEventSwitch() eventSwitch.SetLogger(logger.With("module", "types")) - _, err := eventSwitch.Start() - if err != nil { - cmn.Exit(cmn.Fmt("Failed to start switch: %v", err)) + if _, err := eventSwitch.Start(); err != nil { + return nil, fmt.Errorf("Failed to start switch: %v", err) } // Decide whether to fast-sync or not @@ -232,9 +301,10 @@ func NewNode(config *cfg.Config, privValidator types.PrivValidator, clientCreato txIndexer: txIndexer, } node.BaseService = *cmn.NewBaseService(logger, "Node", node) - return node + return node, nil } +// OnStart starts the Node. It implements cmn.Service. func (n *Node) OnStart() error { // Create & add listener protocol, address := ProtocolAndAddress(n.config.P2P.ListenAddress) @@ -270,6 +340,7 @@ func (n *Node) OnStart() error { return nil } +// OnStop stops the Node. It implements cmn.Service. func (n *Node) OnStop() { n.BaseService.OnStop() @@ -285,6 +356,7 @@ func (n *Node) OnStop() { } } +// RunForever waits for an interupt signal and stops the node. func (n *Node) RunForever() { // Sleep forever and then... cmn.TrapSignal(func() { @@ -292,15 +364,15 @@ func (n *Node) RunForever() { }) } -// Add the event switch to reactors, mempool, etc. +// SetEventSwitch adds the event switch to reactors, mempool, etc. func SetEventSwitch(evsw types.EventSwitch, eventables ...types.Eventable) { for _, e := range eventables { e.SetEventSwitch(evsw) } } -// Add a Listener to accept inbound peer connections. -// Add listeners before starting the Node. +// AddListener adds a listener to accept inbound peer connections. +// It should be called before starting the Node. // The first listener is the primary listener (in NodeInfo) func (n *Node) AddListener(l p2p.Listener) { n.sw.AddListener(l) @@ -360,39 +432,48 @@ func (n *Node) startRPC() ([]net.Listener, error) { return listeners, nil } +// Switch returns the Node's Switch. func (n *Node) Switch() *p2p.Switch { return n.sw } +// BlockStore returns the Node's BlockStore. func (n *Node) BlockStore() *bc.BlockStore { return n.blockStore } +// ConsensusState returns the Node's ConsensusState. func (n *Node) ConsensusState() *consensus.ConsensusState { return n.consensusState } +// ConsensusReactor returns the Node's ConsensusReactor. func (n *Node) ConsensusReactor() *consensus.ConsensusReactor { return n.consensusReactor } +// MempoolReactor returns the Node's MempoolReactor. func (n *Node) MempoolReactor() *mempl.MempoolReactor { return n.mempoolReactor } +// EventSwitch returns the Node's EventSwitch. func (n *Node) EventSwitch() types.EventSwitch { return n.evsw } -// XXX: for convenience +// PrivValidator returns the Node's PrivValidator. +// XXX: for convenience only! func (n *Node) PrivValidator() types.PrivValidator { return n.privValidator } +// GenesisDoc returns the Node's GenesisDoc. func (n *Node) GenesisDoc() *types.GenesisDoc { return n.genesisDoc } +// ProxyApp returns the Node's AppConns, representing its connections to the ABCI application. func (n *Node) ProxyApp() proxy.AppConns { return n.proxyApp } @@ -442,15 +523,18 @@ func (n *Node) makeNodeInfo() *p2p.NodeInfo { //------------------------------------------------------------------------------ +// NodeInfo returns the Node's Info from the Switch. func (n *Node) NodeInfo() *p2p.NodeInfo { return n.sw.NodeInfo() } +// DialSeeds dials the given seeds on the Switch. func (n *Node) DialSeeds(seeds []string) error { return n.sw.DialSeeds(n.addrBook, seeds) } -// Defaults to tcp +// ProtocolAndAddress returns the transport protocol +// and the ip address from the given string. Defaults to tcp. func ProtocolAndAddress(listenAddr string) (string, string) { protocol, address := "tcp", listenAddr parts := strings.SplitN(address, "://", 2) diff --git a/node/node_test.go b/node/node_test.go index 3c751af6e..1b6d27f47 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -4,15 +4,19 @@ import ( "testing" "time" - cfg "github.com/tendermint/tendermint/config" + "github.com/stretchr/testify/assert" + "github.com/tendermint/tmlibs/log" + + cfg "github.com/tendermint/tendermint/config" ) func TestNodeStartStop(t *testing.T) { config := cfg.ResetTestRoot("node_node_test") // Create & start node - n := NewNodeDefault(config, log.TestingLogger()) + n, err := NewNodeDefault(config, log.TestingLogger()) + assert.NoError(t, err, "expected no err on NewNodeDefault") n.Start() t.Logf("Started node %v", n.sw.NodeInfo()) diff --git a/rpc/test/helpers.go b/rpc/test/helpers.go index 82b14ce34..55e27f5b8 100644 --- a/rpc/test/helpers.go +++ b/rpc/test/helpers.go @@ -81,6 +81,11 @@ func NewTendermint(app abci.Application) *nm.Node { privValidatorFile := config.PrivValidatorFile() privValidator := types.LoadOrGenPrivValidatorFS(privValidatorFile) papp := proxy.NewLocalClientCreator(app) - node := nm.NewNode(config, privValidator, papp, logger) + node, err := nm.NewNode(config, privValidator, papp, + nm.DefaultGenesisDocProviderFunc(config), + nm.DefaultDBProvider, logger) + if err != nil { + panic(err) + } return node } diff --git a/state/execution_test.go b/state/execution_test.go index 3ebc05d37..2987667c5 100644 --- a/state/execution_test.go +++ b/state/execution_test.go @@ -55,13 +55,14 @@ func makeTxs(blockNum int) (txs []types.Tx) { } func state() *State { - return MakeGenesisState(dbm.NewMemDB(), &types.GenesisDoc{ + s, _ := MakeGenesisState(dbm.NewMemDB(), &types.GenesisDoc{ ChainID: chainID, Validators: []types.GenesisValidator{ types.GenesisValidator{privKey.PubKey(), 10000, "test"}, }, AppHash: nil, }) + return s } func makeBlock(num int, state *State) *types.Block { diff --git a/state/state.go b/state/state.go index 2ebbec10e..53ec8cc03 100644 --- a/state/state.go +++ b/state/state.go @@ -68,14 +68,17 @@ type State struct { // GetState loads the most recent state from the database, // or creates a new one from the given genesisFile and persists the result // to the database. -func GetState(stateDB dbm.DB, genesisFile string) *State { +func GetState(stateDB dbm.DB, genesisFile string) (*State, error) { + var err error state := LoadState(stateDB) if state == nil { - state = MakeGenesisStateFromFile(stateDB, genesisFile) + state, err = MakeGenesisStateFromFile(stateDB, genesisFile) + if err != nil { + return nil, err + } state.Save() } - - return state + return state, nil } // LoadState loads the State from the database. @@ -316,25 +319,34 @@ func (vi *ValidatorsInfo) Bytes() []byte { // file. // // Used during replay and in tests. -func MakeGenesisStateFromFile(db dbm.DB, genDocFile string) *State { +func MakeGenesisStateFromFile(db dbm.DB, genDocFile string) (*State, error) { + genDoc, err := MakeGenesisDocFromFile(genDocFile) + if err != nil { + return nil, err + } + return MakeGenesisState(db, genDoc) +} + +// MakeGenesisDocFromFile reads and unmarshals genesis doc from the given file. +func MakeGenesisDocFromFile(genDocFile string) (*types.GenesisDoc, error) { genDocJSON, err := ioutil.ReadFile(genDocFile) if err != nil { - cmn.Exit(cmn.Fmt("Couldn't read GenesisDoc file: %v", err)) + return nil, fmt.Errorf("Couldn't read GenesisDoc file: %v", err) } genDoc, err := types.GenesisDocFromJSON(genDocJSON) if err != nil { - cmn.Exit(cmn.Fmt("Error reading GenesisDoc: %v", err)) + return nil, fmt.Errorf("Error reading GenesisDoc: %v", err) } - return MakeGenesisState(db, genDoc) + return genDoc, nil } // MakeGenesisState creates state from types.GenesisDoc. // // Used in tests. -func MakeGenesisState(db dbm.DB, genDoc *types.GenesisDoc) *State { +func MakeGenesisState(db dbm.DB, genDoc *types.GenesisDoc) (*State, error) { err := genDoc.ValidateAndComplete() if err != nil { - cmn.Exit(cmn.Fmt("Error in genesis file: %v", err)) + return nil, fmt.Errorf("Error in genesis file: %v", err) } // Make validators slice @@ -363,5 +375,5 @@ func MakeGenesisState(db dbm.DB, genDoc *types.GenesisDoc) *State { AppHash: genDoc.AppHash, TxIndexer: &null.TxIndex{}, // we do not need indexer during replay and in tests LastHeightValidatorsChanged: 1, - } + }, nil } diff --git a/state/state_test.go b/state/state_test.go index 4ff367fa3..2c19a4164 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -23,7 +23,8 @@ import ( func setupTestCase(t *testing.T) (func(t *testing.T), dbm.DB, *State) { config := cfg.ResetTestRoot("state_") stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir()) - state := GetState(stateDB, config.GenesisFile()) + state, err := GetState(stateDB, config.GenesisFile()) + assert.NoError(t, err, "expected no error on GetState") state.SetLogger(log.TestingLogger()) tearDown := func(t *testing.T) {} From 3ca7b10ad4c3e1a40f507489aa857c46acf1f115 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 20 Sep 2017 18:40:41 -0400 Subject: [PATCH 78/92] types: more . -> cmn --- types/priv_validator.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/types/priv_validator.go b/types/priv_validator.go index e5b537789..7fe6e1530 100644 --- a/types/priv_validator.go +++ b/types/priv_validator.go @@ -11,7 +11,7 @@ import ( crypto "github.com/tendermint/go-crypto" data "github.com/tendermint/go-wire/data" - . "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tmlibs/common" ) // TODO: type ? @@ -29,7 +29,7 @@ func voteToStep(vote *Vote) int8 { case VoteTypePrecommit: return stepPrecommit default: - PanicSanity("Unknown vote type") + cmn.PanicSanity("Unknown vote type") return 0 } } @@ -78,12 +78,12 @@ func LoadOrGenPrivValidatorFS(filePath string) *PrivValidatorFS { func LoadPrivValidatorFS(filePath string) *PrivValidatorFS { privValJSONBytes, err := ioutil.ReadFile(filePath) if err != nil { - Exit(err.Error()) + cmn.Exit(err.Error()) } privVal := PrivValidatorFS{} err = json.Unmarshal(privValJSONBytes, &privVal) if err != nil { - Exit(Fmt("Error reading PrivValidator from %v: %v\n", filePath, err)) + cmn.Exit(cmn.Fmt("Error reading PrivValidator from %v: %v\n", filePath, err)) } privVal.filePath = filePath @@ -111,12 +111,12 @@ func GenPrivValidatorFS(filePath string) *PrivValidatorFS { func LoadPrivValidatorFSWithSigner(filePath string, signerFunc func(ValidatorID) Signer) *PrivValidatorFS { privValJSONBytes, err := ioutil.ReadFile(filePath) if err != nil { - Exit(err.Error()) + cmn.Exit(err.Error()) } privVal := PrivValidatorFS{} err = json.Unmarshal(privValJSONBytes, &privVal) if err != nil { - Exit(Fmt("Error reading PrivValidator from %v: %v\n", filePath, err)) + cmn.Exit(cmn.Fmt("Error reading PrivValidator from %v: %v\n", filePath, err)) } privVal.filePath = filePath @@ -143,17 +143,17 @@ func (privVal *PrivValidatorFS) Save() { func (privVal *PrivValidatorFS) save() { if privVal.filePath == "" { - PanicSanity("Cannot save PrivValidator: filePath not set") + cmn.PanicSanity("Cannot save PrivValidator: filePath not set") } jsonBytes, err := json.Marshal(privVal) if err != nil { // `@; BOOM!!! - PanicCrisis(err) + cmn.PanicCrisis(err) } - err = WriteFileAtomic(privVal.filePath, jsonBytes, 0600) + err = cmn.WriteFileAtomic(privVal.filePath, jsonBytes, 0600) if err != nil { // `@; BOOM!!! - PanicCrisis(err) + cmn.PanicCrisis(err) } } @@ -198,7 +198,7 @@ func (privVal *PrivValidatorFS) SignVote(chainID string, vote *Vote) error { defer privVal.mtx.Unlock() signature, err := privVal.signBytesHRS(vote.Height, vote.Round, voteToStep(vote), SignBytes(chainID, vote)) if err != nil { - return errors.New(Fmt("Error signing vote: %v", err)) + return errors.New(cmn.Fmt("Error signing vote: %v", err)) } vote.Signature = signature return nil @@ -246,7 +246,7 @@ func (privVal *PrivValidatorFS) signBytesHRS(height, round int, step int8, signB } else if info.LastStep == step { if info.LastSignBytes != nil { if info.LastSignature.Empty() { - PanicSanity("privVal: LastSignature is nil but LastSignBytes is not!") + cmn.PanicSanity("privVal: LastSignature is nil but LastSignBytes is not!") } // so we dont sign a conflicting vote or proposal // NOTE: proposals are non-deterministic (include time), From 7b99039c34dd513d2f1feca49ad726d46f309123 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 21 Sep 2017 12:19:21 -0400 Subject: [PATCH 79/92] make signBytesHRS a method on LastSignedInfo --- types/priv_validator.go | 248 +++++++++++++++++++++------------------- 1 file changed, 132 insertions(+), 116 deletions(-) diff --git a/types/priv_validator.go b/types/priv_validator.go index 7fe6e1530..892a4f06a 100644 --- a/types/priv_validator.go +++ b/types/priv_validator.go @@ -34,7 +34,8 @@ func voteToStep(vote *Vote) int8 { } } -// PrivValidator defines the functionality of a local Tendermint validator. +// PrivValidator defines the functionality of a local Tendermint validator +// that signs votes, proposals, and heartbeats, and never double signs. type PrivValidator interface { Address() data.Bytes // redundant since .PubKey().Address() PubKey() crypto.PubKey @@ -61,77 +62,56 @@ type PrivValidatorFS struct { filePath string } -// LoadOrGenPrivValidatorFS loads a PrivValidatorFS from the given filePath -// or else generates a new one and saves it to the filePath. -func LoadOrGenPrivValidatorFS(filePath string) *PrivValidatorFS { - var PrivValidatorFS *PrivValidatorFS - if _, err := os.Stat(filePath); err == nil { - PrivValidatorFS = LoadPrivValidatorFS(filePath) - } else { - PrivValidatorFS = GenPrivValidatorFS(filePath) - PrivValidatorFS.Save() - } - return PrivValidatorFS +// Address returns the address of the validator. +// Implements PrivValidator. +func (pv *PrivValidatorFS) Address() data.Bytes { + return pv.ID.Address } -// LoadPrivValidatorFS loads a PrivValidatorFS from the filePath. -func LoadPrivValidatorFS(filePath string) *PrivValidatorFS { - privValJSONBytes, err := ioutil.ReadFile(filePath) - if err != nil { - cmn.Exit(err.Error()) - } - privVal := PrivValidatorFS{} - err = json.Unmarshal(privValJSONBytes, &privVal) - if err != nil { - cmn.Exit(cmn.Fmt("Error reading PrivValidator from %v: %v\n", filePath, err)) - } - - privVal.filePath = filePath - return &privVal +// PubKey returns the public key of the validator. +// Implements PrivValidator. +func (pv *PrivValidatorFS) PubKey() crypto.PubKey { + return pv.ID.PubKey } -// GenPrivValidatorFS generates a new validator with randomly generated private key -// and sets the filePath, but does not call Save(). -func GenPrivValidatorFS(filePath string) *PrivValidatorFS { - privKey := crypto.GenPrivKeyEd25519().Wrap() - return &PrivValidatorFS{ - ID: ValidatorID{privKey.PubKey().Address(), privKey.PubKey()}, - Info: LastSignedInfo{ - LastStep: stepNone, - }, - Signer: NewDefaultSigner(privKey), - filePath: filePath, +// SignVote signs a canonical representation of the vote, along with the chainID. +// Implements PrivValidator. +func (privVal *PrivValidatorFS) SignVote(chainID string, vote *Vote) error { + privVal.mtx.Lock() + defer privVal.mtx.Unlock() + signature, err := privVal.Info.SignBytesHRS(privVal.Signer, + vote.Height, vote.Round, voteToStep(vote), SignBytes(chainID, vote)) + if err != nil { + return errors.New(cmn.Fmt("Error signing vote: %v", err)) } + privVal.save() + vote.Signature = signature + return nil } -// LoadPrivValidatorWithSigner loads a PrivValidatorFS with a custom -// signer object. The PrivValidatorFS handles double signing prevention by persisting -// data to the filePath, while the Signer handles the signing. -// If the filePath does not exist, the PrivValidatorFS must be created manually and saved. -func LoadPrivValidatorFSWithSigner(filePath string, signerFunc func(ValidatorID) Signer) *PrivValidatorFS { - privValJSONBytes, err := ioutil.ReadFile(filePath) - if err != nil { - cmn.Exit(err.Error()) - } - privVal := PrivValidatorFS{} - err = json.Unmarshal(privValJSONBytes, &privVal) +// SignProposal signs a canonical representation of the proposal, along with the chainID. +// Implements PrivValidator. +func (privVal *PrivValidatorFS) SignProposal(chainID string, proposal *Proposal) error { + privVal.mtx.Lock() + defer privVal.mtx.Unlock() + signature, err := privVal.Info.SignBytesHRS(privVal.Signer, + proposal.Height, proposal.Round, stepPropose, SignBytes(chainID, proposal)) if err != nil { - cmn.Exit(cmn.Fmt("Error reading PrivValidator from %v: %v\n", filePath, err)) + return fmt.Errorf("Error signing proposal: %v", err) } - - privVal.filePath = filePath - privVal.Signer = signerFunc(privVal.ID) - return &privVal -} - -// Address returns the address of the validator. -func (pv *PrivValidatorFS) Address() data.Bytes { - return pv.ID.Address + privVal.save() + proposal.Signature = signature + return nil } -// PubKey returns the public key of the validator. -func (pv *PrivValidatorFS) PubKey() crypto.PubKey { - return pv.ID.PubKey +// SignHeartbeat signs a canonical representation of the heartbeat, along with the chainID. +// Implements PrivValidator. +func (privVal *PrivValidatorFS) SignHeartbeat(chainID string, heartbeat *Heartbeat) error { + privVal.mtx.Lock() + defer privVal.mtx.Unlock() + var err error + heartbeat.Signature, err = privVal.Signer.Sign(SignBytes(chainID, heartbeat)) + return err } // Save persists the PrivValidatorFS to disk. @@ -192,43 +172,102 @@ func (privVal *PrivValidatorFS) Reset() { privVal.Save() } -// SignVote signs a canonical representation of the vote, along with the chainID. -func (privVal *PrivValidatorFS) SignVote(chainID string, vote *Vote) error { - privVal.mtx.Lock() - defer privVal.mtx.Unlock() - signature, err := privVal.signBytesHRS(vote.Height, vote.Round, voteToStep(vote), SignBytes(chainID, vote)) +// String returns a string representation of the PrivValidatorFS. +func (privVal *PrivValidatorFS) String() string { + info := privVal.Info + return fmt.Sprintf("PrivValidator{%v LH:%v, LR:%v, LS:%v}", privVal.Address(), info.LastHeight, info.LastRound, info.LastStep) +} + +// LoadOrGenPrivValidatorFS loads a PrivValidatorFS from the given filePath +// or else generates a new one and saves it to the filePath. +func LoadOrGenPrivValidatorFS(filePath string) *PrivValidatorFS { + var PrivValidatorFS *PrivValidatorFS + if _, err := os.Stat(filePath); err == nil { + PrivValidatorFS = LoadPrivValidatorFS(filePath) + } else { + PrivValidatorFS = GenPrivValidatorFS(filePath) + PrivValidatorFS.Save() + } + return PrivValidatorFS +} + +// LoadPrivValidatorFS loads a PrivValidatorFS from the filePath. +func LoadPrivValidatorFS(filePath string) *PrivValidatorFS { + privValJSONBytes, err := ioutil.ReadFile(filePath) if err != nil { - return errors.New(cmn.Fmt("Error signing vote: %v", err)) + cmn.Exit(err.Error()) } - vote.Signature = signature - return nil + privVal := PrivValidatorFS{} + err = json.Unmarshal(privValJSONBytes, &privVal) + if err != nil { + cmn.Exit(cmn.Fmt("Error reading PrivValidator from %v: %v\n", filePath, err)) + } + + privVal.filePath = filePath + return &privVal } -// SignProposal signs a canonical representation of the proposal, along with the chainID. -func (privVal *PrivValidatorFS) SignProposal(chainID string, proposal *Proposal) error { - privVal.mtx.Lock() - defer privVal.mtx.Unlock() - signature, err := privVal.signBytesHRS(proposal.Height, proposal.Round, stepPropose, SignBytes(chainID, proposal)) +// GenPrivValidatorFS generates a new validator with randomly generated private key +// and sets the filePath, but does not call Save(). +func GenPrivValidatorFS(filePath string) *PrivValidatorFS { + privKey := crypto.GenPrivKeyEd25519().Wrap() + return &PrivValidatorFS{ + ID: ValidatorID{privKey.PubKey().Address(), privKey.PubKey()}, + Info: LastSignedInfo{ + LastStep: stepNone, + }, + Signer: NewDefaultSigner(privKey), + filePath: filePath, + } +} + +// LoadPrivValidatorWithSigner loads a PrivValidatorFS with a custom +// signer object. The PrivValidatorFS handles double signing prevention by persisting +// data to the filePath, while the Signer handles the signing. +// If the filePath does not exist, the PrivValidatorFS must be created manually and saved. +func LoadPrivValidatorFSWithSigner(filePath string, signerFunc func(ValidatorID) Signer) *PrivValidatorFS { + privValJSONBytes, err := ioutil.ReadFile(filePath) if err != nil { - return fmt.Errorf("Error signing proposal: %v", err) + cmn.Exit(err.Error()) } - proposal.Signature = signature - return nil + privVal := PrivValidatorFS{} + err = json.Unmarshal(privValJSONBytes, &privVal) + if err != nil { + cmn.Exit(cmn.Fmt("Error reading PrivValidator from %v: %v\n", filePath, err)) + } + + privVal.filePath = filePath + privVal.Signer = signerFunc(privVal.ID) + return &privVal } -// SignHeartbeat signs a canonical representation of the heartbeat, along with the chainID. -func (privVal *PrivValidatorFS) SignHeartbeat(chainID string, heartbeat *Heartbeat) error { - privVal.mtx.Lock() - defer privVal.mtx.Unlock() - var err error - heartbeat.Signature, err = privVal.Signer.Sign(SignBytes(chainID, heartbeat)) - return err +//------------------------------------- + +// ValidatorID contains the identity of the validator. +type ValidatorID struct { + Address data.Bytes `json:"address"` + PubKey crypto.PubKey `json:"pub_key"` } -// check if there's a regression. Else sign and write the hrs+signature to disk -func (privVal *PrivValidatorFS) signBytesHRS(height, round int, step int8, signBytes []byte) (crypto.Signature, error) { +//------------------------------------- + +// LastSignedInfo contains information about the latest +// data signed by a validator to help prevent double signing. +type LastSignedInfo struct { + LastHeight int `json:"last_height"` + LastRound int `json:"last_round"` + LastStep int8 `json:"last_step"` + LastSignature crypto.Signature `json:"last_signature,omitempty"` // so we dont lose signatures + LastSignBytes data.Bytes `json:"last_signbytes,omitempty"` // so we dont lose signatures +} + +// SignBytesHRS signs the given signBytes with the signer if the height/round/step (HRS) +// are greater than the latest state of the LastSignedInfo. If the HRS are equal, +// it returns the LastSignedInfo.LastSignature. +func (info *LastSignedInfo) SignBytesHRS(signer Signer, + height, round int, step int8, signBytes []byte) (crypto.Signature, error) { + sig := crypto.Signature{} - info := privVal.Info // If height regression, err if info.LastHeight > height { return sig, errors.New("Height regression") @@ -262,46 +301,23 @@ func (privVal *PrivValidatorFS) signBytesHRS(height, round int, step int8, signB } // Sign - sig, err := privVal.Signer.Sign(signBytes) + sig, err := signer.Sign(signBytes) if err != nil { return sig, err } // Persist height/round/step - privVal.Info.LastHeight = height - privVal.Info.LastRound = round - privVal.Info.LastStep = step - privVal.Info.LastSignature = sig - privVal.Info.LastSignBytes = signBytes - privVal.save() + info.LastHeight = height + info.LastRound = round + info.LastStep = step + info.LastSignature = sig + info.LastSignBytes = signBytes return sig, nil } -// String returns a string representation of the PrivValidatorFS. -func (privVal *PrivValidatorFS) String() string { - info := privVal.Info - return fmt.Sprintf("PrivValidator{%v LH:%v, LR:%v, LS:%v}", privVal.Address(), info.LastHeight, info.LastRound, info.LastStep) -} - //------------------------------------- -// ValidatorID contains the identity of the validator. -type ValidatorID struct { - Address data.Bytes `json:"address"` - PubKey crypto.PubKey `json:"pub_key"` -} - -// LastSignedInfo contains information about the latest -// data signed by a validator to help prevent double signing. -type LastSignedInfo struct { - LastHeight int `json:"last_height"` - LastRound int `json:"last_round"` - LastStep int8 `json:"last_step"` - LastSignature crypto.Signature `json:"last_signature,omitempty"` // so we dont lose signatures - LastSignBytes data.Bytes `json:"last_signbytes,omitempty"` // so we dont lose signatures -} - // Signer is an interface that defines how to sign messages. // It is the caller's duty to verify the msg before calling Sign, // eg. to avoid double signing. From 75b97a5a658ab96bd2279ee0abdcb644962daf5a Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 21 Sep 2017 16:32:02 -0400 Subject: [PATCH 80/92] PrivValidatorFS is like old PrivValidator, for now --- cmd/tendermint/commands/init.go | 2 +- cmd/tendermint/commands/testnet.go | 2 +- config/toml.go | 10 +- consensus/byzantine_test.go | 10 +- consensus/common_test.go | 8 +- consensus/height_vote_set_test.go | 2 +- consensus/reactor_test.go | 14 +- consensus/replay_test.go | 18 +-- consensus/state.go | 12 +- consensus/state_test.go | 18 +-- node/node.go | 6 +- types/priv_validator.go | 211 ++++++++++++----------------- types/priv_validator_test.go | 17 +-- types/proposal_test.go | 2 +- types/validator.go | 2 +- types/vote_set_test.go | 54 ++++---- 16 files changed, 167 insertions(+), 221 deletions(-) diff --git a/cmd/tendermint/commands/init.go b/cmd/tendermint/commands/init.go index d8e8bde40..cbafac3ef 100644 --- a/cmd/tendermint/commands/init.go +++ b/cmd/tendermint/commands/init.go @@ -29,7 +29,7 @@ func initFiles(cmd *cobra.Command, args []string) { ChainID: cmn.Fmt("test-chain-%v", cmn.RandStr(6)), } genDoc.Validators = []types.GenesisValidator{types.GenesisValidator{ - PubKey: privValidator.PubKey(), + PubKey: privValidator.GetPubKey(), Power: 10, }} diff --git a/cmd/tendermint/commands/testnet.go b/cmd/tendermint/commands/testnet.go index 7fc4f94fe..ac6f337a9 100644 --- a/cmd/tendermint/commands/testnet.go +++ b/cmd/tendermint/commands/testnet.go @@ -47,7 +47,7 @@ func testnetFiles(cmd *cobra.Command, args []string) { privValFile := path.Join(dataDir, mach, "priv_validator.json") privVal := types.LoadPrivValidatorFS(privValFile) genVals[i] = types.GenesisValidator{ - PubKey: privVal.PubKey(), + PubKey: privVal.GetPubKey(), Power: 1, Name: mach, } diff --git a/config/toml.go b/config/toml.go index 02ae3fc61..4366695db 100644 --- a/config/toml.go +++ b/config/toml.go @@ -127,22 +127,16 @@ var testGenesis = `{ }` var testPrivValidator = `{ - "id": { "address": "D028C9981F7A87F3093672BF0D5B0E2A1B3ED456", "pub_key": { "type": "ed25519", "data": "3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8" - } - }, - "signer": { + }, "priv_key": { "type": "ed25519", "data": "27F82582AEFAE7AB151CFB01C48BB6C1A0DA78F9BDDA979A9F70A84D074EB07D3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8" - } - }, - "info": { + }, "last_height": 0, "last_round": 0, "last_step": 0 - } }` diff --git a/consensus/byzantine_test.go b/consensus/byzantine_test.go index d8b6b0495..0f2d7b040 100644 --- a/consensus/byzantine_test.go +++ b/consensus/byzantine_test.go @@ -272,12 +272,12 @@ func NewByzantinePrivValidator(pv types.PrivValidator) *ByzantinePrivValidator { } } -func (privVal *ByzantinePrivValidator) Address() data.Bytes { - return privVal.pv.Address() +func (privVal *ByzantinePrivValidator) GetAddress() data.Bytes { + return privVal.pv.GetAddress() } -func (privVal *ByzantinePrivValidator) PubKey() crypto.PubKey { - return privVal.pv.PubKey() +func (privVal *ByzantinePrivValidator) GetPubKey() crypto.PubKey { + return privVal.pv.GetPubKey() } func (privVal *ByzantinePrivValidator) SignVote(chainID string, vote *types.Vote) (err error) { @@ -296,5 +296,5 @@ func (privVal *ByzantinePrivValidator) SignHeartbeat(chainID string, heartbeat * } func (privVal *ByzantinePrivValidator) String() string { - return Fmt("PrivValidator{%X}", privVal.Address) + return Fmt("PrivValidator{%X}", privVal.GetAddress()) } diff --git a/consensus/common_test.go b/consensus/common_test.go index a746ec59c..b16afc3d0 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -65,7 +65,7 @@ func NewValidatorStub(privValidator types.PrivValidator, valIndex int) *validato func (vs *validatorStub) signVote(voteType byte, hash []byte, header types.PartSetHeader) (*types.Vote, error) { vote := &types.Vote{ ValidatorIndex: vs.Index, - ValidatorAddress: vs.PrivValidator.Address(), + ValidatorAddress: vs.PrivValidator.GetAddress(), Height: vs.Height, Round: vs.Round, Type: voteType, @@ -142,7 +142,7 @@ func signAddVotes(to *ConsensusState, voteType byte, hash []byte, header types.P func validatePrevote(t *testing.T, cs *ConsensusState, round int, privVal *validatorStub, blockHash []byte) { prevotes := cs.Votes.Prevotes(round) var vote *types.Vote - if vote = prevotes.GetByAddress(privVal.Address()); vote == nil { + if vote = prevotes.GetByAddress(privVal.GetAddress()); vote == nil { panic("Failed to find prevote from validator") } if blockHash == nil { @@ -159,7 +159,7 @@ func validatePrevote(t *testing.T, cs *ConsensusState, round int, privVal *valid func validateLastPrecommit(t *testing.T, cs *ConsensusState, privVal *validatorStub, blockHash []byte) { votes := cs.LastCommit var vote *types.Vote - if vote = votes.GetByAddress(privVal.Address()); vote == nil { + if vote = votes.GetByAddress(privVal.GetAddress()); vote == nil { panic("Failed to find precommit from validator") } if !bytes.Equal(vote.BlockID.Hash, blockHash) { @@ -170,7 +170,7 @@ func validateLastPrecommit(t *testing.T, cs *ConsensusState, privVal *validatorS func validatePrecommit(t *testing.T, cs *ConsensusState, thisRound, lockRound int, privVal *validatorStub, votedBlockHash, lockedBlockHash []byte) { precommits := cs.Votes.Precommits(thisRound) var vote *types.Vote - if vote = precommits.GetByAddress(privVal.Address()); vote == nil { + if vote = precommits.GetByAddress(privVal.GetAddress()); vote == nil { panic("Failed to find precommit from validator") } diff --git a/consensus/height_vote_set_test.go b/consensus/height_vote_set_test.go index 3d22a1dab..7e03e40f5 100644 --- a/consensus/height_vote_set_test.go +++ b/consensus/height_vote_set_test.go @@ -47,7 +47,7 @@ func TestPeerCatchupRounds(t *testing.T) { func makeVoteHR(t *testing.T, height, round int, privVals []*types.PrivValidatorFS, valIndex int) *types.Vote { privVal := privVals[valIndex] vote := &types.Vote{ - ValidatorAddress: privVal.Address(), + ValidatorAddress: privVal.GetAddress(), ValidatorIndex: valIndex, Height: height, Round: round, diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index 752c759f1..623a65413 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -119,7 +119,7 @@ func TestVotingPowerChange(t *testing.T) { // map of active validators activeVals := make(map[string]struct{}) for i := 0; i < nVals; i++ { - activeVals[string(css[i].privValidator.Address())] = struct{}{} + activeVals[string(css[i].privValidator.GetAddress())] = struct{}{} } // wait till everyone makes block 1 @@ -132,7 +132,7 @@ func TestVotingPowerChange(t *testing.T) { //--------------------------------------------------------------------------- t.Log("---------------------------- Testing changing the voting power of one validator a few times") - val1PubKey := css[0].privValidator.PubKey() + val1PubKey := css[0].privValidator.GetPubKey() updateValidatorTx := dummy.MakeValSetChangeTx(val1PubKey.Bytes(), 25) previousTotalVotingPower := css[0].GetRoundState().LastValidators.TotalVotingPower() @@ -180,7 +180,7 @@ func TestValidatorSetChanges(t *testing.T) { // map of active validators activeVals := make(map[string]struct{}) for i := 0; i < nVals; i++ { - activeVals[string(css[i].privValidator.Address())] = struct{}{} + activeVals[string(css[i].privValidator.GetAddress())] = struct{}{} } // wait till everyone makes block 1 @@ -193,7 +193,7 @@ func TestValidatorSetChanges(t *testing.T) { //--------------------------------------------------------------------------- t.Log("---------------------------- Testing adding one validator") - newValidatorPubKey1 := css[nVals].privValidator.PubKey() + newValidatorPubKey1 := css[nVals].privValidator.GetPubKey() newValidatorTx1 := dummy.MakeValSetChangeTx(newValidatorPubKey1.Bytes(), uint64(testMinPower)) // wait till everyone makes block 2 @@ -219,7 +219,7 @@ func TestValidatorSetChanges(t *testing.T) { //--------------------------------------------------------------------------- t.Log("---------------------------- Testing changing the voting power of one validator") - updateValidatorPubKey1 := css[nVals].privValidator.PubKey() + updateValidatorPubKey1 := css[nVals].privValidator.GetPubKey() updateValidatorTx1 := dummy.MakeValSetChangeTx(updateValidatorPubKey1.Bytes(), 25) previousTotalVotingPower := css[nVals].GetRoundState().LastValidators.TotalVotingPower() @@ -235,10 +235,10 @@ func TestValidatorSetChanges(t *testing.T) { //--------------------------------------------------------------------------- t.Log("---------------------------- Testing adding two validators at once") - newValidatorPubKey2 := css[nVals+1].privValidator.PubKey() + newValidatorPubKey2 := css[nVals+1].privValidator.GetPubKey() newValidatorTx2 := dummy.MakeValSetChangeTx(newValidatorPubKey2.Bytes(), uint64(testMinPower)) - newValidatorPubKey3 := css[nVals+2].privValidator.PubKey() + newValidatorPubKey3 := css[nVals+2].privValidator.GetPubKey() newValidatorTx3 := dummy.MakeValSetChangeTx(newValidatorPubKey3.Bytes(), uint64(testMinPower)) waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, newValidatorTx2, newValidatorTx3) diff --git a/consensus/replay_test.go b/consensus/replay_test.go index c6504380d..765434677 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -184,10 +184,10 @@ func setupReplayTest(t *testing.T, thisCase *testCase, nLines int, crashAfter bo cs := fixedConsensusStateDummy() // set the last step according to when we crashed vs the wal - toPV(cs.privValidator).Info.LastHeight = 1 // first block - toPV(cs.privValidator).Info.LastStep = thisCase.stepMap[lineStep] + toPV(cs.privValidator).LastHeight = 1 // first block + toPV(cs.privValidator).LastStep = thisCase.stepMap[lineStep] - t.Logf("[WARN] setupReplayTest LastStep=%v", toPV(cs.privValidator).Info.LastStep) + t.Logf("[WARN] setupReplayTest LastStep=%v", toPV(cs.privValidator).LastStep) newBlockCh := subscribeToEvent(cs.evsw, "tester", types.EventStringNewBlock(), 1) @@ -230,8 +230,8 @@ func TestWALCrashBeforeWritePropose(t *testing.T) { msg := readTimedWALMessage(t, proposalMsg) proposal := msg.Msg.(msgInfo).Msg.(*ProposalMessage) // Set LastSig - toPV(cs.privValidator).Info.LastSignBytes = types.SignBytes(cs.state.ChainID, proposal.Proposal) - toPV(cs.privValidator).Info.LastSignature = proposal.Proposal.Signature + toPV(cs.privValidator).LastSignBytes = types.SignBytes(cs.state.ChainID, proposal.Proposal) + toPV(cs.privValidator).LastSignature = proposal.Proposal.Signature runReplayTest(t, cs, walFile, newBlockCh, thisCase, lineNum) } } @@ -255,8 +255,8 @@ func testReplayCrashBeforeWriteVote(t *testing.T, thisCase *testCase, lineNum in msg := readTimedWALMessage(t, voteMsg) vote := msg.Msg.(msgInfo).Msg.(*VoteMessage) // Set LastSig - toPV(cs.privValidator).Info.LastSignBytes = types.SignBytes(cs.state.ChainID, vote.Vote) - toPV(cs.privValidator).Info.LastSignature = vote.Vote.Signature + toPV(cs.privValidator).LastSignBytes = types.SignBytes(cs.state.ChainID, vote.Vote) + toPV(cs.privValidator).LastSignature = vote.Vote.Signature }) runReplayTest(t, cs, walFile, newBlockCh, thisCase, lineNum) } @@ -332,7 +332,7 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) { t.Fatalf(err.Error()) } - state, store := stateAndStore(config, privVal.PubKey()) + state, store := stateAndStore(config, privVal.GetPubKey()) store.chain = chain store.commits = commits @@ -346,7 +346,7 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) { // run nBlocks against a new client to build up the app state. // use a throwaway tendermint state proxyApp := proxy.NewAppConns(clientCreator2, nil) - state, _ := stateAndStore(config, privVal.PubKey()) + state, _ := stateAndStore(config, privVal.GetPubKey()) buildAppStateFromChain(proxyApp, state, chain, nBlocks, mode) } diff --git a/consensus/state.go b/consensus/state.go index f141cad58..648fc0559 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -817,7 +817,7 @@ func (cs *ConsensusState) needProofBlock(height int) bool { func (cs *ConsensusState) proposalHeartbeat(height, round int) { counter := 0 - addr := cs.privValidator.Address() + addr := cs.privValidator.GetAddress() valIndex, v := cs.Validators.GetByAddress(addr) if v == nil { // not a validator @@ -878,7 +878,7 @@ func (cs *ConsensusState) enterPropose(height int, round int) { if !cs.isProposer() { cs.Logger.Info("enterPropose: Not our turn to propose", "proposer", cs.Validators.GetProposer().Address, "privValidator", cs.privValidator) - if cs.Validators.HasAddress(cs.privValidator.Address()) { + if cs.Validators.HasAddress(cs.privValidator.GetAddress()) { cs.Logger.Debug("This node is a validator") } else { cs.Logger.Debug("This node is not a validator") @@ -891,7 +891,7 @@ func (cs *ConsensusState) enterPropose(height int, round int) { } func (cs *ConsensusState) isProposer() bool { - return bytes.Equal(cs.Validators.GetProposer().Address, cs.privValidator.Address()) + return bytes.Equal(cs.Validators.GetProposer().Address, cs.privValidator.GetAddress()) } func (cs *ConsensusState) defaultDecideProposal(height, round int) { @@ -1436,7 +1436,7 @@ func (cs *ConsensusState) tryAddVote(vote *types.Vote, peerKey string) error { if err == ErrVoteHeightMismatch { return err } else if _, ok := err.(*types.ErrVoteConflictingVotes); ok { - if bytes.Equal(vote.ValidatorAddress, cs.privValidator.Address()) { + if bytes.Equal(vote.ValidatorAddress, cs.privValidator.GetAddress()) { cs.Logger.Error("Found conflicting vote from ourselves. Did you unsafe_reset a validator?", "height", vote.Height, "round", vote.Round, "type", vote.Type) return err } @@ -1565,7 +1565,7 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool, } func (cs *ConsensusState) signVote(type_ byte, hash []byte, header types.PartSetHeader) (*types.Vote, error) { - addr := cs.privValidator.Address() + addr := cs.privValidator.GetAddress() valIndex, _ := cs.Validators.GetByAddress(addr) vote := &types.Vote{ ValidatorAddress: addr, @@ -1582,7 +1582,7 @@ func (cs *ConsensusState) signVote(type_ byte, hash []byte, header types.PartSet // sign the vote and publish on internalMsgQueue func (cs *ConsensusState) signAddVote(type_ byte, hash []byte, header types.PartSetHeader) *types.Vote { // if we don't have a key or we're not in the validator set, do nothing - if cs.privValidator == nil || !cs.Validators.HasAddress(cs.privValidator.Address()) { + if cs.privValidator == nil || !cs.Validators.HasAddress(cs.privValidator.GetAddress()) { return nil } vote, err := cs.signVote(type_, hash, header) diff --git a/consensus/state_test.go b/consensus/state_test.go index 977d6445c..c4a6769e9 100644 --- a/consensus/state_test.go +++ b/consensus/state_test.go @@ -65,7 +65,7 @@ func TestProposerSelection0(t *testing.T) { // lets commit a block and ensure proposer for the next height is correct prop := cs1.GetRoundState().Validators.GetProposer() - if !bytes.Equal(prop.Address, cs1.privValidator.Address()) { + if !bytes.Equal(prop.Address, cs1.privValidator.GetAddress()) { t.Fatalf("expected proposer to be validator %d. Got %X", 0, prop.Address) } @@ -79,7 +79,7 @@ func TestProposerSelection0(t *testing.T) { <-newRoundCh prop = cs1.GetRoundState().Validators.GetProposer() - if !bytes.Equal(prop.Address, vss[1].Address()) { + if !bytes.Equal(prop.Address, vss[1].GetAddress()) { panic(Fmt("expected proposer to be validator %d. Got %X", 1, prop.Address)) } } @@ -100,7 +100,7 @@ func TestProposerSelection2(t *testing.T) { // everyone just votes nil. we get a new proposer each round for i := 0; i < len(vss); i++ { prop := cs1.GetRoundState().Validators.GetProposer() - if !bytes.Equal(prop.Address, vss[(i+2)%len(vss)].Address()) { + if !bytes.Equal(prop.Address, vss[(i+2)%len(vss)].GetAddress()) { panic(Fmt("expected proposer to be validator %d. Got %X", (i+2)%len(vss), prop.Address)) } @@ -613,7 +613,7 @@ func TestLockPOLUnlock(t *testing.T) { timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1) newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1) unlockCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringUnlock(), 1) - voteCh := subscribeToVoter(cs1, cs1.privValidator.Address()) + voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress()) // everything done from perspective of cs1 @@ -707,7 +707,7 @@ func TestLockPOLSafety1(t *testing.T) { timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1) timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1) newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1) - voteCh := subscribeToVoter(cs1, cs1.privValidator.Address()) + voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress()) // start round and wait for propose and prevote startTestRound(cs1, cs1.Height, 0) @@ -829,7 +829,7 @@ func TestLockPOLSafety2(t *testing.T) { timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1) newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1) unlockCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringUnlock(), 1) - voteCh := subscribeToVoter(cs1, cs1.privValidator.Address()) + voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress()) // the block for R0: gets polkad but we miss it // (even though we signed it, shhh) @@ -921,7 +921,7 @@ func TestSlashingPrevotes(t *testing.T) { proposalCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringCompleteProposal() , 1) timeoutWaitCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringTimeoutWait() , 1) newRoundCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringNewRound() , 1) - voteCh := subscribeToVoter(cs1, cs1.privValidator.Address()) + voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress()) // start round and wait for propose and prevote startTestRound(cs1, cs1.Height, 0) @@ -956,7 +956,7 @@ func TestSlashingPrecommits(t *testing.T) { proposalCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringCompleteProposal() , 1) timeoutWaitCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringTimeoutWait() , 1) newRoundCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringNewRound() , 1) - voteCh := subscribeToVoter(cs1, cs1.privValidator.Address()) + voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress()) // start round and wait for propose and prevote startTestRound(cs1, cs1.Height, 0) @@ -1003,7 +1003,7 @@ func TestHalt1(t *testing.T) { timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1) newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1) newBlockCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewBlock(), 1) - voteCh := subscribeToVoter(cs1, cs1.privValidator.Address()) + voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress()) // start round and wait for propose and prevote startTestRound(cs1, cs1.Height, 0) diff --git a/node/node.go b/node/node.go index 761f4d5e4..0fb064514 100644 --- a/node/node.go +++ b/node/node.go @@ -188,13 +188,13 @@ func NewNode(config *cfg.Config, fastSync := config.FastSync if state.Validators.Size() == 1 { addr, _ := state.Validators.GetByIndex(0) - if bytes.Equal(privValidator.Address(), addr) { + if bytes.Equal(privValidator.GetAddress(), addr) { fastSync = false } } // Log whether this node is a validator or an observer - if state.Validators.HasAddress(privValidator.Address()) { + if state.Validators.HasAddress(privValidator.GetAddress()) { consensusLogger.Info("This node is a validator") } else { consensusLogger.Info("This node is not a validator") @@ -386,7 +386,7 @@ func (n *Node) ConfigureRPC() { rpccore.SetConsensusState(n.consensusState) rpccore.SetMempool(n.mempoolReactor.Mempool) rpccore.SetSwitch(n.sw) - rpccore.SetPubKey(n.privValidator.PubKey()) + rpccore.SetPubKey(n.privValidator.GetPubKey()) rpccore.SetGenesisDoc(n.genesisDoc) rpccore.SetAddrBook(n.addrBook) rpccore.SetProxyAppQuery(n.proxyApp.Query()) diff --git a/types/priv_validator.go b/types/priv_validator.go index 892a4f06a..6db240966 100644 --- a/types/priv_validator.go +++ b/types/priv_validator.go @@ -37,8 +37,8 @@ func voteToStep(vote *Vote) int8 { // PrivValidator defines the functionality of a local Tendermint validator // that signs votes, proposals, and heartbeats, and never double signs. type PrivValidator interface { - Address() data.Bytes // redundant since .PubKey().Address() - PubKey() crypto.PubKey + GetAddress() data.Bytes // redundant since .PubKey().Address() + GetPubKey() crypto.PubKey SignVote(chainID string, vote *Vote) error SignProposal(chainID string, proposal *Proposal) error @@ -49,29 +49,34 @@ type PrivValidator interface { // to prevent double signing. The Signer itself can be mutated to use // something besides the default, for instance a hardware signer. type PrivValidatorFS struct { - ID ValidatorID `json:"id"` - Signer Signer `json:"signer"` + Address data.Bytes `json:"address"` + PubKey crypto.PubKey `json:"pub_key"` + LastHeight int `json:"last_height"` + LastRound int `json:"last_round"` + LastStep int8 `json:"last_step"` + LastSignature crypto.Signature `json:"last_signature,omitempty"` // so we dont lose signatures + LastSignBytes data.Bytes `json:"last_signbytes,omitempty"` // so we dont lose signatures - // mutable state to be persisted to disk - // after each signature to prevent double signing - mtx sync.Mutex - Info LastSignedInfo `json:"info"` + // PrivKey should be empty if a Signer other than the default is being used. + PrivKey crypto.PrivKey `json:"priv_key"` + Signer `json:"-"` // For persistence. // Overloaded for testing. filePath string + mtx sync.Mutex } -// Address returns the address of the validator. +// GetAddress returns the address of the validator. // Implements PrivValidator. -func (pv *PrivValidatorFS) Address() data.Bytes { - return pv.ID.Address +func (pv *PrivValidatorFS) GetAddress() data.Bytes { + return pv.Address } -// PubKey returns the public key of the validator. +// GetPubKey returns the public key of the validator. // Implements PrivValidator. -func (pv *PrivValidatorFS) PubKey() crypto.PubKey { - return pv.ID.PubKey +func (pv *PrivValidatorFS) GetPubKey() crypto.PubKey { + return pv.PubKey } // SignVote signs a canonical representation of the vote, along with the chainID. @@ -79,12 +84,10 @@ func (pv *PrivValidatorFS) PubKey() crypto.PubKey { func (privVal *PrivValidatorFS) SignVote(chainID string, vote *Vote) error { privVal.mtx.Lock() defer privVal.mtx.Unlock() - signature, err := privVal.Info.SignBytesHRS(privVal.Signer, - vote.Height, vote.Round, voteToStep(vote), SignBytes(chainID, vote)) + signature, err := privVal.signBytesHRS(vote.Height, vote.Round, voteToStep(vote), SignBytes(chainID, vote)) if err != nil { return errors.New(cmn.Fmt("Error signing vote: %v", err)) } - privVal.save() vote.Signature = signature return nil } @@ -94,12 +97,10 @@ func (privVal *PrivValidatorFS) SignVote(chainID string, vote *Vote) error { func (privVal *PrivValidatorFS) SignProposal(chainID string, proposal *Proposal) error { privVal.mtx.Lock() defer privVal.mtx.Unlock() - signature, err := privVal.Info.SignBytesHRS(privVal.Signer, - proposal.Height, proposal.Round, stepPropose, SignBytes(chainID, proposal)) + signature, err := privVal.signBytesHRS(proposal.Height, proposal.Round, stepPropose, SignBytes(chainID, proposal)) if err != nil { return fmt.Errorf("Error signing proposal: %v", err) } - privVal.save() proposal.Signature = signature return nil } @@ -137,45 +138,75 @@ func (privVal *PrivValidatorFS) save() { } } -// UnmarshalJSON unmarshals the given jsonString -// into a PrivValidatorFS using a DefaultSigner. -func (pv *PrivValidatorFS) UnmarshalJSON(jsonString []byte) error { - idAndInfo := &struct { - ID ValidatorID `json:"id"` - Info LastSignedInfo `json:"info"` - }{} - if err := json.Unmarshal(jsonString, idAndInfo); err != nil { - return err +// signBytesHRS signs the given signBytes if the height/round/step (HRS) +// are greater than the latest state. If the HRS are equal, +// it returns the privValidator.LastSignature. +func (privVal *PrivValidatorFS) signBytesHRS(height, round int, step int8, signBytes []byte) (crypto.Signature, error) { + + sig := crypto.Signature{} + // If height regression, err + if privVal.LastHeight > height { + return sig, errors.New("Height regression") + } + // More cases for when the height matches + if privVal.LastHeight == height { + // If round regression, err + if privVal.LastRound > round { + return sig, errors.New("Round regression") + } + // If step regression, err + if privVal.LastRound == round { + if privVal.LastStep > step { + return sig, errors.New("Step regression") + } else if privVal.LastStep == step { + if privVal.LastSignBytes != nil { + if privVal.LastSignature.Empty() { + cmn.PanicSanity("privVal: LastSignature is nil but LastSignBytes is not!") + } + // so we dont sign a conflicting vote or proposal + // NOTE: proposals are non-deterministic (include time), + // so we can actually lose them, but will still never sign conflicting ones + if bytes.Equal(privVal.LastSignBytes, signBytes) { + // log.Notice("Using privVal.LastSignature", "sig", privVal.LastSignature) + return privVal.LastSignature, nil + } + } + return sig, errors.New("Step regression") + } + } } - signer := &struct { - Signer *DefaultSigner `json:"signer"` - }{} - if err := json.Unmarshal(jsonString, signer); err != nil { - return err + // Sign + sig, err := privVal.Signer.Sign(signBytes) + if err != nil { + return sig, err } - pv.ID = idAndInfo.ID - pv.Info = idAndInfo.Info - pv.Signer = signer.Signer - return nil + // Persist height/round/step + privVal.LastHeight = height + privVal.LastRound = round + privVal.LastStep = step + privVal.LastSignature = sig + privVal.LastSignBytes = signBytes + privVal.save() + + return sig, nil } -// Reset resets all fields in the PrivValidatorFS.Info. +// Reset resets all fields in the PrivValidatorFS. // NOTE: Unsafe! func (privVal *PrivValidatorFS) Reset() { - privVal.Info.LastHeight = 0 - privVal.Info.LastRound = 0 - privVal.Info.LastStep = 0 - privVal.Info.LastSignature = crypto.Signature{} - privVal.Info.LastSignBytes = nil + privVal.LastHeight = 0 + privVal.LastRound = 0 + privVal.LastStep = 0 + privVal.LastSignature = crypto.Signature{} + privVal.LastSignBytes = nil privVal.Save() } // String returns a string representation of the PrivValidatorFS. func (privVal *PrivValidatorFS) String() string { - info := privVal.Info - return fmt.Sprintf("PrivValidator{%v LH:%v, LR:%v, LS:%v}", privVal.Address(), info.LastHeight, info.LastRound, info.LastStep) + return fmt.Sprintf("PrivValidator{%v LH:%v, LR:%v, LS:%v}", privVal.GetAddress(), privVal.LastHeight, privVal.LastRound, privVal.LastStep) } // LoadOrGenPrivValidatorFS loads a PrivValidatorFS from the given filePath @@ -204,6 +235,7 @@ func LoadPrivValidatorFS(filePath string) *PrivValidatorFS { } privVal.filePath = filePath + privVal.Signer = NewDefaultSigner(privVal.PrivKey) return &privVal } @@ -212,10 +244,10 @@ func LoadPrivValidatorFS(filePath string) *PrivValidatorFS { func GenPrivValidatorFS(filePath string) *PrivValidatorFS { privKey := crypto.GenPrivKeyEd25519().Wrap() return &PrivValidatorFS{ - ID: ValidatorID{privKey.PubKey().Address(), privKey.PubKey()}, - Info: LastSignedInfo{ - LastStep: stepNone, - }, + Address: privKey.PubKey().Address(), + PubKey: privKey.PubKey(), + PrivKey: privKey, + LastStep: stepNone, Signer: NewDefaultSigner(privKey), filePath: filePath, } @@ -225,7 +257,7 @@ func GenPrivValidatorFS(filePath string) *PrivValidatorFS { // signer object. The PrivValidatorFS handles double signing prevention by persisting // data to the filePath, while the Signer handles the signing. // If the filePath does not exist, the PrivValidatorFS must be created manually and saved. -func LoadPrivValidatorFSWithSigner(filePath string, signerFunc func(ValidatorID) Signer) *PrivValidatorFS { +func LoadPrivValidatorFSWithSigner(filePath string, signerFunc func(crypto.PubKey) Signer) *PrivValidatorFS { privValJSONBytes, err := ioutil.ReadFile(filePath) if err != nil { cmn.Exit(err.Error()) @@ -237,85 +269,12 @@ func LoadPrivValidatorFSWithSigner(filePath string, signerFunc func(ValidatorID) } privVal.filePath = filePath - privVal.Signer = signerFunc(privVal.ID) + privVal.Signer = signerFunc(privVal.PubKey) return &privVal } //------------------------------------- -// ValidatorID contains the identity of the validator. -type ValidatorID struct { - Address data.Bytes `json:"address"` - PubKey crypto.PubKey `json:"pub_key"` -} - -//------------------------------------- - -// LastSignedInfo contains information about the latest -// data signed by a validator to help prevent double signing. -type LastSignedInfo struct { - LastHeight int `json:"last_height"` - LastRound int `json:"last_round"` - LastStep int8 `json:"last_step"` - LastSignature crypto.Signature `json:"last_signature,omitempty"` // so we dont lose signatures - LastSignBytes data.Bytes `json:"last_signbytes,omitempty"` // so we dont lose signatures -} - -// SignBytesHRS signs the given signBytes with the signer if the height/round/step (HRS) -// are greater than the latest state of the LastSignedInfo. If the HRS are equal, -// it returns the LastSignedInfo.LastSignature. -func (info *LastSignedInfo) SignBytesHRS(signer Signer, - height, round int, step int8, signBytes []byte) (crypto.Signature, error) { - - sig := crypto.Signature{} - // If height regression, err - if info.LastHeight > height { - return sig, errors.New("Height regression") - } - // More cases for when the height matches - if info.LastHeight == height { - // If round regression, err - if info.LastRound > round { - return sig, errors.New("Round regression") - } - // If step regression, err - if info.LastRound == round { - if info.LastStep > step { - return sig, errors.New("Step regression") - } else if info.LastStep == step { - if info.LastSignBytes != nil { - if info.LastSignature.Empty() { - cmn.PanicSanity("privVal: LastSignature is nil but LastSignBytes is not!") - } - // so we dont sign a conflicting vote or proposal - // NOTE: proposals are non-deterministic (include time), - // so we can actually lose them, but will still never sign conflicting ones - if bytes.Equal(info.LastSignBytes, signBytes) { - // log.Notice("Using info.LastSignature", "sig", info.LastSignature) - return info.LastSignature, nil - } - } - return sig, errors.New("Step regression") - } - } - } - - // Sign - sig, err := signer.Sign(signBytes) - if err != nil { - return sig, err - } - - // Persist height/round/step - info.LastHeight = height - info.LastRound = round - info.LastStep = step - info.LastSignature = sig - info.LastSignBytes = signBytes - - return sig, nil -} - //------------------------------------- // Signer is an interface that defines how to sign messages. @@ -353,7 +312,7 @@ func (pvs PrivValidatorsByAddress) Len() int { } func (pvs PrivValidatorsByAddress) Less(i, j int) bool { - return bytes.Compare(pvs[i].Address(), pvs[j].Address()) == -1 + return bytes.Compare(pvs[i].GetAddress(), pvs[j].GetAddress()) == -1 } func (pvs PrivValidatorsByAddress) Swap(i, j int) { diff --git a/types/priv_validator_test.go b/types/priv_validator_test.go index 58a43d489..d56cb9ec9 100644 --- a/types/priv_validator_test.go +++ b/types/priv_validator_test.go @@ -29,25 +29,19 @@ func TestLoadValidator(t *testing.T) { require.Nil(err, "%+v", err) serialized := fmt.Sprintf(`{ - "id": { "address": "%s", "pub_key": { "type": "ed25519", "data": "%s" - } - }, - "info": { + }, "last_height": 0, "last_round": 0, "last_step": 0, - "last_signature": null - }, - "signer": { + "last_signature": null, "priv_key": { "type": "ed25519", "data": "%s" } - } }`, addrStr, pubStr, privStr) val := PrivValidatorFS{} @@ -55,10 +49,9 @@ func TestLoadValidator(t *testing.T) { require.Nil(err, "%+v", err) // make sure the values match - assert.EqualValues(addrBytes, val.Address()) - assert.EqualValues(pubKey, val.PubKey()) - valPrivKey := val.Signer.(*DefaultSigner).PrivKey - assert.EqualValues(privKey, valPrivKey) + assert.EqualValues(addrBytes, val.GetAddress()) + assert.EqualValues(pubKey, val.GetPubKey()) + assert.EqualValues(privKey, val.PrivKey) // export it and make sure it is the same out, err := json.Marshal(val) diff --git a/types/proposal_test.go b/types/proposal_test.go index 2e4e7e198..d1c991849 100644 --- a/types/proposal_test.go +++ b/types/proposal_test.go @@ -38,7 +38,7 @@ func BenchmarkProposalVerifySignature(b *testing.B) { signBytes := SignBytes("test_chain_id", testProposal) privVal := GenPrivValidatorFS("") signature, _ := privVal.Signer.Sign(signBytes) - pubKey := privVal.PubKey() + pubKey := privVal.GetPubKey() for i := 0; i < b.N; i++ { pubKey.VerifyBytes(SignBytes("test_chain_id", testProposal), signature) diff --git a/types/validator.go b/types/validator.go index e52831e27..506828913 100644 --- a/types/validator.go +++ b/types/validator.go @@ -113,6 +113,6 @@ func RandValidator(randPower bool, minPower int64) (*Validator, *PrivValidatorFS if randPower { votePower += int64(cmn.RandUint32()) } - val := NewValidator(privVal.PubKey(), votePower) + val := NewValidator(privVal.GetPubKey(), votePower) return val, privVal } diff --git a/types/vote_set_test.go b/types/vote_set_test.go index ab1456ed9..80ee6135c 100644 --- a/types/vote_set_test.go +++ b/types/vote_set_test.go @@ -76,7 +76,7 @@ func TestAddVote(t *testing.T) { // t.Logf(">> %v", voteSet) - if voteSet.GetByAddress(val0.Address()) != nil { + if voteSet.GetByAddress(val0.GetAddress()) != nil { t.Errorf("Expected GetByAddress(val0.Address) to be nil") } if voteSet.BitArray().GetIndex(0) { @@ -88,7 +88,7 @@ func TestAddVote(t *testing.T) { } vote := &Vote{ - ValidatorAddress: val0.Address(), + ValidatorAddress: val0.GetAddress(), ValidatorIndex: 0, // since privValidators are in order Height: height, Round: round, @@ -100,7 +100,7 @@ func TestAddVote(t *testing.T) { t.Error(err) } - if voteSet.GetByAddress(val0.Address()) == nil { + if voteSet.GetByAddress(val0.GetAddress()) == nil { t.Errorf("Expected GetByAddress(val0.Address) to be present") } if !voteSet.BitArray().GetIndex(0) { @@ -126,7 +126,7 @@ func Test2_3Majority(t *testing.T) { } // 6 out of 10 voted for nil. for i := 0; i < 6; i++ { - vote := withValidator(voteProto, privValidators[i].Address(), i) + vote := withValidator(voteProto, privValidators[i].GetAddress(), i) signAddVote(privValidators[i], vote, voteSet) } blockID, ok := voteSet.TwoThirdsMajority() @@ -136,7 +136,7 @@ func Test2_3Majority(t *testing.T) { // 7th validator voted for some blockhash { - vote := withValidator(voteProto, privValidators[6].Address(), 6) + vote := withValidator(voteProto, privValidators[6].GetAddress(), 6) signAddVote(privValidators[6], withBlockHash(vote, RandBytes(32)), voteSet) blockID, ok = voteSet.TwoThirdsMajority() if ok || !blockID.IsZero() { @@ -146,7 +146,7 @@ func Test2_3Majority(t *testing.T) { // 8th validator voted for nil. { - vote := withValidator(voteProto, privValidators[7].Address(), 7) + vote := withValidator(voteProto, privValidators[7].GetAddress(), 7) signAddVote(privValidators[7], vote, voteSet) blockID, ok = voteSet.TwoThirdsMajority() if !ok || !blockID.IsZero() { @@ -174,7 +174,7 @@ func Test2_3MajorityRedux(t *testing.T) { // 66 out of 100 voted for nil. for i := 0; i < 66; i++ { - vote := withValidator(voteProto, privValidators[i].Address(), i) + vote := withValidator(voteProto, privValidators[i].GetAddress(), i) signAddVote(privValidators[i], vote, voteSet) } blockID, ok := voteSet.TwoThirdsMajority() @@ -184,7 +184,7 @@ func Test2_3MajorityRedux(t *testing.T) { // 67th validator voted for nil { - vote := withValidator(voteProto, privValidators[66].Address(), 66) + vote := withValidator(voteProto, privValidators[66].GetAddress(), 66) signAddVote(privValidators[66], withBlockHash(vote, nil), voteSet) blockID, ok = voteSet.TwoThirdsMajority() if ok || !blockID.IsZero() { @@ -194,7 +194,7 @@ func Test2_3MajorityRedux(t *testing.T) { // 68th validator voted for a different BlockParts PartSetHeader { - vote := withValidator(voteProto, privValidators[67].Address(), 67) + vote := withValidator(voteProto, privValidators[67].GetAddress(), 67) blockPartsHeader := PartSetHeader{blockPartsTotal, crypto.CRandBytes(32)} signAddVote(privValidators[67], withBlockPartsHeader(vote, blockPartsHeader), voteSet) blockID, ok = voteSet.TwoThirdsMajority() @@ -205,7 +205,7 @@ func Test2_3MajorityRedux(t *testing.T) { // 69th validator voted for different BlockParts Total { - vote := withValidator(voteProto, privValidators[68].Address(), 68) + vote := withValidator(voteProto, privValidators[68].GetAddress(), 68) blockPartsHeader := PartSetHeader{blockPartsTotal + 1, blockPartsHeader.Hash} signAddVote(privValidators[68], withBlockPartsHeader(vote, blockPartsHeader), voteSet) blockID, ok = voteSet.TwoThirdsMajority() @@ -216,7 +216,7 @@ func Test2_3MajorityRedux(t *testing.T) { // 70th validator voted for different BlockHash { - vote := withValidator(voteProto, privValidators[69].Address(), 69) + vote := withValidator(voteProto, privValidators[69].GetAddress(), 69) signAddVote(privValidators[69], withBlockHash(vote, RandBytes(32)), voteSet) blockID, ok = voteSet.TwoThirdsMajority() if ok || !blockID.IsZero() { @@ -226,7 +226,7 @@ func Test2_3MajorityRedux(t *testing.T) { // 71st validator voted for the right BlockHash & BlockPartsHeader { - vote := withValidator(voteProto, privValidators[70].Address(), 70) + vote := withValidator(voteProto, privValidators[70].GetAddress(), 70) signAddVote(privValidators[70], vote, voteSet) blockID, ok = voteSet.TwoThirdsMajority() if !ok || !blockID.Equals(BlockID{blockHash, blockPartsHeader}) { @@ -250,7 +250,7 @@ func TestBadVotes(t *testing.T) { // val0 votes for nil. { - vote := withValidator(voteProto, privValidators[0].Address(), 0) + vote := withValidator(voteProto, privValidators[0].GetAddress(), 0) added, err := signAddVote(privValidators[0], vote, voteSet) if !added || err != nil { t.Errorf("Expected VoteSet.Add to succeed") @@ -259,7 +259,7 @@ func TestBadVotes(t *testing.T) { // val0 votes again for some block. { - vote := withValidator(voteProto, privValidators[0].Address(), 0) + vote := withValidator(voteProto, privValidators[0].GetAddress(), 0) added, err := signAddVote(privValidators[0], withBlockHash(vote, RandBytes(32)), voteSet) if added || err == nil { t.Errorf("Expected VoteSet.Add to fail, conflicting vote.") @@ -268,7 +268,7 @@ func TestBadVotes(t *testing.T) { // val1 votes on another height { - vote := withValidator(voteProto, privValidators[1].Address(), 1) + vote := withValidator(voteProto, privValidators[1].GetAddress(), 1) added, err := signAddVote(privValidators[1], withHeight(vote, height+1), voteSet) if added || err == nil { t.Errorf("Expected VoteSet.Add to fail, wrong height") @@ -277,7 +277,7 @@ func TestBadVotes(t *testing.T) { // val2 votes on another round { - vote := withValidator(voteProto, privValidators[2].Address(), 2) + vote := withValidator(voteProto, privValidators[2].GetAddress(), 2) added, err := signAddVote(privValidators[2], withRound(vote, round+1), voteSet) if added || err == nil { t.Errorf("Expected VoteSet.Add to fail, wrong round") @@ -286,7 +286,7 @@ func TestBadVotes(t *testing.T) { // val3 votes of another type. { - vote := withValidator(voteProto, privValidators[3].Address(), 3) + vote := withValidator(voteProto, privValidators[3].GetAddress(), 3) added, err := signAddVote(privValidators[3], withType(vote, VoteTypePrecommit), voteSet) if added || err == nil { t.Errorf("Expected VoteSet.Add to fail, wrong type") @@ -311,7 +311,7 @@ func TestConflicts(t *testing.T) { // val0 votes for nil. { - vote := withValidator(voteProto, privValidators[0].Address(), 0) + vote := withValidator(voteProto, privValidators[0].GetAddress(), 0) added, err := signAddVote(privValidators[0], vote, voteSet) if !added || err != nil { t.Errorf("Expected VoteSet.Add to succeed") @@ -320,7 +320,7 @@ func TestConflicts(t *testing.T) { // val0 votes again for blockHash1. { - vote := withValidator(voteProto, privValidators[0].Address(), 0) + vote := withValidator(voteProto, privValidators[0].GetAddress(), 0) added, err := signAddVote(privValidators[0], withBlockHash(vote, blockHash1), voteSet) if added { t.Errorf("Expected VoteSet.Add to fail, conflicting vote.") @@ -335,7 +335,7 @@ func TestConflicts(t *testing.T) { // val0 votes again for blockHash1. { - vote := withValidator(voteProto, privValidators[0].Address(), 0) + vote := withValidator(voteProto, privValidators[0].GetAddress(), 0) added, err := signAddVote(privValidators[0], withBlockHash(vote, blockHash1), voteSet) if !added { t.Errorf("Expected VoteSet.Add to succeed, called SetPeerMaj23().") @@ -350,7 +350,7 @@ func TestConflicts(t *testing.T) { // val0 votes again for blockHash1. { - vote := withValidator(voteProto, privValidators[0].Address(), 0) + vote := withValidator(voteProto, privValidators[0].GetAddress(), 0) added, err := signAddVote(privValidators[0], withBlockHash(vote, blockHash2), voteSet) if added { t.Errorf("Expected VoteSet.Add to fail, duplicate SetPeerMaj23() from peerA") @@ -362,7 +362,7 @@ func TestConflicts(t *testing.T) { // val1 votes for blockHash1. { - vote := withValidator(voteProto, privValidators[1].Address(), 1) + vote := withValidator(voteProto, privValidators[1].GetAddress(), 1) added, err := signAddVote(privValidators[1], withBlockHash(vote, blockHash1), voteSet) if !added || err != nil { t.Errorf("Expected VoteSet.Add to succeed") @@ -379,7 +379,7 @@ func TestConflicts(t *testing.T) { // val2 votes for blockHash2. { - vote := withValidator(voteProto, privValidators[2].Address(), 2) + vote := withValidator(voteProto, privValidators[2].GetAddress(), 2) added, err := signAddVote(privValidators[2], withBlockHash(vote, blockHash2), voteSet) if !added || err != nil { t.Errorf("Expected VoteSet.Add to succeed") @@ -399,7 +399,7 @@ func TestConflicts(t *testing.T) { // val2 votes for blockHash1. { - vote := withValidator(voteProto, privValidators[2].Address(), 2) + vote := withValidator(voteProto, privValidators[2].GetAddress(), 2) added, err := signAddVote(privValidators[2], withBlockHash(vote, blockHash1), voteSet) if !added { t.Errorf("Expected VoteSet.Add to succeed") @@ -439,7 +439,7 @@ func TestMakeCommit(t *testing.T) { // 6 out of 10 voted for some block. for i := 0; i < 6; i++ { - vote := withValidator(voteProto, privValidators[i].Address(), i) + vote := withValidator(voteProto, privValidators[i].GetAddress(), i) signAddVote(privValidators[i], vote, voteSet) } @@ -448,7 +448,7 @@ func TestMakeCommit(t *testing.T) { // 7th voted for some other block. { - vote := withValidator(voteProto, privValidators[6].Address(), 6) + vote := withValidator(voteProto, privValidators[6].GetAddress(), 6) vote = withBlockHash(vote, RandBytes(32)) vote = withBlockPartsHeader(vote, PartSetHeader{123, RandBytes(32)}) signAddVote(privValidators[6], vote, voteSet) @@ -456,7 +456,7 @@ func TestMakeCommit(t *testing.T) { // The 8th voted like everyone else. { - vote := withValidator(voteProto, privValidators[7].Address(), 7) + vote := withValidator(voteProto, privValidators[7].GetAddress(), 7) signAddVote(privValidators[7], vote, voteSet) } From 8ae2ffda8984e7b8d693521bfa7de907e141b7a6 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 21 Sep 2017 16:58:18 -0400 Subject: [PATCH 81/92] put funcs back in order to simplify review --- config/toml.go | 24 ++-- types/priv_validator.go | 256 ++++++++++++++++++++-------------------- 2 files changed, 138 insertions(+), 142 deletions(-) diff --git a/config/toml.go b/config/toml.go index 4366695db..5dcbe5332 100644 --- a/config/toml.go +++ b/config/toml.go @@ -127,16 +127,16 @@ var testGenesis = `{ }` var testPrivValidator = `{ - "address": "D028C9981F7A87F3093672BF0D5B0E2A1B3ED456", - "pub_key": { - "type": "ed25519", - "data": "3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8" - }, - "priv_key": { - "type": "ed25519", - "data": "27F82582AEFAE7AB151CFB01C48BB6C1A0DA78F9BDDA979A9F70A84D074EB07D3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8" - }, - "last_height": 0, - "last_round": 0, - "last_step": 0 + "address": "D028C9981F7A87F3093672BF0D5B0E2A1B3ED456", + "pub_key": { + "type": "ed25519", + "data": "3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8" + }, + "priv_key": { + "type": "ed25519", + "data": "27F82582AEFAE7AB151CFB01C48BB6C1A0DA78F9BDDA979A9F70A84D074EB07D3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8" + }, + "last_height": 0, + "last_round": 0, + "last_step": 0 }` diff --git a/types/priv_validator.go b/types/priv_validator.go index 6db240966..4357c6f3b 100644 --- a/types/priv_validator.go +++ b/types/priv_validator.go @@ -67,6 +67,32 @@ type PrivValidatorFS struct { mtx sync.Mutex } +// Signer is an interface that defines how to sign messages. +// It is the caller's duty to verify the msg before calling Sign, +// eg. to avoid double signing. +// Currently, the only callers are SignVote, SignProposal, and SignHeartbeat. +type Signer interface { + Sign(msg []byte) (crypto.Signature, error) +} + +// DefaultSigner implements Signer. +// It uses a standard, unencrypted crypto.PrivKey. +type DefaultSigner struct { + PrivKey crypto.PrivKey `json:"priv_key"` +} + +// NewDefaultSigner returns an instance of DefaultSigner. +func NewDefaultSigner(priv crypto.PrivKey) *DefaultSigner { + return &DefaultSigner{ + PrivKey: priv, + } +} + +// Sign implements Signer. It signs the byte slice with a private key. +func (ds *DefaultSigner) Sign(msg []byte) (crypto.Signature, error) { + return ds.PrivKey.Sign(msg), nil +} + // GetAddress returns the address of the validator. // Implements PrivValidator. func (pv *PrivValidatorFS) GetAddress() data.Bytes { @@ -79,40 +105,68 @@ func (pv *PrivValidatorFS) GetPubKey() crypto.PubKey { return pv.PubKey } -// SignVote signs a canonical representation of the vote, along with the chainID. -// Implements PrivValidator. -func (privVal *PrivValidatorFS) SignVote(chainID string, vote *Vote) error { - privVal.mtx.Lock() - defer privVal.mtx.Unlock() - signature, err := privVal.signBytesHRS(vote.Height, vote.Round, voteToStep(vote), SignBytes(chainID, vote)) - if err != nil { - return errors.New(cmn.Fmt("Error signing vote: %v", err)) +// GenPrivValidatorFS generates a new validator with randomly generated private key +// and sets the filePath, but does not call Save(). +func GenPrivValidatorFS(filePath string) *PrivValidatorFS { + privKey := crypto.GenPrivKeyEd25519().Wrap() + return &PrivValidatorFS{ + Address: privKey.PubKey().Address(), + PubKey: privKey.PubKey(), + PrivKey: privKey, + LastStep: stepNone, + Signer: NewDefaultSigner(privKey), + filePath: filePath, } - vote.Signature = signature - return nil } -// SignProposal signs a canonical representation of the proposal, along with the chainID. -// Implements PrivValidator. -func (privVal *PrivValidatorFS) SignProposal(chainID string, proposal *Proposal) error { - privVal.mtx.Lock() - defer privVal.mtx.Unlock() - signature, err := privVal.signBytesHRS(proposal.Height, proposal.Round, stepPropose, SignBytes(chainID, proposal)) +// LoadPrivValidatorFS loads a PrivValidatorFS from the filePath. +func LoadPrivValidatorFS(filePath string) *PrivValidatorFS { + privValJSONBytes, err := ioutil.ReadFile(filePath) if err != nil { - return fmt.Errorf("Error signing proposal: %v", err) + cmn.Exit(err.Error()) } - proposal.Signature = signature - return nil + privVal := PrivValidatorFS{} + err = json.Unmarshal(privValJSONBytes, &privVal) + if err != nil { + cmn.Exit(cmn.Fmt("Error reading PrivValidator from %v: %v\n", filePath, err)) + } + + privVal.filePath = filePath + privVal.Signer = NewDefaultSigner(privVal.PrivKey) + return &privVal } -// SignHeartbeat signs a canonical representation of the heartbeat, along with the chainID. -// Implements PrivValidator. -func (privVal *PrivValidatorFS) SignHeartbeat(chainID string, heartbeat *Heartbeat) error { - privVal.mtx.Lock() - defer privVal.mtx.Unlock() - var err error - heartbeat.Signature, err = privVal.Signer.Sign(SignBytes(chainID, heartbeat)) - return err +// LoadOrGenPrivValidatorFS loads a PrivValidatorFS from the given filePath +// or else generates a new one and saves it to the filePath. +func LoadOrGenPrivValidatorFS(filePath string) *PrivValidatorFS { + var PrivValidatorFS *PrivValidatorFS + if _, err := os.Stat(filePath); err == nil { + PrivValidatorFS = LoadPrivValidatorFS(filePath) + } else { + PrivValidatorFS = GenPrivValidatorFS(filePath) + PrivValidatorFS.Save() + } + return PrivValidatorFS +} + +// LoadPrivValidatorWithSigner loads a PrivValidatorFS with a custom +// signer object. The PrivValidatorFS handles double signing prevention by persisting +// data to the filePath, while the Signer handles the signing. +// If the filePath does not exist, the PrivValidatorFS must be created manually and saved. +func LoadPrivValidatorFSWithSigner(filePath string, signerFunc func(crypto.PubKey) Signer) *PrivValidatorFS { + privValJSONBytes, err := ioutil.ReadFile(filePath) + if err != nil { + cmn.Exit(err.Error()) + } + privVal := PrivValidatorFS{} + err = json.Unmarshal(privValJSONBytes, &privVal) + if err != nil { + cmn.Exit(cmn.Fmt("Error reading PrivValidator from %v: %v\n", filePath, err)) + } + + privVal.filePath = filePath + privVal.Signer = signerFunc(privVal.PubKey) + return &privVal } // Save persists the PrivValidatorFS to disk. @@ -138,6 +192,43 @@ func (privVal *PrivValidatorFS) save() { } } +// Reset resets all fields in the PrivValidatorFS. +// NOTE: Unsafe! +func (privVal *PrivValidatorFS) Reset() { + privVal.LastHeight = 0 + privVal.LastRound = 0 + privVal.LastStep = 0 + privVal.LastSignature = crypto.Signature{} + privVal.LastSignBytes = nil + privVal.Save() +} + +// SignVote signs a canonical representation of the vote, along with the chainID. +// Implements PrivValidator. +func (privVal *PrivValidatorFS) SignVote(chainID string, vote *Vote) error { + privVal.mtx.Lock() + defer privVal.mtx.Unlock() + signature, err := privVal.signBytesHRS(vote.Height, vote.Round, voteToStep(vote), SignBytes(chainID, vote)) + if err != nil { + return errors.New(cmn.Fmt("Error signing vote: %v", err)) + } + vote.Signature = signature + return nil +} + +// SignProposal signs a canonical representation of the proposal, along with the chainID. +// Implements PrivValidator. +func (privVal *PrivValidatorFS) SignProposal(chainID string, proposal *Proposal) error { + privVal.mtx.Lock() + defer privVal.mtx.Unlock() + signature, err := privVal.signBytesHRS(proposal.Height, proposal.Round, stepPropose, SignBytes(chainID, proposal)) + if err != nil { + return fmt.Errorf("Error signing proposal: %v", err) + } + proposal.Signature = signature + return nil +} + // signBytesHRS signs the given signBytes if the height/round/step (HRS) // are greater than the latest state. If the HRS are equal, // it returns the privValidator.LastSignature. @@ -193,15 +284,14 @@ func (privVal *PrivValidatorFS) signBytesHRS(height, round int, step int8, signB return sig, nil } -// Reset resets all fields in the PrivValidatorFS. -// NOTE: Unsafe! -func (privVal *PrivValidatorFS) Reset() { - privVal.LastHeight = 0 - privVal.LastRound = 0 - privVal.LastStep = 0 - privVal.LastSignature = crypto.Signature{} - privVal.LastSignBytes = nil - privVal.Save() +// SignHeartbeat signs a canonical representation of the heartbeat, along with the chainID. +// Implements PrivValidator. +func (privVal *PrivValidatorFS) SignHeartbeat(chainID string, heartbeat *Heartbeat) error { + privVal.mtx.Lock() + defer privVal.mtx.Unlock() + var err error + heartbeat.Signature, err = privVal.Signer.Sign(SignBytes(chainID, heartbeat)) + return err } // String returns a string representation of the PrivValidatorFS. @@ -209,100 +299,6 @@ func (privVal *PrivValidatorFS) String() string { return fmt.Sprintf("PrivValidator{%v LH:%v, LR:%v, LS:%v}", privVal.GetAddress(), privVal.LastHeight, privVal.LastRound, privVal.LastStep) } -// LoadOrGenPrivValidatorFS loads a PrivValidatorFS from the given filePath -// or else generates a new one and saves it to the filePath. -func LoadOrGenPrivValidatorFS(filePath string) *PrivValidatorFS { - var PrivValidatorFS *PrivValidatorFS - if _, err := os.Stat(filePath); err == nil { - PrivValidatorFS = LoadPrivValidatorFS(filePath) - } else { - PrivValidatorFS = GenPrivValidatorFS(filePath) - PrivValidatorFS.Save() - } - return PrivValidatorFS -} - -// LoadPrivValidatorFS loads a PrivValidatorFS from the filePath. -func LoadPrivValidatorFS(filePath string) *PrivValidatorFS { - privValJSONBytes, err := ioutil.ReadFile(filePath) - if err != nil { - cmn.Exit(err.Error()) - } - privVal := PrivValidatorFS{} - err = json.Unmarshal(privValJSONBytes, &privVal) - if err != nil { - cmn.Exit(cmn.Fmt("Error reading PrivValidator from %v: %v\n", filePath, err)) - } - - privVal.filePath = filePath - privVal.Signer = NewDefaultSigner(privVal.PrivKey) - return &privVal -} - -// GenPrivValidatorFS generates a new validator with randomly generated private key -// and sets the filePath, but does not call Save(). -func GenPrivValidatorFS(filePath string) *PrivValidatorFS { - privKey := crypto.GenPrivKeyEd25519().Wrap() - return &PrivValidatorFS{ - Address: privKey.PubKey().Address(), - PubKey: privKey.PubKey(), - PrivKey: privKey, - LastStep: stepNone, - Signer: NewDefaultSigner(privKey), - filePath: filePath, - } -} - -// LoadPrivValidatorWithSigner loads a PrivValidatorFS with a custom -// signer object. The PrivValidatorFS handles double signing prevention by persisting -// data to the filePath, while the Signer handles the signing. -// If the filePath does not exist, the PrivValidatorFS must be created manually and saved. -func LoadPrivValidatorFSWithSigner(filePath string, signerFunc func(crypto.PubKey) Signer) *PrivValidatorFS { - privValJSONBytes, err := ioutil.ReadFile(filePath) - if err != nil { - cmn.Exit(err.Error()) - } - privVal := PrivValidatorFS{} - err = json.Unmarshal(privValJSONBytes, &privVal) - if err != nil { - cmn.Exit(cmn.Fmt("Error reading PrivValidator from %v: %v\n", filePath, err)) - } - - privVal.filePath = filePath - privVal.Signer = signerFunc(privVal.PubKey) - return &privVal -} - -//------------------------------------- - -//------------------------------------- - -// Signer is an interface that defines how to sign messages. -// It is the caller's duty to verify the msg before calling Sign, -// eg. to avoid double signing. -// Currently, the only callers are SignVote, SignProposal, and SignHeartbeat. -type Signer interface { - Sign(msg []byte) (crypto.Signature, error) -} - -// DefaultSigner implements Signer. -// It uses a standard, unencrypted crypto.PrivKey. -type DefaultSigner struct { - PrivKey crypto.PrivKey `json:"priv_key"` -} - -// NewDefaultSigner returns an instance of DefaultSigner. -func NewDefaultSigner(priv crypto.PrivKey) *DefaultSigner { - return &DefaultSigner{ - PrivKey: priv, - } -} - -// Sign implements Signer. It signs the byte slice with a private key. -func (ds *DefaultSigner) Sign(msg []byte) (crypto.Signature, error) { - return ds.PrivKey.Sign(msg), nil -} - //------------------------------------- type PrivValidatorsByAddress []*PrivValidatorFS From 2131f8d3309124922416305a99d5b1ebef02589f Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 21 Sep 2017 17:08:17 -0400 Subject: [PATCH 82/92] some fixes from review --- cmd/tendermint/commands/run_node.go | 19 +++---------------- cmd/tendermint/main.go | 9 ++++++--- node/node.go | 28 +++++++++++++++------------- node/node_test.go | 4 ++-- types/priv_validator.go | 24 +++++++----------------- types/priv_validator_test.go | 26 +++++++++++++------------- 6 files changed, 46 insertions(+), 64 deletions(-) diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index 6a793706f..b3ef821aa 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -6,7 +6,7 @@ import ( "github.com/spf13/cobra" cfg "github.com/tendermint/tendermint/config" - "github.com/tendermint/tendermint/node" + nm "github.com/tendermint/tendermint/node" "github.com/tendermint/tendermint/proxy" "github.com/tendermint/tendermint/types" ) @@ -53,26 +53,13 @@ func DefaultSignerAndApp(config *cfg.Config) (types.PrivValidator, proxy.ClientC // NewRunNodeCmd returns the command that allows the CLI to start a // node. It can be used with a custom PrivValidator and in-process ABCI application. -func NewRunNodeCmd(signerAndApp FuncSignerAndApp) *cobra.Command { +func NewRunNodeCmd(nodeFunc nm.NodeProvider) *cobra.Command { cmd := &cobra.Command{ Use: "node", Short: "Run the tendermint node", RunE: func(cmd *cobra.Command, args []string) error { - genDocFile := config.GenesisFile() - genDoc, err := types.GenesisDocFromFile(genDocFile) - if err != nil { - return err - } - config.ChainID = genDoc.ChainID - // Create & start node - privVal, clientCreator := signerAndApp(config) - n, err := node.NewNode(config, - privVal, - clientCreator, - node.DefaultGenesisDocProviderFunc(config), - node.DefaultDBProvider, - logger.With("module", "node")) + n, err := nodeFunc(config, logger) if err != nil { return fmt.Errorf("Failed to create node: %v", err) } diff --git a/cmd/tendermint/main.go b/cmd/tendermint/main.go index 1e2c00051..d1aba6f26 100644 --- a/cmd/tendermint/main.go +++ b/cmd/tendermint/main.go @@ -6,6 +6,7 @@ import ( "github.com/tendermint/tmlibs/cli" cmd "github.com/tendermint/tendermint/cmd/tendermint/commands" + nm "github.com/tendermint/tendermint/node" ) func main() { @@ -26,12 +27,14 @@ func main() { // Users wishing to: // * Use an external signer for their validators // * Supply an in-proc abci app + // * Supply a genesis doc file from another source + // * Provide their own DB implementation // can copy this file and use something other than the - // DefaultSignerAndApp function - signerAndApp := cmd.DefaultSignerAndApp + // DefaultNewNode function + nodeFunc := nm.DefaultNewNode // Create & start node - rootCmd.AddCommand(cmd.NewRunNodeCmd(signerAndApp)) + rootCmd.AddCommand(cmd.NewRunNodeCmd(nodeFunc)) cmd := cli.PrepareBaseCmd(rootCmd, "TM", os.ExpandEnv("$HOME/.tendermint")) cmd.Execute() diff --git a/node/node.go b/node/node.go index 0fb064514..834cb4c35 100644 --- a/node/node.go +++ b/node/node.go @@ -65,6 +65,21 @@ func DefaultGenesisDocProviderFunc(config *cfg.Config) GenesisDocProvider { } } +// NodeProvider takes a config and a logger and returns a ready to go Node. +type NodeProvider func(*cfg.Config, log.Logger) (*Node, error) + +// DefaultNewNode returns a Tendermint node with default settings for the +// PrivValidator, ClientCreator, GenesisDoc, and DBProvider. +// It implements NodeProvider. +func DefaultNewNode(config *cfg.Config, logger log.Logger) (*Node, error) { + return NewNode(config, + types.LoadOrGenPrivValidatorFS(config.PrivValidatorFile()), + proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()), + DefaultGenesisDocProviderFunc(config), + DefaultDBProvider, + logger) +} + //------------------------------------------------------------------------------ // Node is the highest level interface to a full Tendermint node. @@ -94,19 +109,6 @@ type Node struct { txIndexer txindex.TxIndexer } -// NewNodeDefault returns a Tendermint node with default settings for the -// PrivValidator, ClientCreator, GenesisDoc, and DBProvider, -func NewNodeDefault(config *cfg.Config, logger log.Logger) (*Node, error) { - // Get PrivValidator - privValidator := types.LoadOrGenPrivValidatorFS(config.PrivValidatorFile()) - return NewNode(config, - privValidator, - proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()), - DefaultGenesisDocProviderFunc(config), - DefaultDBProvider, - logger) -} - // NewNode returns a new, ready to go, Tendermint Node. func NewNode(config *cfg.Config, privValidator types.PrivValidator, diff --git a/node/node_test.go b/node/node_test.go index 1b6d27f47..641e606c3 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -15,8 +15,8 @@ func TestNodeStartStop(t *testing.T) { config := cfg.ResetTestRoot("node_node_test") // Create & start node - n, err := NewNodeDefault(config, log.TestingLogger()) - assert.NoError(t, err, "expected no err on NewNodeDefault") + n, err := DefaultNewNode(config, log.TestingLogger()) + assert.NoError(t, err, "expected no err on DefaultNewNode") n.Start() t.Logf("Started node %v", n.sw.NodeInfo()) diff --git a/types/priv_validator.go b/types/priv_validator.go index 4357c6f3b..47276f288 100644 --- a/types/priv_validator.go +++ b/types/priv_validator.go @@ -121,19 +121,9 @@ func GenPrivValidatorFS(filePath string) *PrivValidatorFS { // LoadPrivValidatorFS loads a PrivValidatorFS from the filePath. func LoadPrivValidatorFS(filePath string) *PrivValidatorFS { - privValJSONBytes, err := ioutil.ReadFile(filePath) - if err != nil { - cmn.Exit(err.Error()) - } - privVal := PrivValidatorFS{} - err = json.Unmarshal(privValJSONBytes, &privVal) - if err != nil { - cmn.Exit(cmn.Fmt("Error reading PrivValidator from %v: %v\n", filePath, err)) - } - - privVal.filePath = filePath - privVal.Signer = NewDefaultSigner(privVal.PrivKey) - return &privVal + return LoadPrivValidatorFSWithSigner(filePath, func(privVal PrivValidator) Signer { + return NewDefaultSigner(privVal.(*PrivValidatorFS).PrivKey) + }) } // LoadOrGenPrivValidatorFS loads a PrivValidatorFS from the given filePath @@ -153,20 +143,20 @@ func LoadOrGenPrivValidatorFS(filePath string) *PrivValidatorFS { // signer object. The PrivValidatorFS handles double signing prevention by persisting // data to the filePath, while the Signer handles the signing. // If the filePath does not exist, the PrivValidatorFS must be created manually and saved. -func LoadPrivValidatorFSWithSigner(filePath string, signerFunc func(crypto.PubKey) Signer) *PrivValidatorFS { +func LoadPrivValidatorFSWithSigner(filePath string, signerFunc func(PrivValidator) Signer) *PrivValidatorFS { privValJSONBytes, err := ioutil.ReadFile(filePath) if err != nil { cmn.Exit(err.Error()) } - privVal := PrivValidatorFS{} + privVal := &PrivValidatorFS{} err = json.Unmarshal(privValJSONBytes, &privVal) if err != nil { cmn.Exit(cmn.Fmt("Error reading PrivValidator from %v: %v\n", filePath, err)) } privVal.filePath = filePath - privVal.Signer = signerFunc(privVal.PubKey) - return &privVal + privVal.Signer = signerFunc(privVal) + return privVal } // Save persists the PrivValidatorFS to disk. diff --git a/types/priv_validator_test.go b/types/priv_validator_test.go index d56cb9ec9..5b6026740 100644 --- a/types/priv_validator_test.go +++ b/types/priv_validator_test.go @@ -29,19 +29,19 @@ func TestLoadValidator(t *testing.T) { require.Nil(err, "%+v", err) serialized := fmt.Sprintf(`{ - "address": "%s", - "pub_key": { - "type": "ed25519", - "data": "%s" - }, - "last_height": 0, - "last_round": 0, - "last_step": 0, - "last_signature": null, - "priv_key": { - "type": "ed25519", - "data": "%s" - } + "address": "%s", + "pub_key": { + "type": "ed25519", + "data": "%s" + }, + "last_height": 0, + "last_round": 0, + "last_step": 0, + "last_signature": null, + "priv_key": { + "type": "ed25519", + "data": "%s" + } }`, addrStr, pubStr, privStr) val := PrivValidatorFS{} From 756818f940ea68b8f64044c49683b7b038421c83 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 21 Sep 2017 21:44:36 -0400 Subject: [PATCH 83/92] fixes from review --- .../commands/reset_priv_validator.go | 6 +++--- cmd/tendermint/commands/run_node.go | 19 ++----------------- cmd/tendermint/main.go | 2 +- node/node.go | 4 ++-- types/priv_validator.go | 4 ++-- 5 files changed, 10 insertions(+), 25 deletions(-) diff --git a/cmd/tendermint/commands/reset_priv_validator.go b/cmd/tendermint/commands/reset_priv_validator.go index 6255f9613..b9c08715f 100644 --- a/cmd/tendermint/commands/reset_priv_validator.go +++ b/cmd/tendermint/commands/reset_priv_validator.go @@ -27,7 +27,7 @@ var ResetPrivValidatorCmd = &cobra.Command{ // ResetAll removes the privValidator files. // Exported so other CLI tools can use it func ResetAll(dbDir, privValFile string, logger log.Logger) { - resetPrivValidatorLocal(privValFile, logger) + resetPrivValidatorFS(privValFile, logger) os.RemoveAll(dbDir) logger.Info("Removed all data", "dir", dbDir) } @@ -41,10 +41,10 @@ func resetAll(cmd *cobra.Command, args []string) { // XXX: this is totally unsafe. // it's only suitable for testnets. func resetPrivValidator(cmd *cobra.Command, args []string) { - resetPrivValidatorLocal(config.PrivValidatorFile(), logger) + resetPrivValidatorFS(config.PrivValidatorFile(), logger) } -func resetPrivValidatorLocal(privValFile string, logger log.Logger) { +func resetPrivValidatorFS(privValFile string, logger log.Logger) { // Get PrivValidator if _, err := os.Stat(privValFile); err == nil { privValidator := types.LoadPrivValidatorFS(privValFile) diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index b3ef821aa..f0a1eede2 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -5,10 +5,7 @@ import ( "github.com/spf13/cobra" - cfg "github.com/tendermint/tendermint/config" nm "github.com/tendermint/tendermint/node" - "github.com/tendermint/tendermint/proxy" - "github.com/tendermint/tendermint/types" ) // AddNodeFlags exposes some common configuration options on the command-line @@ -39,27 +36,15 @@ func AddNodeFlags(cmd *cobra.Command) { cmd.Flags().Bool("consensus.create_empty_blocks", config.Consensus.CreateEmptyBlocks, "Set this to false to only produce blocks when there are txs or when the AppHash changes") } -// FuncSignerAndApp takes a config and returns a PrivValidator and ClientCreator. -// It allows other projects to make Tendermint binaries with custom signers and applications. -type FuncSignerAndApp func(*cfg.Config) (types.PrivValidator, proxy.ClientCreator) - -// DefaultSignerAndApp is a default FuncSignerAndApp that returns a PrivValidatorFS -// and a DefaultClientCreator using the relevant fields from the config. -func DefaultSignerAndApp(config *cfg.Config) (types.PrivValidator, proxy.ClientCreator) { - privValidator := types.LoadOrGenPrivValidatorFS(config.PrivValidatorFile()) - clientCreator := proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()) - return privValidator, clientCreator -} - // NewRunNodeCmd returns the command that allows the CLI to start a // node. It can be used with a custom PrivValidator and in-process ABCI application. -func NewRunNodeCmd(nodeFunc nm.NodeProvider) *cobra.Command { +func NewRunNodeCmd(nodeProvider nm.NodeProvider) *cobra.Command { cmd := &cobra.Command{ Use: "node", Short: "Run the tendermint node", RunE: func(cmd *cobra.Command, args []string) error { // Create & start node - n, err := nodeFunc(config, logger) + n, err := nodeProvider(config, logger) if err != nil { return fmt.Errorf("Failed to create node: %v", err) } diff --git a/cmd/tendermint/main.go b/cmd/tendermint/main.go index d1aba6f26..86ca1531d 100644 --- a/cmd/tendermint/main.go +++ b/cmd/tendermint/main.go @@ -28,7 +28,7 @@ func main() { // * Use an external signer for their validators // * Supply an in-proc abci app // * Supply a genesis doc file from another source - // * Provide their own DB implementation + // * Provide their own DB implementation // can copy this file and use something other than the // DefaultNewNode function nodeFunc := nm.DefaultNewNode diff --git a/node/node.go b/node/node.go index 834cb4c35..4286fece2 100644 --- a/node/node.go +++ b/node/node.go @@ -113,7 +113,7 @@ type Node struct { func NewNode(config *cfg.Config, privValidator types.PrivValidator, clientCreator proxy.ClientCreator, - genDocProvider GenesisDocProvider, + genesisDocProvider GenesisDocProvider, dbProvider DBProvider, logger log.Logger) (*Node, error) { @@ -134,7 +134,7 @@ func NewNode(config *cfg.Config, } state := sm.LoadState(stateDB) if state == nil { - genDoc, err := genDocProvider() + genDoc, err := genesisDocProvider() if err != nil { return nil, err } diff --git a/types/priv_validator.go b/types/priv_validator.go index 47276f288..66f02a6f6 100644 --- a/types/priv_validator.go +++ b/types/priv_validator.go @@ -258,7 +258,7 @@ func (privVal *PrivValidatorFS) signBytesHRS(height, round int, step int8, signB } // Sign - sig, err := privVal.Signer.Sign(signBytes) + sig, err := privVal.Sign(signBytes) if err != nil { return sig, err } @@ -280,7 +280,7 @@ func (privVal *PrivValidatorFS) SignHeartbeat(chainID string, heartbeat *Heartbe privVal.mtx.Lock() defer privVal.mtx.Unlock() var err error - heartbeat.Signature, err = privVal.Signer.Sign(SignBytes(chainID, heartbeat)) + heartbeat.Signature, err = privVal.Sign(SignBytes(chainID, heartbeat)) return err } From 24f7b9387ace8b80832b5daa901bc77a40acccd5 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 22 Sep 2017 00:05:39 -0400 Subject: [PATCH 84/92] more tests --- types/priv_validator.go | 10 +-- types/priv_validator_test.go | 118 ++++++++++++++++++++++++++++++++++- 2 files changed, 122 insertions(+), 6 deletions(-) diff --git a/types/priv_validator.go b/types/priv_validator.go index 66f02a6f6..8834eb7cc 100644 --- a/types/priv_validator.go +++ b/types/priv_validator.go @@ -129,14 +129,14 @@ func LoadPrivValidatorFS(filePath string) *PrivValidatorFS { // LoadOrGenPrivValidatorFS loads a PrivValidatorFS from the given filePath // or else generates a new one and saves it to the filePath. func LoadOrGenPrivValidatorFS(filePath string) *PrivValidatorFS { - var PrivValidatorFS *PrivValidatorFS + var privVal *PrivValidatorFS if _, err := os.Stat(filePath); err == nil { - PrivValidatorFS = LoadPrivValidatorFS(filePath) + privVal = LoadPrivValidatorFS(filePath) } else { - PrivValidatorFS = GenPrivValidatorFS(filePath) - PrivValidatorFS.Save() + privVal = GenPrivValidatorFS(filePath) + privVal.Save() } - return PrivValidatorFS + return privVal } // LoadPrivValidatorWithSigner loads a PrivValidatorFS with a custom diff --git a/types/priv_validator_test.go b/types/priv_validator_test.go index 5b6026740..ac91de861 100644 --- a/types/priv_validator_test.go +++ b/types/priv_validator_test.go @@ -4,14 +4,44 @@ import ( "encoding/hex" "encoding/json" "fmt" + "os" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" crypto "github.com/tendermint/go-crypto" + "github.com/tendermint/go-wire/data" + cmn "github.com/tendermint/tmlibs/common" ) -func TestLoadValidator(t *testing.T) { +func TestGenLoadValidator(t *testing.T) { + assert := assert.New(t) + + _, tempFilePath := cmn.Tempfile("priv_validator_") + privVal := GenPrivValidatorFS(tempFilePath) + + height := 100 + privVal.LastHeight = height + privVal.Save() + addr := privVal.GetAddress() + + privVal = LoadPrivValidatorFS(tempFilePath) + assert.Equal(addr, privVal.GetAddress(), "expected privval addr to be the same") + assert.Equal(height, privVal.LastHeight, "expected privval.LastHeight to have been saved") +} + +func TestLoadOrGenValidator(t *testing.T) { + assert := assert.New(t) + + _, tempFilePath := cmn.Tempfile("priv_validator_") + os.Remove(tempFilePath) + privVal := LoadOrGenPrivValidatorFS(tempFilePath) + addr := privVal.GetAddress() + privVal = LoadOrGenPrivValidatorFS(tempFilePath) + assert.Equal(addr, privVal.GetAddress(), "expected privval addr to be the same") +} + +func TestUnmarshalValidator(t *testing.T) { assert, require := assert.New(t), require.New(t) // create some fixed values @@ -58,3 +88,89 @@ func TestLoadValidator(t *testing.T) { require.Nil(err, "%+v", err) assert.JSONEq(serialized, string(out)) } + +func TestSignVote(t *testing.T) { + assert := assert.New(t) + + _, tempFilePath := cmn.Tempfile("priv_validator_") + privVal := GenPrivValidatorFS(tempFilePath) + + block1 := BlockID{[]byte{1, 2, 3}, PartSetHeader{}} + block2 := BlockID{[]byte{3, 2, 1}, PartSetHeader{}} + height, round := 10, 1 + voteType := VoteTypePrevote + + // sign a vote for first time + vote := newVote(privVal.Address, 0, height, round, voteType, block1) + err := privVal.SignVote("mychainid", vote) + assert.NoError(err, "expected no error signing vote") + + // try to sign the same vote again; should be fine + err = privVal.SignVote("mychainid", vote) + assert.NoError(err, "expected no error on signing same vote") + + // now try some bad votes + cases := []*Vote{ + newVote(privVal.Address, 0, height, round-1, voteType, block1), // round regression + newVote(privVal.Address, 0, height-1, round, voteType, block1), // height regression + newVote(privVal.Address, 0, height-2, round+4, voteType, block1), // height regression and different round + newVote(privVal.Address, 0, height, round, voteType, block2), // different block + } + + for _, c := range cases { + err = privVal.SignVote("mychainid", c) + assert.Error(err, "expected error on signing conflicting vote") + } +} + +func TestSignProposal(t *testing.T) { + assert := assert.New(t) + + _, tempFilePath := cmn.Tempfile("priv_validator_") + privVal := GenPrivValidatorFS(tempFilePath) + + block1 := PartSetHeader{5, []byte{1, 2, 3}} + block2 := PartSetHeader{10, []byte{3, 2, 1}} + height, round := 10, 1 + + // sign a proposal for first time + proposal := newProposal(height, round, block1) + err := privVal.SignProposal("mychainid", proposal) + assert.NoError(err, "expected no error signing proposal") + + // try to sign the same proposal again; should be fine + err = privVal.SignProposal("mychainid", proposal) + assert.NoError(err, "expected no error on signing same proposal") + + // now try some bad Proposals + cases := []*Proposal{ + newProposal(height, round-1, block1), // round regression + newProposal(height-1, round, block1), // height regression + newProposal(height-2, round+4, block1), // height regression and different round + newProposal(height, round, block2), // different block + } + + for _, c := range cases { + err = privVal.SignProposal("mychainid", c) + assert.Error(err, "expected error on signing conflicting proposal") + } +} + +func newVote(addr data.Bytes, idx, height, round int, typ byte, blockID BlockID) *Vote { + return &Vote{ + ValidatorAddress: addr, + ValidatorIndex: idx, + Height: height, + Round: round, + Type: typ, + BlockID: blockID, + } +} + +func newProposal(height, round int, partsHeader PartSetHeader) *Proposal { + return &Proposal{ + Height: height, + Round: round, + BlockPartsHeader: partsHeader, + } +} From ddb3d8945d4cea38a90819f263d3be5834ceb2a4 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 19 Jul 2017 15:03:36 -0400 Subject: [PATCH 85/92] p2p: allow listener with no external connection --- node/node.go | 13 +------------ p2p/listener.go | 11 +++++++---- rpc/lib/types/types.go | 1 + 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/node/node.go b/node/node.go index 4286fece2..824a0926c 100644 --- a/node/node.go +++ b/node/node.go @@ -309,7 +309,7 @@ func NewNode(config *cfg.Config, // OnStart starts the Node. It implements cmn.Service. func (n *Node) OnStart() error { // Create & add listener - protocol, address := ProtocolAndAddress(n.config.P2P.ListenAddress) + protocol, address := cmn.ProtocolAndAddress(n.config.P2P.ListenAddress) l := p2p.NewDefaultListener(protocol, address, n.config.P2P.SkipUPNP, n.Logger.With("module", "p2p")) n.sw.AddListener(l) @@ -535,15 +535,4 @@ func (n *Node) DialSeeds(seeds []string) error { return n.sw.DialSeeds(n.addrBook, seeds) } -// ProtocolAndAddress returns the transport protocol -// and the ip address from the given string. Defaults to tcp. -func ProtocolAndAddress(listenAddr string) (string, string) { - protocol, address := "tcp", listenAddr - parts := strings.SplitN(address, "://", 2) - if len(parts) == 2 { - protocol, address = parts[0], parts[1] - } - return protocol, address -} - //------------------------------------------------------------------------------ diff --git a/p2p/listener.go b/p2p/listener.go index 02d958d61..a382fbeda 100644 --- a/p2p/listener.go +++ b/p2p/listener.go @@ -87,7 +87,7 @@ func NewDefaultListener(protocol string, lAddr string, skipUPNP bool, logger log } // Otherwise just use the local address... if extAddr == nil { - extAddr = getNaiveExternalAddress(listenerPort) + extAddr = getNaiveExternalAddress(listenerPort, false, logger) } if extAddr == nil { cmn.PanicCrisis("Could not determine external address!") @@ -197,7 +197,7 @@ func getUPNPExternalAddress(externalPort, internalPort int, logger log.Logger) * } // TODO: use syscalls: http://pastebin.com/9exZG4rh -func getNaiveExternalAddress(port int) *NetAddress { +func getNaiveExternalAddress(port int, settleForLocal bool, logger log.Logger) *NetAddress { addrs, err := net.InterfaceAddrs() if err != nil { cmn.PanicCrisis(cmn.Fmt("Could not fetch interface addresses: %v", err)) @@ -209,10 +209,13 @@ func getNaiveExternalAddress(port int) *NetAddress { continue } v4 := ipnet.IP.To4() - if v4 == nil || v4[0] == 127 { + if v4 == nil || (!settleForLocal && v4[0] == 127) { continue } // loopback return NewNetAddressIPPort(ipnet.IP, uint16(port)) } - return nil + + // try again, but settle for local + logger.Info("Node may not be connected to internet. Settling for local address") + return getNaiveExternalAddress(port, true, logger) } diff --git a/rpc/lib/types/types.go b/rpc/lib/types/types.go index 0267c529c..4928db514 100644 --- a/rpc/lib/types/types.go +++ b/rpc/lib/types/types.go @@ -144,6 +144,7 @@ type WSRPCContext struct { // // Determine if its a unix or tcp socket. // If tcp, must specify the port; `0.0.0.0` will return incorrectly as "unix" since there's no port +// TODO: deprecate func SocketType(listenAddr string) string { socketType := "unix" if len(strings.Split(listenAddr, ":")) >= 2 { From df857266b69ce40d7965333cc232d5e56a4e6cb0 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 22 Sep 2017 10:14:05 -0400 Subject: [PATCH 86/92] update glide --- glide.lock | 15 ++++++++++++--- glide.yaml | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/glide.lock b/glide.lock index 6ab0354eb..90a648419 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 41581813ff97225a7feb86b5accb0fe4acb3e198b64592d7452240e9473c479f -updated: 2017-08-03T19:17:16.410522485Z +hash: 06ac555971ac41296cf6c869dfdf659ae911652d419e1a8c841aa791ff990556 +updated: 2017-09-22T10:13:57.706322194-04:00 imports: - name: github.com/btcsuite/btcd version: b8df516b4b267acf2de46be593a9d948d1d2c420 @@ -19,6 +19,12 @@ imports: - log/term - name: github.com/go-logfmt/logfmt version: 390ab7935ee28ec6b286364bba9b4dd6410cb3d5 +- name: github.com/go-playground/locales + version: 1e5f1161c6416a5ff48840eb8724a394e48cc534 + subpackages: + - currency +- name: github.com/go-playground/universal-translator + version: 71201497bace774495daed26a3874fd339e0b538 - name: github.com/go-stack/stack version: 100eb0c0a9c5b306ca2fb4f165df21d80ada4b82 - name: github.com/gogo/protobuf @@ -120,7 +126,7 @@ imports: - iavl - testutil - name: github.com/tendermint/tmlibs - version: 7ce4da1eee6004d627e780c8fe91e96d9b99e459 + version: 2130c329eb56aca8509ee1fec40d766f6541d8e7 subpackages: - autofile - cli @@ -128,6 +134,7 @@ imports: - clist - common - db + - events - flowrate - log - merkle @@ -184,6 +191,8 @@ imports: - status - tap - transport +- name: gopkg.in/go-playground/validator.v9 + version: d529ee1b0f30352444f507cc6cdac96bfd12decc - name: gopkg.in/yaml.v2 version: cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b testImports: diff --git a/glide.yaml b/glide.yaml index 78f9fec6e..0a57ff599 100644 --- a/glide.yaml +++ b/glide.yaml @@ -26,7 +26,7 @@ import: subpackages: - data - package: github.com/tendermint/tmlibs - version: ~0.2.2 + version: ~0.3.0 subpackages: - autofile - cli From 8311f5c6116efe3aa6157582104c664c887534f7 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 22 Sep 2017 11:42:40 -0400 Subject: [PATCH 87/92] abci.Info takes a struct; less merkleeyes --- consensus/mempool_test.go | 2 +- consensus/replay.go | 5 +++-- consensus/replay_test.go | 7 ++++--- glide.lock | 15 +++++++-------- glide.yaml | 10 ++-------- proxy/app_conn.go | 20 ++++++++++---------- proxy/app_conn_test.go | 10 +++++----- rpc/client/event_test.go | 15 ++++++++++++--- rpc/client/main_test.go | 6 +++--- rpc/client/mock/abci.go | 3 ++- rpc/client/rpc_test.go | 11 +++++------ rpc/core/abci.go | 3 ++- state/execution.go | 5 ++++- test/p2p/data/data/init.sh | 7 ------- 14 files changed, 60 insertions(+), 59 deletions(-) delete mode 100755 test/p2p/data/data/init.sh diff --git a/consensus/mempool_test.go b/consensus/mempool_test.go index 0f726b39e..f17d19f52 100644 --- a/consensus/mempool_test.go +++ b/consensus/mempool_test.go @@ -183,7 +183,7 @@ func NewCounterApplication() *CounterApplication { return &CounterApplication{} } -func (app *CounterApplication) Info() abci.ResponseInfo { +func (app *CounterApplication) Info(req abci.RequestInfo) abci.ResponseInfo { return abci.ResponseInfo{Data: Fmt("txs:%v", app.txCount)} } diff --git a/consensus/replay.go b/consensus/replay.go index 2104ffd5e..c5cb42191 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -19,6 +19,7 @@ import ( "github.com/tendermint/tendermint/proxy" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" + "github.com/tendermint/tendermint/version" ) // Functionality to replay blocks and messages on recovery from a crash. @@ -199,7 +200,7 @@ func (h *Handshaker) NBlocks() int { // TODO: retry the handshake/replay if it fails ? func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error { // handshake is done via info request on the query conn - res, err := proxyApp.Query().InfoSync() + res, err := proxyApp.Query().InfoSync(abci.RequestInfo{version.Version}) if err != nil { return errors.New(cmn.Fmt("Error calling Info: %v", err)) } @@ -235,7 +236,7 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp p // If appBlockHeight == 0 it means that we are at genesis and hence should send InitChain if appBlockHeight == 0 { validators := types.TM2PB.Validators(h.state.Validators) - proxyApp.Consensus().InitChainSync(validators) + proxyApp.Consensus().InitChainSync(abci.RequestInitChain{validators}) } // First handle edge cases and constraints on the storeBlockHeight diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 765434677..c478a0958 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -13,6 +13,7 @@ import ( "time" "github.com/tendermint/abci/example/dummy" + abci "github.com/tendermint/abci/types" crypto "github.com/tendermint/go-crypto" wire "github.com/tendermint/go-wire" cmn "github.com/tendermint/tmlibs/common" @@ -358,7 +359,7 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) { } // get the latest app hash from the app - res, err := proxyApp.Query().InfoSync() + res, err := proxyApp.Query().InfoSync(abci.RequestInfo{""}) if err != nil { t.Fatal(err) } @@ -396,7 +397,7 @@ func buildAppStateFromChain(proxyApp proxy.AppConns, } validators := types.TM2PB.Validators(state.Validators) - proxyApp.Consensus().InitChainSync(validators) + proxyApp.Consensus().InitChainSync(abci.RequestInitChain{validators}) defer proxyApp.Stop() switch mode { @@ -430,7 +431,7 @@ func buildTMStateFromChain(config *cfg.Config, state *sm.State, chain []*types.B defer proxyApp.Stop() validators := types.TM2PB.Validators(state.Validators) - proxyApp.Consensus().InitChainSync(validators) + proxyApp.Consensus().InitChainSync(abci.RequestInitChain{validators}) var latestAppHash []byte diff --git a/glide.lock b/glide.lock index 90a648419..20d7166c3 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 06ac555971ac41296cf6c869dfdf659ae911652d419e1a8c841aa791ff990556 -updated: 2017-09-22T10:13:57.706322194-04:00 +hash: b55559e3306e64b9cbd25456874e392ee3372778beb081190956864073aa8b88 +updated: 2017-09-22T11:30:04.200558244-04:00 imports: - name: github.com/btcsuite/btcd version: b8df516b4b267acf2de46be593a9d948d1d2c420 @@ -99,7 +99,7 @@ imports: - leveldb/table - leveldb/util - name: github.com/tendermint/abci - version: 864d1f80b36b440bde030a5c18d8ac3aa8c2949d + version: 191c4b6d176169ffc7f9972d490fa362a3b7d940 subpackages: - client - example/counter @@ -112,19 +112,16 @@ imports: - edwards25519 - extra25519 - name: github.com/tendermint/go-crypto - version: 95b7c9e09c49b91bfbb71bb63dd514eb55450f16 + version: 311e8c1bf00fa5868daad4f8ea56dcad539182c0 - name: github.com/tendermint/go-wire version: 5f88da3dbc1a72844e6dfaf274ce87f851d488eb subpackages: - data - data/base58 - name: github.com/tendermint/merkleeyes - version: 102aaf5a8ffda1846413fb22805a94def2045b9f + version: 2f6e5d31e7a35045d8d0a5895cb1fec33dd4d32b subpackages: - - app - - client - iavl - - testutil - name: github.com/tendermint/tmlibs version: 2130c329eb56aca8509ee1fec40d766f6541d8e7 subpackages: @@ -209,3 +206,5 @@ testImports: subpackages: - assert - require +- name: github.com/tendermint/iavl + version: 8fa7ec23377d7af91fa84b83a16abd24cc3acc71 diff --git a/glide.yaml b/glide.yaml index 0a57ff599..7b924aacc 100644 --- a/glide.yaml +++ b/glide.yaml @@ -14,13 +14,13 @@ import: - package: github.com/spf13/cobra - package: github.com/spf13/viper - package: github.com/tendermint/abci - version: v0.5.0 + version: ~0.6.0 subpackages: - client - example/dummy - types - package: github.com/tendermint/go-crypto - version: ~0.2.2 + version: ~0.3.0 - package: github.com/tendermint/go-wire version: ~0.6.2 subpackages: @@ -46,12 +46,6 @@ import: subpackages: - context - package: google.golang.org/grpc -- package: github.com/tendermint/merkleeyes - version: ~0.2.4 - subpackages: - - app - - iavl - - testutil testImport: - package: github.com/go-kit/kit subpackages: diff --git a/proxy/app_conn.go b/proxy/app_conn.go index 8b007737c..9121e8db8 100644 --- a/proxy/app_conn.go +++ b/proxy/app_conn.go @@ -12,9 +12,9 @@ type AppConnConsensus interface { SetResponseCallback(abcicli.Callback) Error() error - InitChainSync(validators []*types.Validator) (err error) + InitChainSync(types.RequestInitChain) (err error) - BeginBlockSync(hash []byte, header *types.Header) (err error) + BeginBlockSync(types.RequestBeginBlock) (err error) DeliverTxAsync(tx []byte) *abcicli.ReqRes EndBlockSync(height uint64) (types.ResponseEndBlock, error) CommitSync() (res types.Result) @@ -34,8 +34,8 @@ type AppConnQuery interface { Error() error EchoSync(string) (res types.Result) - InfoSync() (resInfo types.ResponseInfo, err error) - QuerySync(reqQuery types.RequestQuery) (resQuery types.ResponseQuery, err error) + InfoSync(types.RequestInfo) (types.ResponseInfo, error) + QuerySync(types.RequestQuery) (types.ResponseQuery, error) // SetOptionSync(key string, value string) (res types.Result) } @@ -61,12 +61,12 @@ func (app *appConnConsensus) Error() error { return app.appConn.Error() } -func (app *appConnConsensus) InitChainSync(validators []*types.Validator) (err error) { - return app.appConn.InitChainSync(validators) +func (app *appConnConsensus) InitChainSync(req types.RequestInitChain) (err error) { + return app.appConn.InitChainSync(req) } -func (app *appConnConsensus) BeginBlockSync(hash []byte, header *types.Header) (err error) { - return app.appConn.BeginBlockSync(hash, header) +func (app *appConnConsensus) BeginBlockSync(req types.RequestBeginBlock) (err error) { + return app.appConn.BeginBlockSync(req) } func (app *appConnConsensus) DeliverTxAsync(tx []byte) *abcicli.ReqRes { @@ -135,8 +135,8 @@ func (app *appConnQuery) EchoSync(msg string) (res types.Result) { return app.appConn.EchoSync(msg) } -func (app *appConnQuery) InfoSync() (types.ResponseInfo, error) { - return app.appConn.InfoSync() +func (app *appConnQuery) InfoSync(req types.RequestInfo) (types.ResponseInfo, error) { + return app.appConn.InfoSync(req) } func (app *appConnQuery) QuerySync(reqQuery types.RequestQuery) (types.ResponseQuery, error) { diff --git a/proxy/app_conn_test.go b/proxy/app_conn_test.go index 159e0b3e1..0c700140b 100644 --- a/proxy/app_conn_test.go +++ b/proxy/app_conn_test.go @@ -17,7 +17,7 @@ import ( type AppConnTest interface { EchoAsync(string) *abcicli.ReqRes FlushSync() error - InfoSync() (types.ResponseInfo, error) + InfoSync(types.RequestInfo) (types.ResponseInfo, error) } type appConnTest struct { @@ -36,8 +36,8 @@ func (app *appConnTest) FlushSync() error { return app.appConn.FlushSync() } -func (app *appConnTest) InfoSync() (types.ResponseInfo, error) { - return app.appConn.InfoSync() +func (app *appConnTest) InfoSync(req types.RequestInfo) (types.ResponseInfo, error) { + return app.appConn.InfoSync(req) } //---------------------------------------- @@ -109,7 +109,7 @@ func BenchmarkEcho(b *testing.B) { proxy.FlushSync() b.StopTimer() - // info := proxy.InfoSync() + // info := proxy.InfoSync(types.RequestInfo{""}) //b.Log("N: ", b.N, info) } @@ -138,7 +138,7 @@ func TestInfo(t *testing.T) { proxy := NewAppConnTest(cli) t.Log("Connected") - resInfo, err := proxy.InfoSync() + resInfo, err := proxy.InfoSync(types.RequestInfo{""}) if err != nil { t.Errorf("Unexpected error: %v", err) } diff --git a/rpc/client/event_test.go b/rpc/client/event_test.go index ad5522e49..a10021821 100644 --- a/rpc/client/event_test.go +++ b/rpc/client/event_test.go @@ -5,11 +5,20 @@ import ( "time" "github.com/stretchr/testify/require" - merktest "github.com/tendermint/merkleeyes/testutil" + + cmn "github.com/tendermint/tmlibs/common" + "github.com/tendermint/tendermint/rpc/client" "github.com/tendermint/tendermint/types" ) +// MakeTxKV returns a text transaction, allong with expected key, value pair +func MakeTxKV() ([]byte, []byte, []byte) { + k := []byte(cmn.RandStr(8)) + v := []byte(cmn.RandStr(8)) + return k, v, append(k, append([]byte("="), v...)...) +} + func TestHeaderEvents(t *testing.T) { require := require.New(t) for i, c := range GetClients() { @@ -76,7 +85,7 @@ func TestTxEventsSentWithBroadcastTxAsync(t *testing.T) { } // make the tx - _, _, tx := merktest.MakeTxKV() + _, _, tx := MakeTxKV() evtTyp := types.EventStringTx(types.Tx(tx)) // send async @@ -109,7 +118,7 @@ func TestTxEventsSentWithBroadcastTxSync(t *testing.T) { } // make the tx - _, _, tx := merktest.MakeTxKV() + _, _, tx := MakeTxKV() evtTyp := types.EventStringTx(types.Tx(tx)) // send async diff --git a/rpc/client/main_test.go b/rpc/client/main_test.go index 5ec911e5d..7c5acd390 100644 --- a/rpc/client/main_test.go +++ b/rpc/client/main_test.go @@ -4,7 +4,7 @@ import ( "os" "testing" - meapp "github.com/tendermint/merkleeyes/app" + "github.com/tendermint/abci/example/dummy" nm "github.com/tendermint/tendermint/node" rpctest "github.com/tendermint/tendermint/rpc/test" ) @@ -12,8 +12,8 @@ import ( var node *nm.Node func TestMain(m *testing.M) { - // start a tendermint node (and merkleeyes) in the background to test against - app := meapp.NewMerkleEyesApp("", 100) + // start a tendermint node (and dummy) in the background to test against + app := dummy.NewDummyApplication() node = rpctest.StartTendermint(app) code := m.Run() diff --git a/rpc/client/mock/abci.go b/rpc/client/mock/abci.go index 71fa90b2a..db3fa4f1d 100644 --- a/rpc/client/mock/abci.go +++ b/rpc/client/mock/abci.go @@ -6,6 +6,7 @@ import ( "github.com/tendermint/tendermint/rpc/client" ctypes "github.com/tendermint/tendermint/rpc/core/types" "github.com/tendermint/tendermint/types" + "github.com/tendermint/tendermint/version" ) // ABCIApp will send all abci related request to the named app, @@ -20,7 +21,7 @@ func (a ABCIApp) _assertABCIClient() client.ABCIClient { } func (a ABCIApp) ABCIInfo() (*ctypes.ResultABCIInfo, error) { - return &ctypes.ResultABCIInfo{a.App.Info()}, nil + return &ctypes.ResultABCIInfo{a.App.Info(abci.RequestInfo{version.Version})}, nil } func (a ABCIApp) ABCIQuery(path string, data data.Bytes, prove bool) (*ctypes.ResultABCIQuery, error) { diff --git a/rpc/client/rpc_test.go b/rpc/client/rpc_test.go index 33cd4b432..77de9f6e8 100644 --- a/rpc/client/rpc_test.go +++ b/rpc/client/rpc_test.go @@ -7,7 +7,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/tendermint/merkleeyes/iavl" - merktest "github.com/tendermint/merkleeyes/testutil" "github.com/tendermint/tendermint/rpc/client" rpctest "github.com/tendermint/tendermint/rpc/test" "github.com/tendermint/tendermint/types" @@ -49,7 +48,7 @@ func TestInfo(t *testing.T) { require.Nil(t, err, "%d: %+v", i, err) // TODO: this is not correct - fix merkleeyes! // assert.EqualValues(t, status.LatestBlockHeight, info.Response.LastBlockHeight) - assert.True(t, strings.HasPrefix(info.Response.Data, "size")) + assert.True(t, strings.Contains(info.Response.Data, "size")) } } @@ -115,7 +114,7 @@ func TestAppCalls(t *testing.T) { assert.NotNil(err) // no block yet // write something - k, v, tx := merktest.MakeTxKV() + k, v, tx := MakeTxKV() bres, err := c.BroadcastTxCommit(tx) require.Nil(err, "%d: %+v", i, err) require.True(bres.DeliverTx.Code.IsOK()) @@ -193,7 +192,7 @@ func TestBroadcastTxSync(t *testing.T) { initMempoolSize := mempool.Size() for i, c := range GetClients() { - _, _, tx := merktest.MakeTxKV() + _, _, tx := MakeTxKV() bres, err := c.BroadcastTxSync(tx) require.Nil(err, "%d: %+v", i, err) require.True(bres.Code.IsOK()) @@ -211,7 +210,7 @@ func TestBroadcastTxCommit(t *testing.T) { mempool := node.MempoolReactor().Mempool for i, c := range GetClients() { - _, _, tx := merktest.MakeTxKV() + _, _, tx := MakeTxKV() bres, err := c.BroadcastTxCommit(tx) require.Nil(err, "%d: %+v", i, err) require.True(bres.CheckTx.Code.IsOK()) @@ -226,7 +225,7 @@ func TestTx(t *testing.T) { // first we broadcast a tx c := getHTTPClient() - _, _, tx := merktest.MakeTxKV() + _, _, tx := MakeTxKV() bres, err := c.BroadcastTxCommit(tx) require.Nil(err, "%+v", err) diff --git a/rpc/core/abci.go b/rpc/core/abci.go index 21508caf7..06275a9e3 100644 --- a/rpc/core/abci.go +++ b/rpc/core/abci.go @@ -4,6 +4,7 @@ import ( abci "github.com/tendermint/abci/types" data "github.com/tendermint/go-wire/data" ctypes "github.com/tendermint/tendermint/rpc/core/types" + "github.com/tendermint/tendermint/version" ) // Query the application for some information. @@ -86,7 +87,7 @@ func ABCIQuery(path string, data data.Bytes, prove bool) (*ctypes.ResultABCIQuer // } // ``` func ABCIInfo() (*ctypes.ResultABCIInfo, error) { - resInfo, err := proxyAppQuery.InfoSync() + resInfo, err := proxyAppQuery.InfoSync(abci.RequestInfo{version.Version}) if err != nil { return nil, err } diff --git a/state/execution.go b/state/execution.go index 1785f6025..b917bfbe8 100644 --- a/state/execution.go +++ b/state/execution.go @@ -83,7 +83,10 @@ func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnCo proxyAppConn.SetResponseCallback(proxyCb) // Begin block - err := proxyAppConn.BeginBlockSync(block.Hash(), types.TM2PB.Header(block.Header)) + err := proxyAppConn.BeginBlockSync(abci.RequestBeginBlock{ + block.Hash(), + types.TM2PB.Header(block.Header), + }) if err != nil { logger.Error("Error in proxyAppConn.BeginBlock", "err", err) return nil, err diff --git a/test/p2p/data/data/init.sh b/test/p2p/data/data/init.sh deleted file mode 100755 index 41122a547..000000000 --- a/test/p2p/data/data/init.sh +++ /dev/null @@ -1,7 +0,0 @@ -#! /bin/bash -# This is a sample bash script for MerkleEyes. -# NOTE: mintnet expects data.sock to be created - -go get github.com/tendermint/merkleeyes/cmd/merkleeyes - -merkleeyes server --address="unix:///data/tendermint/data/data.sock" \ No newline at end of file From 7d983a548bc5d66bf9d32b2438f6091f3d8db481 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 22 Sep 2017 11:44:25 -0400 Subject: [PATCH 88/92] changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9aa179de6..4ccc7e0a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,7 +28,7 @@ BUG FIXES: - Graceful handling/recovery for apps that have non-determinism or fail to halt - Graceful handling/recovery for violations of safety, or liveness -## 0.11.0 (Date) +## 0.11.0 (September 22, 2017) BREAKING: - state: every validator set change is persisted to disk, which required some changes to the `State` structure. @@ -38,6 +38,7 @@ BREAKING: - rpc: various changes to match JSONRPC spec (http://www.jsonrpc.org/specification), including breaking ones: - requests that previously returned HTTP code 4XX now return 200 with an error code in the JSONRPC. - `rpctypes.RPCResponse` uses new `RPCError` type instead of `string`. + - abci: Info, BeginBlock, InitChain all take structs FEATURES: - rpc: `/validators?height=X` allows querying of validators at previous heights. From db034e079adbc1972da1d2e91194c0faa80843f7 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 22 Sep 2017 11:44:57 -0400 Subject: [PATCH 89/92] version bump --- version/version.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/version/version.go b/version/version.go index 97dbbfa0e..504aee68b 100644 --- a/version/version.go +++ b/version/version.go @@ -1,12 +1,12 @@ package version const Maj = "0" -const Min = "10" -const Fix = "4" +const Min = "11" +const Fix = "0" var ( // The full version string - Version = "0.10.4" + Version = "0.11.0" // GitCommit is set with --ldflags "-X main.gitCommit=$(git rev-parse HEAD)" GitCommit string From d1a00c684e161a6df867ff91dfd58fe98baaafc9 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 22 Sep 2017 12:00:37 -0400 Subject: [PATCH 90/92] types: comments --- types/block_meta.go | 2 ++ types/heartbeat.go | 5 +++++ types/keys.go | 1 + types/proposal.go | 10 +++++++++- types/protobuf.go | 3 ++- types/services.go | 10 ++++++++++ types/tx.go | 14 +++++++++----- types/validator.go | 2 ++ types/validator_set.go | 2 ++ 9 files changed, 42 insertions(+), 7 deletions(-) diff --git a/types/block_meta.go b/types/block_meta.go index 8e5bd43e5..6dd502e4f 100644 --- a/types/block_meta.go +++ b/types/block_meta.go @@ -1,10 +1,12 @@ package types +// BlockMeta contains meta information about a block - namely, it's ID and Header. type BlockMeta struct { BlockID BlockID `json:"block_id"` // the block hash and partsethash Header *Header `json:"header"` // The block's Header } +// NewBlockMeta returns a new BlockMeta from the block and its blockParts. func NewBlockMeta(block *Block, blockParts *PartSet) *BlockMeta { return &BlockMeta{ BlockID: BlockID{block.Hash(), blockParts.Header()}, diff --git a/types/heartbeat.go b/types/heartbeat.go index 378f6202b..40a7b01b0 100644 --- a/types/heartbeat.go +++ b/types/heartbeat.go @@ -10,6 +10,8 @@ import ( cmn "github.com/tendermint/tmlibs/common" ) +// Heartbeat is a simple vote-like structure so validators can alert others that +// they are alive and waiting for transactions. type Heartbeat struct { ValidatorAddress data.Bytes `json:"validator_address"` ValidatorIndex int `json:"validator_index"` @@ -19,6 +21,7 @@ type Heartbeat struct { Signature crypto.Signature `json:"signature"` } +// WriteSignBytes writes the Heartbeat for signing. func (heartbeat *Heartbeat) WriteSignBytes(chainID string, w io.Writer, n *int, err *error) { wire.WriteJSON(CanonicalJSONOnceHeartbeat{ chainID, @@ -26,11 +29,13 @@ func (heartbeat *Heartbeat) WriteSignBytes(chainID string, w io.Writer, n *int, }, w, n, err) } +// Copy makes a copy of the Heartbeat. func (heartbeat *Heartbeat) Copy() *Heartbeat { heartbeatCopy := *heartbeat return &heartbeatCopy } +// String returns a string representation of the Heartbeat. func (heartbeat *Heartbeat) String() string { if heartbeat == nil { return "nil-heartbeat" diff --git a/types/keys.go b/types/keys.go index 90591b959..992551191 100644 --- a/types/keys.go +++ b/types/keys.go @@ -1,5 +1,6 @@ package types +// UNSTABLE var ( PeerStateKey = "ConsensusReactor.peerState" PeerMempoolChKey = "MempoolReactor.peerMempoolCh" diff --git a/types/proposal.go b/types/proposal.go index 8406403c1..8fe959807 100644 --- a/types/proposal.go +++ b/types/proposal.go @@ -15,6 +15,11 @@ var ( ErrInvalidBlockPartHash = errors.New("Error invalid block part hash") ) +// Proposal defines a block proposal for the consensus. +// It refers to the block only by its PartSetHeader. +// It must be signed by the correct proposer for the given Height/Round +// to be considered valid. It may depend on votes from a previous round, +// a so-called Proof-of-Lock (POL) round, as noted in the POLRound and POLBlockID. type Proposal struct { Height int `json:"height"` Round int `json:"round"` @@ -24,7 +29,8 @@ type Proposal struct { Signature crypto.Signature `json:"signature"` } -// polRound: -1 if no polRound. +// NewProposal returns a new Proposal. +// If there is no POLRound, polRound should be -1. func NewProposal(height int, round int, blockPartsHeader PartSetHeader, polRound int, polBlockID BlockID) *Proposal { return &Proposal{ Height: height, @@ -35,11 +41,13 @@ func NewProposal(height int, round int, blockPartsHeader PartSetHeader, polRound } } +// String returns a string representation of the Proposal. func (p *Proposal) String() string { return fmt.Sprintf("Proposal{%v/%v %v (%v,%v) %v}", p.Height, p.Round, p.BlockPartsHeader, p.POLRound, p.POLBlockID, p.Signature) } +// WriteSignBytes writes the Proposal bytes for signing func (p *Proposal) WriteSignBytes(chainID string, w io.Writer, n *int, err *error) { wire.WriteJSON(CanonicalJSONOnceProposal{ ChainID: chainID, diff --git a/types/protobuf.go b/types/protobuf.go index 59994fea7..c8c9f8434 100644 --- a/types/protobuf.go +++ b/types/protobuf.go @@ -4,7 +4,8 @@ import ( "github.com/tendermint/abci/types" ) -// Convert tendermint types to protobuf types +// TM2PB is used for converting Tendermint types to protobuf types. +// UNSTABLE var TM2PB = tm2pb{} type tm2pb struct{} diff --git a/types/services.go b/types/services.go index 1805a0090..e34d846b5 100644 --- a/types/services.go +++ b/types/services.go @@ -4,6 +4,8 @@ import ( abci "github.com/tendermint/abci/types" ) +// NOTE: all types in this file are considered UNSTABLE + //------------------------------------------------------ // blockchain services types // NOTE: Interfaces used by RPC must be thread safe! @@ -12,8 +14,10 @@ import ( //------------------------------------------------------ // mempool +// Mempool defines the mempool interface. // Updates to the mempool need to be synchronized with committing a block // so apps can reset their transient state on Commit +// UNSTABLE type Mempool interface { Lock() Unlock() @@ -28,6 +32,8 @@ type Mempool interface { EnableTxsAvailable() } +// MockMempool is an empty implementation of a Mempool, useful for testing. +// UNSTABLE type MockMempool struct { } @@ -44,6 +50,8 @@ func (m MockMempool) EnableTxsAvailable() {} //------------------------------------------------------ // blockstore +// BlockStoreRPC is the block store interface used by the RPC. +// UNSTABLE type BlockStoreRPC interface { Height() int @@ -55,6 +63,8 @@ type BlockStoreRPC interface { LoadSeenCommit(height int) *Commit } +// BlockStore defines the BlockStore interface. +// UNSTABLE type BlockStore interface { BlockStoreRPC SaveBlock(block *Block, blockParts *PartSet, seenCommit *Commit) diff --git a/types/tx.go b/types/tx.go index 0334452e1..fbea8ff55 100644 --- a/types/tx.go +++ b/types/tx.go @@ -10,22 +10,26 @@ import ( "github.com/tendermint/tmlibs/merkle" ) +// Tx represents a transaction, which may contain arbitrary bytes. type Tx []byte -// NOTE: this is the hash of the go-wire encoded Tx. -// Tx has no types at this level, so just length-prefixed. -// Alternatively, it may make sense to add types here and let -// []byte be type 0x1 so we can have versioned txs if need be in the future. +// Hash returns the hash of the go-wire encoded Tx. +// Tx has no types at this level, so go-wire encoding only adds length-prefix. +// NOTE: It may make sense to add types here one day and let []byte be type 0x1 +// so we can have versioned txs if need be in the future. func (tx Tx) Hash() []byte { return merkle.SimpleHashFromBinary(tx) } +// String returns a string representation of the Tx. func (tx Tx) String() string { return fmt.Sprintf("Tx{%X}", []byte(tx)) } +// Txs is a slice of Tx. type Txs []Tx +// Hash returns the simple Merkle root hash of the Txs. func (txs Txs) Hash() []byte { // Recursive impl. // Copied from tmlibs/merkle to avoid allocations @@ -51,7 +55,7 @@ func (txs Txs) Index(tx Tx) int { return -1 } -// Index returns the index of this transaction hash in the list, or -1 if not found +// IndexByHash returns the index of this transaction hash in the list, or -1 if not found func (txs Txs) IndexByHash(hash []byte) int { for i := range txs { if bytes.Equal(txs[i].Hash(), hash) { diff --git a/types/validator.go b/types/validator.go index 506828913..7b167b273 100644 --- a/types/validator.go +++ b/types/validator.go @@ -106,6 +106,8 @@ func (vc validatorCodec) Compare(o1 interface{}, o2 interface{}) int { //-------------------------------------------------------------------------------- // For testing... +// RandValidator returns a randomized validator, useful for testing. +// UNSTABLE func RandValidator(randPower bool, minPower int64) (*Validator, *PrivValidatorFS) { _, tempFilePath := cmn.Tempfile("priv_validator_") privVal := GenPrivValidatorFS(tempFilePath) diff --git a/types/validator_set.go b/types/validator_set.go index 1e9e3e314..3ec389a48 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -368,7 +368,9 @@ func (ac accumComparable) Less(o interface{}) bool { //---------------------------------------- // For testing +// RandValidatorSet returns a randomized validator set, useful for testing. // NOTE: PrivValidator are in order. +// UNSTABLE func RandValidatorSet(numValidators int, votingPower int64) (*ValidatorSet, []*PrivValidatorFS) { vals := make([]*Validator, numValidators) privValidators := make([]*PrivValidatorFS, numValidators) From 661d336dd5fa1297ab07a60906cc2ee19e6d5857 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 22 Sep 2017 12:29:53 -0400 Subject: [PATCH 91/92] glide --- glide.lock | 72 +++++++++++++++++++++++++++--------------------------- glide.yaml | 4 +++ 2 files changed, 40 insertions(+), 36 deletions(-) diff --git a/glide.lock b/glide.lock index 20d7166c3..432e67e3c 100644 --- a/glide.lock +++ b/glide.lock @@ -1,18 +1,16 @@ -hash: b55559e3306e64b9cbd25456874e392ee3372778beb081190956864073aa8b88 -updated: 2017-09-22T11:30:04.200558244-04:00 +hash: e9abc35ed3adbcf850d3fec184572ed23a46c9489474f2867f9e57673149e56b +updated: 2017-09-22T12:27:41.940717166-04:00 imports: - name: github.com/btcsuite/btcd - version: b8df516b4b267acf2de46be593a9d948d1d2c420 + version: 4803a8291c92a1d2d41041b942a9a9e37deab065 subpackages: - btcec -- name: github.com/btcsuite/fastsha256 - version: 637e656429416087660c84436a2a035d69d54e2e - name: github.com/ebuchman/fail-test version: 95f809107225be108efcf10a3509e4ea6ceef3c4 - name: github.com/fsnotify/fsnotify version: 4da3e2cfbabc9f751898f250b49f2439785783a1 - name: github.com/go-kit/kit - version: d67bb4c202e3b91377d1079b110a6c9ce23ab2f8 + version: 0d313fb5fb3a94d87d61e6434785264e87a5d740 subpackages: - log - log/level @@ -26,22 +24,25 @@ imports: - name: github.com/go-playground/universal-translator version: 71201497bace774495daed26a3874fd339e0b538 - name: github.com/go-stack/stack - version: 100eb0c0a9c5b306ca2fb4f165df21d80ada4b82 + version: 817915b46b97fd7bb80e8ab6b69f01a53ac3eebf - name: github.com/gogo/protobuf - version: 9df9efe4c742f1a2bfdedf1c3b6902fc6e814c6b + version: 2adc21fd136931e0388e278825291678e1d98309 subpackages: - proto - name: github.com/golang/protobuf - version: 18c9bb3261723cd5401db4d0c9fbc5c3b6c70fe8 + version: 130e6b02ab059e7b717a096f397c5b60111cae74 subpackages: - proto + - ptypes - ptypes/any + - ptypes/duration + - ptypes/timestamp - name: github.com/golang/snappy version: 553a641470496b2327abcac10b36396bd98e45c9 - name: github.com/gorilla/websocket - version: a91eba7f97777409bc2c443f5534d41dd20c5720 + version: 6f34763140ed8887aed6a044912009832b4733d7 - name: github.com/hashicorp/hcl - version: 392dba7d905ed5d04a5794ba89f558b27e2ba1ca + version: 68e816d1c783414e79bc65b3994d9ab6b0a722ab subpackages: - hcl/ast - hcl/parser @@ -58,33 +59,31 @@ imports: - name: github.com/kr/logfmt version: b84e30acd515aadc4b783ad4ff83aff3299bdfe0 - name: github.com/magiconair/properties - version: 51463bfca2576e06c62a8504b5c0f06d61312647 + version: 8d7837e64d3c1ee4e54a880c5a920ab4316fc90a - name: github.com/mitchellh/mapstructure - version: cc8532a8e9a55ea36402aa21efdf403a60d34096 -- name: github.com/pelletier/go-buffruneio - version: c37440a7cf42ac63b919c752ca73a85067e05992 + version: d0303fe809921458f417bcf828397a65db30a7e4 - name: github.com/pelletier/go-toml - version: 5ccdfb18c776b740aecaf085c4d9a2779199c279 + version: 1d6b12b7cb290426e27e6b4e38b89fcda3aeef03 - name: github.com/pkg/errors version: 645ef00459ed84a119197bfb8d8205042c6df63d - name: github.com/rcrowley/go-metrics version: 1f30fe9094a513ce4c700b9a54458bbb0c96996c - name: github.com/spf13/afero - version: 9be650865eab0c12963d8753212f4f9c66cdcf12 + version: ee1bd8ee15a1306d1f9201acc41ef39cd9f99a1b subpackages: - mem - name: github.com/spf13/cast version: acbeb36b902d72a7a4c18e8f3241075e7ab763e4 - name: github.com/spf13/cobra - version: 4cdb38c072b86bf795d2c81de50784d9fdd6eb77 + version: b78744579491c1ceeaaa3b40205e56b0591b93a3 - name: github.com/spf13/jwalterweatherman - version: 8f07c835e5cc1450c082fe3a439cf87b0cbb2d99 + version: 12bd96e66386c1960ab0f74ced1362f66f552f7b - name: github.com/spf13/pflag - version: e57e3eeb33f795204c1ca35f56c44f83227c6e66 + version: 7aff26db30c1be810f9de5038ec5ef96ac41fd7c - name: github.com/spf13/viper - version: 0967fc9aceab2ce9da34061253ac10fb99bba5b2 + version: 25b30aa063fc18e48662b86996252eabdcf2f0c7 - name: github.com/syndtr/goleveldb - version: 8c81ea47d4c41a385645e133e15510fc6a2a74b4 + version: b89cc31ef7977104127d34c1bd31ebd1a9db2199 subpackages: - leveldb - leveldb/cache @@ -119,7 +118,7 @@ imports: - data - data/base58 - name: github.com/tendermint/merkleeyes - version: 2f6e5d31e7a35045d8d0a5895cb1fec33dd4d32b + version: 2a93256d2c6fbcc3b55673c0d2b96a7e32c6238b subpackages: - iavl - name: github.com/tendermint/tmlibs @@ -137,7 +136,7 @@ imports: - merkle - test - name: golang.org/x/crypto - version: c7af5bf2638a1164f2eb5467c39c6cffbd13a02e + version: 7d9177d70076375b9a59c8fde23d52d9c4a7ecd5 subpackages: - curve25519 - nacl/box @@ -148,7 +147,7 @@ imports: - ripemd160 - salsa20/salsa - name: golang.org/x/net - version: feeb485667d1fdabe727840fe00adc22431bc86e + version: 0744d001aa8470aaa53df28d32e5ceeb8af9bd70 subpackages: - context - http2 @@ -158,43 +157,46 @@ imports: - lex/httplex - trace - name: golang.org/x/sys - version: e62c3de784db939836898e5c19ffd41bece347da + version: 429f518978ab01db8bb6f44b66785088e7fba58b subpackages: - unix - name: golang.org/x/text - version: 470f45bf29f4147d6fbd7dfd0a02a848e49f5bf4 + version: 1cbadb444a806fd9430d14ad08967ed91da4fa0a subpackages: - secure/bidirule - transform - unicode/bidi - unicode/norm - name: google.golang.org/genproto - version: 411e09b969b1170a9f0c467558eb4c4c110d9c77 + version: 1e559d0a00eef8a9a43151db4665280bd8dd5886 subpackages: - googleapis/rpc/status - name: google.golang.org/grpc - version: 844f573616520565fdc6fb4db242321b5456fd6d + version: d4b75ebd4f9f8c4a2b1cdadbdbe0d7920431ccca subpackages: + - balancer - codes + - connectivity - credentials - - grpclb/grpc_lb_v1 + - grpclb/grpc_lb_v1/messages - grpclog - internal - keepalive - metadata - naming - peer + - resolver - stats - status - tap - transport - name: gopkg.in/go-playground/validator.v9 - version: d529ee1b0f30352444f507cc6cdac96bfd12decc + version: a021b2ec9a8a8bb970f3f15bc42617cb520e8a64 - name: gopkg.in/yaml.v2 - version: cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b + version: eb3733d160e74a9c7e442f435eb3bea458e1d19f testImports: - name: github.com/davecgh/go-spew - version: 6d212800a42e8ab5c146b8ace3490ee17e5225f9 + version: 04cdfd42973bb9c8589fd6a731800cf222fde1a9 subpackages: - spew - name: github.com/pmezard/go-difflib @@ -202,9 +204,7 @@ testImports: subpackages: - difflib - name: github.com/stretchr/testify - version: 69483b4bd14f5845b5a1e55bca19e954e827f1d0 + version: 890a5c3458b43e6104ff5da8dfa139d013d77544 subpackages: - assert - require -- name: github.com/tendermint/iavl - version: 8fa7ec23377d7af91fa84b83a16abd24cc3acc71 diff --git a/glide.yaml b/glide.yaml index 7b924aacc..95afb8ba1 100644 --- a/glide.yaml +++ b/glide.yaml @@ -25,6 +25,10 @@ import: version: ~0.6.2 subpackages: - data +- package: github.com/tendermint/merkleeyes + version: master + subpackages: + - iavl - package: github.com/tendermint/tmlibs version: ~0.3.0 subpackages: From 3e92d295e4b42286dfc598c73d066d5fb787c6b5 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 22 Sep 2017 13:25:10 -0400 Subject: [PATCH 92/92] glide for tmlibs 0.3.1 --- glide.lock | 6 +++--- glide.yaml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/glide.lock b/glide.lock index 432e67e3c..6e295a12a 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: e9abc35ed3adbcf850d3fec184572ed23a46c9489474f2867f9e57673149e56b -updated: 2017-09-22T12:27:41.940717166-04:00 +hash: e3649cac7b1b9a23c024a9d1bbebd5a147861d55da2bca77c95129b6021850b4 +updated: 2017-09-22T13:24:29.443800586-04:00 imports: - name: github.com/btcsuite/btcd version: 4803a8291c92a1d2d41041b942a9a9e37deab065 @@ -122,7 +122,7 @@ imports: subpackages: - iavl - name: github.com/tendermint/tmlibs - version: 2130c329eb56aca8509ee1fec40d766f6541d8e7 + version: 9997e3a3b46db1d2f88aa9816ed0e7915dad6ac1 subpackages: - autofile - cli diff --git a/glide.yaml b/glide.yaml index 95afb8ba1..58f25711d 100644 --- a/glide.yaml +++ b/glide.yaml @@ -30,7 +30,7 @@ import: subpackages: - iavl - package: github.com/tendermint/tmlibs - version: ~0.3.0 + version: ~0.3.1 subpackages: - autofile - cli