From 416f03c05b3b89559fb0fe3472be8764f7837108 Mon Sep 17 00:00:00 2001 From: Zarko Milosevic Date: Mon, 5 Mar 2018 14:42:23 +0100 Subject: [PATCH 01/54] Add light client spec --- docs/specification/new-spec/light-client.md | 114 ++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 docs/specification/new-spec/light-client.md diff --git a/docs/specification/new-spec/light-client.md b/docs/specification/new-spec/light-client.md new file mode 100644 index 000000000..0ed9d36d4 --- /dev/null +++ b/docs/specification/new-spec/light-client.md @@ -0,0 +1,114 @@ +# Light client + +A light client is a process that connects to the Tendermint Full Node(s) and then tries to verify the Merkle proofs +about the blockchain application. In this document we describe mechanisms that ensures that the Tendermint light client +has the same level of security as Full Node processes (without being itself a Full Node). + +To be able to validate a Merkle proof, a light client needs to validate the blockchain header that contains the root app hash. +Validating a blockchain header in Tendermint consists in verifying that the header is committed (signed) by >2/3 of the +voting power of the corresponding validator set. As the validator set is a dynamic set (it is changing), one of the +core functionality of the light client is updating the current validator set, that is then used to verify the +blockchain header, and further the corresponding Merkle proofs. + +For the purpose of this light client specification, we assume that the Tendermint Full Node exposes the following functions over +Tendermint RPC: + +```golang +Header(height int64) (SignedHeader, error) // returns signed header for the given height +Validators(height int64) (ResultValidators, error) // returns validator set for the given height +LastHeader(valSetNumber int64) (SignedHeader, error) // returns last header signed by the validator set with the given validator set number + +type SignedHeader struct { + Header Header + Commit Commit + ValSetNumber int64 +} + +type ResultValidators struct { + BlockHeight int64 + Validators []Validator + // time the current validator set is initialised, i.e, time of the last validator change before header BlockHeight + ValSetTime int64 +} +``` + +We assume that Tendermint keeps track of the validator set changes and that each time a validator set is changed it is +being assigned the next sequence number. We can call this number the validator set sequence number. Tendermint also remembers +the Time from the header when the next validator set is initialised (starts to be in power), and we refer to this time +as validator set init time. +Furthermore, we assume that each validator set change is signed (committed) by the current validator set. More precisely, +given a block `H` that contains transactions that are modifying the current validator set, the Merkle root hash of the next +validator set (modified based on transactions from block H) will be in block `H+1` (and signed by the current validator +set), and then starting from the block `H+2`, it will be signed by the next validator set. + +Note that the real Tendermint RPC API is slightly different (for example, response messages contain more data and function +names are slightly different); we shortened (and modified) it for the purpose of this document to make the spec more +clear and simple. Furthermore, note that in case of the third function, the returned header has `ValSetNumber` equals to +`valSetNumber+1`. + + +Locally, light client manages the following state: + +```golang +valSet []Validator // current validator set (last known and verified validator set) +valSetNumber int64 // sequence number of the current validator set +valSetHash []byte // hash of the current validator set +valSetTime int64 // time when the current validator set is initialised +``` + +The light client is initialised with the trusted validator set, for example based on the known validator set hash, +validator set sequence number and the validator set init time. +The core of the light client logic is captured by the VerifyAndUpdate function that is used to 1) verify if the given header is valid, +and 2) update the validator set (when the given header is valid and it is more recent than the seen headers). + +```golang +VerifyAndUpdate(signedHeader SignedHeader): + assertThat signedHeader.valSetNumber >= valSetNumber + if isValid(signedHeader) and signedHeader.Header.Time <= valSetTime + UNBONDING_PERIOD then + setValidatorSet(signedHeader) + return true + else + updateValidatorSet(signedHeader.ValSetNumber) + return VerifyAndUpdate(signedHeader) + +isValid(signedHeader SignedHeader): + valSetOfTheHeader = Validators(signedHeader.Header.Height) + assertThat Hash(valSetOfTheHeader) == signedHeader.Header.ValSetHash + assertThat signedHeader is passing basic validation + if votingPower(signedHeader.Commit) > 2/3 * votingPower(valSetOfTheHeader) then return true + else + return false + +setValidatorSet(signedHeader SignedHeader): + nextValSet = Validators(signedHeader.Header.Height) + assertThat Hash(nextValSet) == signedHeader.Header.ValidatorsHash + valSet = nextValSet.Validators + valSetHash = signedHeader.Header.ValidatorsHash + valSetNumber = signedHeader.ValSetNumber + valSetTime = nextValSet.ValSetTime + +votingPower(commit Commit): + votingPower = 0 + for each precommit in commit.Precommits do: + if precommit.ValidatorAddress is in valSet and signature of the precommit verifies then + votingPower += valSet[precommit.ValidatorAddress].VotingPower + return votingPower + +votingPower(validatorSet []Validator): + for each validator in validatorSet do: + votingPower += validator.VotingPower + return votingPower + +updateValidatorSet(valSetNumberOfTheHeader): + while valSetNumber != valSetNumberOfTheHeader do + signedHeader = LastHeader(valSetNumber) + if isValid(signedHeader) then + setValidatorSet(signedHeader) + else return error + return +``` + +Note that in the logic above we assume that the light client will always go upward with respect to header verifications, +i.e., that it will always be used to verify more recent headers. In case a light client needs to be used to verify older +headers (go backward) the same mechanisms and similar logic can be used. In case a call to the FullNode or subsequent +checks fail, a light client need to implement some recovery strategy, for example connecting to other FullNode. From 4c2f56626ac3161d871bd4b44ec80ea0ff3b1284 Mon Sep 17 00:00:00 2001 From: Emmanuel T Odeke Date: Sat, 10 Mar 2018 21:15:55 -0800 Subject: [PATCH 02/54] lite/proxy: Validation* tests and hardening for nil dereferences Updates https://github.com/tendermint/tendermint/issues/1017 Ensure that the Validate* functions in proxy are tests and cover the case of sneakish bugs that have been encountered a few times from nil dereferences. The lite package should theoretically never panic with a nil dereference. It is meant to contain the certifiers hence it should never panic with such. Requires the following bugs to be fixed first; * https://github.com/tendermint/tendermint/issues/1298 * https://github.com/tendermint/tendermint/issues/1299 --- lite/proxy/block.go | 9 ++ lite/proxy/validate_test.go | 256 ++++++++++++++++++++++++++++++++++++ 2 files changed, 265 insertions(+) create mode 100644 lite/proxy/validate_test.go diff --git a/lite/proxy/block.go b/lite/proxy/block.go index 60cd00f0d..4cff9ee68 100644 --- a/lite/proxy/block.go +++ b/lite/proxy/block.go @@ -11,11 +11,17 @@ import ( ) func ValidateBlockMeta(meta *types.BlockMeta, check lite.Commit) error { + if meta == nil { + return errors.New("expecting a non-nil BlockMeta") + } // TODO: check the BlockID?? return ValidateHeader(meta.Header, check) } func ValidateBlock(meta *types.Block, check lite.Commit) error { + if meta == nil { + return errors.New("expecting a non-nil Block") + } err := ValidateHeader(meta.Header, check) if err != nil { return err @@ -27,6 +33,9 @@ func ValidateBlock(meta *types.Block, check lite.Commit) error { } func ValidateHeader(head *types.Header, check lite.Commit) error { + if head == nil { + return errors.New("expecting a non-nil Header") + } // make sure they are for the same height (obvious fail) if head.Height != check.Height() { return certerr.ErrHeightMismatch(head.Height, check.Height()) diff --git a/lite/proxy/validate_test.go b/lite/proxy/validate_test.go new file mode 100644 index 000000000..971a5e5b3 --- /dev/null +++ b/lite/proxy/validate_test.go @@ -0,0 +1,256 @@ +package proxy_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/tendermint/tendermint/lite" + "github.com/tendermint/tendermint/lite/proxy" + "github.com/tendermint/tendermint/types" +) + +var ( + deabBeefTxs = types.Txs{[]byte("DE"), []byte("AD"), []byte("BE"), []byte("EF")} + deadBeefRipEmd160Hash = deabBeefTxs.Hash() +) + +func TestValidateBlock(t *testing.T) { + tests := []struct { + block *types.Block + commit lite.Commit + wantErr string + }{ + { + block: nil, wantErr: "non-nil Block", + }, + { + block: &types.Block{}, wantErr: "nil Header", + }, + { + block: &types.Block{Header: new(types.Header)}, + }, + + // Start Header.Height mismatch test + { + block: &types.Block{Header: &types.Header{Height: 10}}, + commit: lite.Commit{Header: &types.Header{Height: 11}}, + wantErr: "don't match - 10 vs 11", + }, + + { + block: &types.Block{Header: &types.Header{Height: 11}}, + commit: lite.Commit{Header: &types.Header{Height: 11}}, + }, + // End Header.Height mismatch test + + // Start Header.Hash mismatch test + { + block: &types.Block{ + Header: &types.Header{ + Height: 11, + Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), + ValidatorsHash: []byte("Tendermint"), + }, + }, + commit: lite.Commit{Header: &types.Header{Height: 11}}, + wantErr: "Headers don't match", + }, + + { + block: &types.Block{ + Header: &types.Header{ + Height: 11, + Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), + ValidatorsHash: []byte("Tendermint"), + }, + }, + commit: lite.Commit{ + Header: &types.Header{ + Height: 11, + Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), + ValidatorsHash: []byte("Tendermint"), + }, + }, + }, + // End Header.Hash mismatch test + + // Start Header.Data hash mismatch test + { + block: &types.Block{ + Header: &types.Header{Height: 11}, + Data: &types.Data{Txs: []types.Tx{[]byte("0xDE"), []byte("AD")}}, + }, + commit: lite.Commit{ + Header: &types.Header{Height: 11}, + Commit: &types.Commit{BlockID: types.BlockID{Hash: []byte("0xDEADBEEF")}}, + }, + wantErr: "Data hash doesn't match header", + }, + { + block: &types.Block{ + Header: &types.Header{Height: 11, DataHash: deadBeefRipEmd160Hash}, + Data: &types.Data{Txs: deabBeefTxs}, + }, + commit: lite.Commit{ + Header: &types.Header{Height: 11}, + Commit: &types.Commit{BlockID: types.BlockID{Hash: []byte("DEADBEEF")}}, + }, + }, + // End Header.Data hash mismatch test + } + + assert := assert.New(t) + + for i, tt := range tests { + err := proxy.ValidateBlock(tt.block, tt.commit) + if tt.wantErr != "" { + if err == nil { + assert.FailNowf("Unexpectedly passed", "#%d", i) + } else { + assert.Contains(err.Error(), tt.wantErr, "#%d should contain the substring\n\n", i) + } + continue + } + + assert.Nil(err, "#%d: expecting a nil error", i) + } +} + +func TestValidateBlockMeta(t *testing.T) { + tests := []struct { + meta *types.BlockMeta + commit lite.Commit + wantErr string + }{ + { + meta: nil, wantErr: "non-nil BlockMeta", + }, + { + meta: &types.BlockMeta{}, wantErr: "non-nil Header", + }, + { + meta: &types.BlockMeta{Header: new(types.Header)}, + }, + + // Start Header.Height mismatch test + { + meta: &types.BlockMeta{Header: &types.Header{Height: 10}}, + commit: lite.Commit{Header: &types.Header{Height: 11}}, + wantErr: "don't match - 10 vs 11", + }, + + { + meta: &types.BlockMeta{Header: &types.Header{Height: 11}}, + commit: lite.Commit{Header: &types.Header{Height: 11}}, + }, + // End Header.Height mismatch test + + // Start Headers don't match test + { + meta: &types.BlockMeta{ + Header: &types.Header{ + Height: 11, + Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), + ValidatorsHash: []byte("Tendermint"), + }, + }, + commit: lite.Commit{Header: &types.Header{Height: 11}}, + wantErr: "Headers don't match", + }, + + { + meta: &types.BlockMeta{ + Header: &types.Header{ + Height: 11, + Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), + ValidatorsHash: []byte("Tendermint"), + }, + }, + commit: lite.Commit{ + Header: &types.Header{ + Height: 11, + Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), + ValidatorsHash: []byte("Tendermint"), + }, + }, + }, + + { + meta: &types.BlockMeta{ + Header: &types.Header{ + Height: 11, + // TODO: (@odeke-em) inquire why ValidatorsHash has to be non-blank + // for the Header to be hashed. Perhaps this is a security hole because + // an aggressor could perhaps pass in headers that don't have + // ValidatorsHash set and we won't be able to validate blocks. + ValidatorsHash: []byte("lite-test"), + // TODO: (@odeke-em) file an issue with Tendermint to get them to update + // to the latest go-wire, then no more need for this value fill to avoid + // the time zero value of less than 1970. + Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), + }, + }, + commit: lite.Commit{ + Header: &types.Header{Height: 11, DataHash: deadBeefRipEmd160Hash}, + }, + wantErr: "Headers don't match", + }, + + { + meta: &types.BlockMeta{ + Header: &types.Header{ + Height: 11, DataHash: deadBeefRipEmd160Hash, + ValidatorsHash: []byte("Tendermint"), + Time: time.Date(2017, 1, 2, 1, 1, 1, 1, time.UTC), + }, + }, + commit: lite.Commit{ + Header: &types.Header{ + Height: 11, DataHash: deadBeefRipEmd160Hash, + ValidatorsHash: []byte("Tendermint"), + Time: time.Date(2017, 1, 2, 2, 1, 1, 1, time.UTC), + }, + Commit: &types.Commit{BlockID: types.BlockID{Hash: []byte("DEADBEEF")}}, + }, + wantErr: "Headers don't match", + }, + + { + meta: &types.BlockMeta{ + Header: &types.Header{ + Height: 11, DataHash: deadBeefRipEmd160Hash, + ValidatorsHash: []byte("Tendermint"), + Time: time.Date(2017, 1, 2, 1, 1, 1, 1, time.UTC), + }, + }, + commit: lite.Commit{ + Header: &types.Header{ + Height: 11, DataHash: deadBeefRipEmd160Hash, + ValidatorsHash: []byte("Tendermint-x"), + Time: time.Date(2017, 1, 2, 1, 1, 1, 1, time.UTC), + }, + Commit: &types.Commit{BlockID: types.BlockID{Hash: []byte("DEADBEEF")}}, + }, + wantErr: "Headers don't match", + }, + // End Headers don't match test + } + + assert := assert.New(t) + + for i, tt := range tests { + err := proxy.ValidateBlockMeta(tt.meta, tt.commit) + if tt.wantErr != "" { + if err == nil { + assert.FailNowf("Unexpectedly passed", "#%d: wanted error %q", i, tt.wantErr) + } else { + assert.Contains(err.Error(), tt.wantErr, "#%d should contain the substring\n\n", i) + } + continue + } + + assert.Nil(err, "#%d: expecting a nil error", i) + } +} From 58f36bb3215c54287d6693119d0071b97052b673 Mon Sep 17 00:00:00 2001 From: Emmanuel T Odeke Date: Sat, 10 Mar 2018 21:50:15 -0800 Subject: [PATCH 03/54] Review feedback from @melekes * Fix typo on naming s/deabBeef/deadBeef/g * Use `assert.*(t,` instead of `assert.New(t);...;assert.*(` --- lite/proxy/validate_test.go | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/lite/proxy/validate_test.go b/lite/proxy/validate_test.go index 971a5e5b3..653bfb753 100644 --- a/lite/proxy/validate_test.go +++ b/lite/proxy/validate_test.go @@ -12,8 +12,8 @@ import ( ) var ( - deabBeefTxs = types.Txs{[]byte("DE"), []byte("AD"), []byte("BE"), []byte("EF")} - deadBeefRipEmd160Hash = deabBeefTxs.Hash() + deadBeefTxs = types.Txs{[]byte("DE"), []byte("AD"), []byte("BE"), []byte("EF")} + deadBeefRipEmd160Hash = deadBeefTxs.Hash() ) func TestValidateBlock(t *testing.T) { @@ -91,7 +91,7 @@ func TestValidateBlock(t *testing.T) { { block: &types.Block{ Header: &types.Header{Height: 11, DataHash: deadBeefRipEmd160Hash}, - Data: &types.Data{Txs: deabBeefTxs}, + Data: &types.Data{Txs: deadBeefTxs}, }, commit: lite.Commit{ Header: &types.Header{Height: 11}, @@ -101,20 +101,18 @@ func TestValidateBlock(t *testing.T) { // End Header.Data hash mismatch test } - assert := assert.New(t) - for i, tt := range tests { err := proxy.ValidateBlock(tt.block, tt.commit) if tt.wantErr != "" { if err == nil { - assert.FailNowf("Unexpectedly passed", "#%d", i) + assert.FailNowf(t, "Unexpectedly passed", "#%d", i) } else { - assert.Contains(err.Error(), tt.wantErr, "#%d should contain the substring\n\n", i) + assert.Contains(t, err.Error(), tt.wantErr, "#%d should contain the substring\n\n", i) } continue } - assert.Nil(err, "#%d: expecting a nil error", i) + assert.Nil(t, err, "#%d: expecting a nil error", i) } } @@ -238,19 +236,17 @@ func TestValidateBlockMeta(t *testing.T) { // End Headers don't match test } - assert := assert.New(t) - for i, tt := range tests { err := proxy.ValidateBlockMeta(tt.meta, tt.commit) if tt.wantErr != "" { if err == nil { - assert.FailNowf("Unexpectedly passed", "#%d: wanted error %q", i, tt.wantErr) + assert.FailNowf(t, "Unexpectedly passed", "#%d: wanted error %q", i, tt.wantErr) } else { - assert.Contains(err.Error(), tt.wantErr, "#%d should contain the substring\n\n", i) + assert.Contains(t, err.Error(), tt.wantErr, "#%d should contain the substring\n\n", i) } continue } - assert.Nil(err, "#%d: expecting a nil error", i) + assert.Nil(t, err, "#%d: expecting a nil error", i) } } From 8813684040212782a76e4ccf9627d7be8901f771 Mon Sep 17 00:00:00 2001 From: Emmanuel T Odeke Date: Sun, 25 Mar 2018 00:27:38 -0600 Subject: [PATCH 04/54] lite/proxy: consolidate some common test headers into a variable Addressing some feedback from @ebuchman in regards to consolidating some common test headers into a variable. I've added that for simple cases, trying to meet in the middle instead of creating helpers that obscure readibility and easy comparison of test cases. --- lite/proxy/validate_test.go | 59 ++++++++++--------------------------- 1 file changed, 15 insertions(+), 44 deletions(-) diff --git a/lite/proxy/validate_test.go b/lite/proxy/validate_test.go index 653bfb753..bd57994d7 100644 --- a/lite/proxy/validate_test.go +++ b/lite/proxy/validate_test.go @@ -12,10 +12,17 @@ import ( ) var ( - deadBeefTxs = types.Txs{[]byte("DE"), []byte("AD"), []byte("BE"), []byte("EF")} + deadBeefTxs = types.Txs{[]byte("DE"), []byte("AD"), []byte("BE"), []byte("EF")} + deadBeefRipEmd160Hash = deadBeefTxs.Hash() ) +var hdrHeight11Tendermint = &types.Header{ + Height: 11, + Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), + ValidatorsHash: []byte("Tendermint"), +} + func TestValidateBlock(t *testing.T) { tests := []struct { block *types.Block @@ -47,32 +54,14 @@ func TestValidateBlock(t *testing.T) { // Start Header.Hash mismatch test { - block: &types.Block{ - Header: &types.Header{ - Height: 11, - Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), - ValidatorsHash: []byte("Tendermint"), - }, - }, + block: &types.Block{Header: hdrHeight11Tendermint}, commit: lite.Commit{Header: &types.Header{Height: 11}}, wantErr: "Headers don't match", }, { - block: &types.Block{ - Header: &types.Header{ - Height: 11, - Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), - ValidatorsHash: []byte("Tendermint"), - }, - }, - commit: lite.Commit{ - Header: &types.Header{ - Height: 11, - Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), - ValidatorsHash: []byte("Tendermint"), - }, - }, + block: &types.Block{Header: hdrHeight11Tendermint}, + commit: lite.Commit{Header: hdrHeight11Tendermint}, }, // End Header.Hash mismatch test @@ -147,32 +136,14 @@ func TestValidateBlockMeta(t *testing.T) { // Start Headers don't match test { - meta: &types.BlockMeta{ - Header: &types.Header{ - Height: 11, - Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), - ValidatorsHash: []byte("Tendermint"), - }, - }, + meta: &types.BlockMeta{Header: hdrHeight11Tendermint}, commit: lite.Commit{Header: &types.Header{Height: 11}}, wantErr: "Headers don't match", }, { - meta: &types.BlockMeta{ - Header: &types.Header{ - Height: 11, - Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), - ValidatorsHash: []byte("Tendermint"), - }, - }, - commit: lite.Commit{ - Header: &types.Header{ - Height: 11, - Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), - ValidatorsHash: []byte("Tendermint"), - }, - }, + meta: &types.BlockMeta{Header: hdrHeight11Tendermint}, + commit: lite.Commit{Header: hdrHeight11Tendermint}, }, { @@ -208,7 +179,7 @@ func TestValidateBlockMeta(t *testing.T) { Header: &types.Header{ Height: 11, DataHash: deadBeefRipEmd160Hash, ValidatorsHash: []byte("Tendermint"), - Time: time.Date(2017, 1, 2, 2, 1, 1, 1, time.UTC), + Time: time.Date(2018, 1, 2, 1, 1, 1, 1, time.UTC), }, Commit: &types.Commit{BlockID: types.BlockID{Hash: []byte("DEADBEEF")}}, }, From 4085c7249659888615d86ab4df3c4f8816f94473 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 27 Mar 2018 12:32:42 +0200 Subject: [PATCH 05/54] sort /tx_search results by height by default Refs #1366 --- state/txindex/kv/kv.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index 24e982725..510074b50 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/hex" "fmt" + "sort" "strconv" "strings" "time" @@ -202,6 +203,11 @@ func (txi *TxIndex) Search(q *query.Query) ([]*types.TxResult, error) { i++ } + // sort by height by default + sort.Slice(results, func(i, j int) bool { + return results[i].Height < results[j].Height + }) + return results, nil } From 2b63f57b4c4506ff4bc799dcc4d346bb2b18ef05 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 27 Mar 2018 12:33:15 +0200 Subject: [PATCH 06/54] fix tx_indexer's matchRange before we're using IteratePrefix, which is wrong because we want full range, not just "account.number=1". --- state/txindex/kv/kv.go | 132 +++++++++++++++++++++++------------- state/txindex/kv/kv_test.go | 37 ++++++++-- 2 files changed, 116 insertions(+), 53 deletions(-) diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index 510074b50..1f544e58b 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -171,10 +171,10 @@ func (txi *TxIndex) Search(q *query.Query) ([]*types.TxResult, error) { for _, r := range ranges { if !hashesInitialized { - hashes = txi.matchRange(r, startKeyForRange(r, height)) + hashes = txi.matchRange(r, []byte(r.key)) hashesInitialized = true } else { - hashes = intersect(hashes, txi.matchRange(r, startKeyForRange(r, height))) + hashes = intersect(hashes, txi.matchRange(r, []byte(r.key))) } } } @@ -242,6 +242,52 @@ type queryRange struct { includeUpperBound bool } +func (r queryRange) lowerBoundValue() interface{} { + if r.lowerBound == nil { + return nil + } + + if r.includeLowerBound { + return r.lowerBound + } else { + switch t := r.lowerBound.(type) { + case int64: + return t + 1 + case time.Time: + return t.Unix() + 1 + default: + panic("not implemented") + } + } +} + +func (r queryRange) AnyBound() interface{} { + if r.lowerBound != nil { + return r.lowerBound + } else { + return r.upperBound + } +} + +func (r queryRange) upperBoundValue() interface{} { + if r.upperBound == nil { + return nil + } + + if r.includeUpperBound { + return r.upperBound + } else { + switch t := r.upperBound.(type) { + case int64: + return t - 1 + case time.Time: + return t.Unix() - 1 + default: + panic("not implemented") + } + } +} + func lookForRanges(conditions []query.Condition) (ranges queryRanges, indexes []int) { ranges = make(queryRanges) for i, c := range conditions { @@ -305,34 +351,49 @@ func (txi *TxIndex) match(c query.Condition, startKey []byte) (hashes [][]byte) return } -func (txi *TxIndex) matchRange(r queryRange, startKey []byte) (hashes [][]byte) { - it := dbm.IteratePrefix(txi.store, startKey) +func (txi *TxIndex) matchRange(r queryRange, prefix []byte) (hashes [][]byte) { + // create a map to prevent duplicates + hashesMap := make(map[string][]byte) + + lowerBound := r.lowerBoundValue() + upperBound := r.upperBoundValue() + + it := dbm.IteratePrefix(txi.store, prefix) defer it.Close() LOOP: for ; it.Valid(); it.Next() { if !isTagKey(it.Key()) { continue } - if r.upperBound != nil { - // no other way to stop iterator other than checking for upperBound - switch (r.upperBound).(type) { - case int64: - v, err := strconv.ParseInt(extractValueFromKey(it.Key()), 10, 64) - if err == nil && v == r.upperBound { - if r.includeUpperBound { - hashes = append(hashes, it.Value()) - } - break LOOP - } - // XXX: passing time in a ABCI Tags is not yet implemented - // case time.Time: - // v := strconv.ParseInt(extractValueFromKey(it.Key()), 10, 64) - // if v == r.upperBound { - // break - // } + switch r.AnyBound().(type) { + case int64: + v, err := strconv.ParseInt(extractValueFromKey(it.Key()), 10, 64) + if err != nil { + continue LOOP + } + include := true + if lowerBound != nil && v < lowerBound.(int64) { + include = false } + if upperBound != nil && v > upperBound.(int64) { + include = false + } + if include { + hashesMap[fmt.Sprintf("%X", it.Value())] = it.Value() + } + // XXX: passing time in a ABCI Tags is not yet implemented + // case time.Time: + // v := strconv.ParseInt(extractValueFromKey(it.Key()), 10, 64) + // if v == r.upperBound { + // break + // } } - hashes = append(hashes, it.Value()) + } + hashes = make([][]byte, len(hashesMap)) + i := 0 + for _, h := range hashesMap { + hashes[i] = h + i++ } return } @@ -350,33 +411,6 @@ func startKey(c query.Condition, height int64) []byte { return []byte(key) } -func startKeyForRange(r queryRange, height int64) []byte { - if r.lowerBound == nil { - return []byte(r.key) - } - - var lowerBound interface{} - if r.includeLowerBound { - lowerBound = r.lowerBound - } else { - switch t := r.lowerBound.(type) { - case int64: - lowerBound = t + 1 - case time.Time: - lowerBound = t.Unix() + 1 - default: - panic("not implemented") - } - } - var key string - if height > 0 { - key = fmt.Sprintf("%s/%v/%d", r.key, lowerBound, height) - } else { - key = fmt.Sprintf("%s/%v", r.key, lowerBound) - } - return []byte(key) -} - func isTagKey(key []byte) bool { return strings.Count(string(key), tagKeySeparator) == 3 } diff --git a/state/txindex/kv/kv_test.go b/state/txindex/kv/kv_test.go index 7ffc36e50..810da23bd 100644 --- a/state/txindex/kv/kv_test.go +++ b/state/txindex/kv/kv_test.go @@ -20,7 +20,7 @@ func TestTxIndex(t *testing.T) { indexer := NewTxIndex(db.NewMemDB()) tx := types.Tx("HELLO WORLD") - txResult := &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: []cmn.KVPair{}, Fee: cmn.KI64Pair{Key: []uint8{}, Value: 0}}} + txResult := &types.TxResult{Height: 1, Index: 0, Tx: tx, Result: abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: []cmn.KVPair{}, Fee: cmn.KI64Pair{Key: []uint8{}, Value: 0}}} hash := tx.Hash() batch := txindex.NewBatch(1) @@ -35,7 +35,7 @@ func TestTxIndex(t *testing.T) { assert.Equal(t, txResult, loadedTxResult) tx2 := types.Tx("BYE BYE WORLD") - txResult2 := &types.TxResult{1, 0, tx2, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: []cmn.KVPair{}, Fee: cmn.KI64Pair{Key: []uint8{}, Value: 0}}} + txResult2 := &types.TxResult{Height: 1, Index: 0, Tx: tx2, Result: abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: []cmn.KVPair{}, Fee: cmn.KI64Pair{Key: []uint8{}, Value: 0}}} hash2 := tx2.Hash() err = indexer.Index(txResult2) @@ -122,6 +122,35 @@ func TestTxSearchOneTxWithMultipleSameTagsButDifferentValues(t *testing.T) { assert.Equal(t, []*types.TxResult{txResult}, results) } +func TestTxSearchMultipleTxs(t *testing.T) { + allowedTags := []string{"account.number"} + indexer := NewTxIndex(db.NewMemDB(), IndexTags(allowedTags)) + + // indexed first, but bigger height (to test the order of transactions) + txResult := txResultWithTags([]cmn.KVPair{ + {Key: []byte("account.number"), Value: []byte("1")}, + }) + txResult.Tx = types.Tx("Bob's account") + txResult.Height = 2 + err := indexer.Index(txResult) + require.NoError(t, err) + + // indexed second, but smaller height (to test the order of transactions) + txResult2 := txResultWithTags([]cmn.KVPair{ + {Key: []byte("account.number"), Value: []byte("2")}, + }) + txResult2.Tx = types.Tx("Alice's account") + txResult2.Height = 1 + err = indexer.Index(txResult2) + require.NoError(t, err) + + results, err := indexer.Search(query.MustParse("account.number >= 1")) + assert.NoError(t, err) + + require.Len(t, results, 2) + assert.Equal(t, []*types.TxResult{txResult2, txResult}, results) +} + func TestIndexAllTags(t *testing.T) { indexer := NewTxIndex(db.NewMemDB(), IndexAllTags()) @@ -146,12 +175,12 @@ func TestIndexAllTags(t *testing.T) { func txResultWithTags(tags []cmn.KVPair) *types.TxResult { tx := types.Tx("HELLO WORLD") - return &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: tags, Fee: cmn.KI64Pair{Key: []uint8{}, Value: 0}}} + return &types.TxResult{Height: 1, Index: 0, Tx: tx, Result: abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: tags, Fee: cmn.KI64Pair{Key: []uint8{}, Value: 0}}} } func benchmarkTxIndex(txsCount int, b *testing.B) { tx := types.Tx("HELLO WORLD") - txResult := &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: []cmn.KVPair{}, Fee: cmn.KI64Pair{Key: []uint8{}, Value: 0}}} + txResult := &types.TxResult{Height: 1, Index: 0, Tx: tx, Result: abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: []cmn.KVPair{}, Fee: cmn.KI64Pair{Key: []uint8{}, Value: 0}}} dir, err := ioutil.TempDir("", "tx_index_db") if err != nil { From 2d857c4b1bf1cd09bf6631aae6945984c392eac8 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 28 Mar 2018 15:44:58 +0200 Subject: [PATCH 07/54] add hash field to ResultTx (/tx and /tx_search endpoints) (#1374) Refs #1367 --- CHANGELOG.md | 5 +++++ rpc/client/rpc_test.go | 22 +++++++++++----------- rpc/core/tx.go | 10 ++++++++-- rpc/core/types/responses.go | 1 + 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aea9b5450..47b39b23a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,11 @@ BUG FIXES: - Graceful handling/recovery for apps that have non-determinism or fail to halt - Graceful handling/recovery for violations of safety, or liveness +## 0.17.2 (TBD) + +IMPROVEMENTS: +- [rpc] `/tx` and `/tx_search` responses now include the transaction hash + ## 0.17.1 (March 27th, 2018) BUG FIXES: diff --git a/rpc/client/rpc_test.go b/rpc/client/rpc_test.go index f23ba6160..c5c5822f9 100644 --- a/rpc/client/rpc_test.go +++ b/rpc/client/rpc_test.go @@ -253,13 +253,11 @@ func TestBroadcastTxCommit(t *testing.T) { } func TestTx(t *testing.T) { - assert, require := assert.New(t), require.New(t) - // first we broadcast a tx c := getHTTPClient() _, _, tx := MakeTxKV() bres, err := c.BroadcastTxCommit(tx) - require.Nil(err, "%+v", err) + require.Nil(t, err, "%+v", err) txHeight := bres.Height txHash := bres.Hash @@ -289,18 +287,19 @@ func TestTx(t *testing.T) { ptx, err := c.Tx(tc.hash, tc.prove) if !tc.valid { - require.NotNil(err) + require.NotNil(t, err) } else { - require.Nil(err, "%+v", err) - assert.EqualValues(txHeight, ptx.Height) - assert.EqualValues(tx, ptx.Tx) - assert.Zero(ptx.Index) - assert.True(ptx.TxResult.IsOK()) + require.Nil(t, err, "%+v", err) + assert.EqualValues(t, txHeight, ptx.Height) + assert.EqualValues(t, tx, ptx.Tx) + assert.Zero(t, ptx.Index) + assert.True(t, ptx.TxResult.IsOK()) + assert.EqualValues(t, txHash, ptx.Hash) // time to verify the proof proof := ptx.Proof - if tc.prove && assert.EqualValues(tx, proof.Data) { - assert.True(proof.Proof.Verify(proof.Index, proof.Total, txHash, proof.RootHash)) + if tc.prove && assert.EqualValues(t, tx, proof.Data) { + assert.True(t, proof.Proof.Verify(proof.Index, proof.Total, txHash, proof.RootHash)) } } } @@ -333,6 +332,7 @@ func TestTxSearch(t *testing.T) { assert.EqualValues(t, tx, ptx.Tx) assert.Zero(t, ptx.Index) assert.True(t, ptx.TxResult.IsOK()) + assert.EqualValues(t, txHash, ptx.Hash) // time to verify the proof proof := ptx.Proof diff --git a/rpc/core/tx.go b/rpc/core/tx.go index f592326b1..18120ded1 100644 --- a/rpc/core/tx.go +++ b/rpc/core/tx.go @@ -44,7 +44,8 @@ import ( // "code": 0 // }, // "index": 0, -// "height": 52 +// "height": 52, +// "hash": "2B8EC32BA2579B3B8606E42C06DE2F7AFA2556EF" // }, // "id": "", // "jsonrpc": "2.0" @@ -67,6 +68,7 @@ import ( // - `tx_result`: the `abci.Result` object // - `index`: `int` - index of the transaction // - `height`: `int` - height of the block where this transaction was in +// - `hash`: `[]byte` - hash of the transaction func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { // if index is disabled, return error @@ -93,6 +95,7 @@ func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { } return &ctypes.ResultTx{ + Hash: hash, Height: height, Index: uint32(index), TxResult: r.Result, @@ -137,7 +140,8 @@ func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { // "tx": "mvZHHa7HhZ4aRT0xMDA=", // "tx_result": {}, // "index": 31, -// "height": 12 +// "height": 12, +// "hash": "2B8EC32BA2579B3B8606E42C06DE2F7AFA2556EF" // } // ], // "id": "", @@ -161,6 +165,7 @@ func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { // - `tx_result`: the `abci.Result` object // - `index`: `int` - index of the transaction // - `height`: `int` - height of the block where this transaction was in +// - `hash`: `[]byte` - hash of the transaction func TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error) { // if index is disabled, return error if _, ok := txIndexer.(*null.TxIndex); ok { @@ -191,6 +196,7 @@ func TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error) { } apiResults[i] = &ctypes.ResultTx{ + Hash: r.Tx.Hash(), Height: height, Index: index, TxResult: r.Result, diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index 7567fe650..5b49e1af6 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -123,6 +123,7 @@ type ResultBroadcastTxCommit struct { } type ResultTx struct { + Hash cmn.HexBytes `json:"hash"` Height int64 `json:"height"` Index uint32 `json:"index"` TxResult abci.ResponseDeliverTx `json:"tx_result"` From 2fa7af46148253e6697a0708a1f76c43d268a357 Mon Sep 17 00:00:00 2001 From: Vladislav Dmitriyev Date: Wed, 28 Mar 2018 16:59:09 +0300 Subject: [PATCH 08/54] [lite] fixed listen address (#1384) --- cmd/tendermint/commands/lite.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/tendermint/commands/lite.go b/cmd/tendermint/commands/lite.go index 5ea20a785..6866e4d6c 100644 --- a/cmd/tendermint/commands/lite.go +++ b/cmd/tendermint/commands/lite.go @@ -34,14 +34,14 @@ var ( ) func init() { - LiteCmd.Flags().StringVar(&listenAddr, "laddr", ":8888", "Serve the proxy on the given port") + LiteCmd.Flags().StringVar(&listenAddr, "laddr", "tcp://localhost:8888", "Serve the proxy on the given address") LiteCmd.Flags().StringVar(&nodeAddr, "node", "tcp://localhost:46657", "Connect to a Tendermint node at this address") LiteCmd.Flags().StringVar(&chainID, "chain-id", "tendermint", "Specify the Tendermint chain ID") LiteCmd.Flags().StringVar(&home, "home-dir", ".tendermint-lite", "Specify the home directory") } func ensureAddrHasSchemeOrDefaultToTCP(addr string) (string, error) { - u, err := url.Parse(nodeAddr) + u, err := url.Parse(addr) if err != nil { return "", err } From 49986b05bc2344d38b15c1db82a2c5c31ab89ca3 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 27 Mar 2018 16:11:55 +0200 Subject: [PATCH 09/54] update tmlibs Refs #1376 --- Gopkg.lock | 6 +++--- Gopkg.toml | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 894d98ead..09dab7a5b 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -268,6 +268,7 @@ version = "v0.7.3" [[projects]] + branch = "develop" name = "github.com/tendermint/tmlibs" packages = [ "autofile", @@ -283,8 +284,7 @@ "pubsub/query", "test" ] - revision = "24da7009c3d8c019b40ba4287495749e3160caca" - version = "v0.7.1" + revision = "0f2811441f4cf44b414df16ceae3c0931c74662e" [[projects]] branch = "master" @@ -381,6 +381,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "4dca5dbd2d280d093d7c8fc423606ab86d6ad1b241b076a7716c2093b5a09231" + inputs-digest = "88f227f33e785669706f12e131610a86fd3c00f5dfd6f22c62d46e0688049726" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 098d5ea9b..8de0de9c2 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -82,9 +82,11 @@ source = "github.com/tendermint/go-amino" version = "~0.7.3" -[[constraint]] +[[override]] +# [[constraint]] name = "github.com/tendermint/tmlibs" - version = "~0.7.1" + # version = "~0.7.1" + branch = "develop" [[constraint]] name = "google.golang.org/grpc" From 22949e6dfd39180be97ca7f17d8df0c370ea884f Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 27 Mar 2018 17:10:21 +0200 Subject: [PATCH 10/54] new tmlibs Parallel implementation --- p2p/conn/connection.go | 4 +- p2p/conn/secret_connection.go | 87 ++++++++++++++++++------------ p2p/conn/secret_connection_test.go | 76 +++++++++++++++++--------- p2p/peer.go | 24 ++++----- 4 files changed, 117 insertions(+), 74 deletions(-) diff --git a/p2p/conn/connection.go b/p2p/conn/connection.go index 9acaf6175..e4ae4df80 100644 --- a/p2p/conn/connection.go +++ b/p2p/conn/connection.go @@ -7,7 +7,6 @@ import ( "io" "math" "net" - "runtime/debug" "sync/atomic" "time" @@ -230,8 +229,7 @@ func (c *MConnection) flush() { // Catch panics, usually caused by remote disconnects. func (c *MConnection) _recover() { if r := recover(); r != nil { - stack := debug.Stack() - err := cmn.StackError{r, stack} + err := cmn.ErrorWrap(r, "recovered from panic") c.stopForError(err) } } diff --git a/p2p/conn/secret_connection.go b/p2p/conn/secret_connection.go index aa6db05bb..3495853e0 100644 --- a/p2p/conn/secret_connection.go +++ b/p2p/conn/secret_connection.go @@ -20,8 +20,8 @@ import ( "golang.org/x/crypto/nacl/secretbox" "golang.org/x/crypto/ripemd160" - "github.com/tendermint/go-crypto" - "github.com/tendermint/go-wire" + crypto "github.com/tendermint/go-crypto" + wire "github.com/tendermint/go-wire" cmn "github.com/tendermint/tmlibs/common" ) @@ -200,26 +200,36 @@ func genEphKeys() (ephPub, ephPriv *[32]byte) { } func shareEphPubKey(conn io.ReadWriteCloser, locEphPub *[32]byte) (remEphPub *[32]byte, err error) { - var err1, err2 error - - cmn.Parallel( - func() { - _, err1 = conn.Write(locEphPub[:]) + // Send our pubkey and receive theirs in tandem. + var trs, _ = cmn.Parallel( + func(_ int) (val interface{}, err error, abort bool) { + var _, err1 = conn.Write(locEphPub[:]) + if err1 != nil { + return nil, err1, true // abort + } else { + return nil, nil, false + } }, - func() { - remEphPub = new([32]byte) - _, err2 = io.ReadFull(conn, remEphPub[:]) + func(_ int) (val interface{}, err error, abort bool) { + var _remEphPub [32]byte + var _, err2 = io.ReadFull(conn, _remEphPub[:]) + if err2 != nil { + return nil, err2, true // abort + } else { + return _remEphPub, nil, false + } }, ) - if err1 != nil { - return nil, err1 - } - if err2 != nil { - return nil, err2 + // If error: + if trs.FirstError() != nil { + err = trs.FirstError() + return } - return remEphPub, nil + // Otherwise: + var _remEphPub = trs.FirstValue().([32]byte) + return &_remEphPub, nil } func computeSharedSecret(remPubKey, locPrivKey *[32]byte) (shrSecret *[32]byte) { @@ -268,33 +278,42 @@ type authSigMessage struct { Sig crypto.Signature } -func shareAuthSignature(sc *SecretConnection, pubKey crypto.PubKey, signature crypto.Signature) (*authSigMessage, error) { - var recvMsg authSigMessage - var err1, err2 error - - cmn.Parallel( - func() { +func shareAuthSignature(sc *SecretConnection, pubKey crypto.PubKey, signature crypto.Signature) (recvMsg *authSigMessage, err error) { + // Send our info and receive theirs in tandem. + var trs, _ = cmn.Parallel( + func(_ int) (val interface{}, err error, abort bool) { msgBytes := wire.BinaryBytes(authSigMessage{pubKey.Wrap(), signature.Wrap()}) - _, err1 = sc.Write(msgBytes) + var _, err1 = sc.Write(msgBytes) + if err1 != nil { + return nil, err1, true // abort + } else { + return nil, nil, false + } }, - func() { + func(_ int) (val interface{}, err error, abort bool) { readBuffer := make([]byte, authSigMsgSize) - _, err2 = io.ReadFull(sc, readBuffer) + var _, err2 = io.ReadFull(sc, readBuffer) if err2 != nil { - return + return nil, err2, true // abort } n := int(0) // not used. - recvMsg = wire.ReadBinary(authSigMessage{}, bytes.NewBuffer(readBuffer), authSigMsgSize, &n, &err2).(authSigMessage) - }) + var _recvMsg = wire.ReadBinary(authSigMessage{}, bytes.NewBuffer(readBuffer), authSigMsgSize, &n, &err2).(authSigMessage) + if err2 != nil { + return nil, err2, true // abort + } else { + return _recvMsg, nil, false + } + }, + ) - if err1 != nil { - return nil, err1 - } - if err2 != nil { - return nil, err2 + // If error: + if trs.FirstError() != nil { + err = trs.FirstError() + return } - return &recvMsg, nil + var _recvMsg = trs.FirstValue().(authSigMessage) + return &_recvMsg, nil } //-------------------------------------------------------------------------------- diff --git a/p2p/conn/secret_connection_test.go b/p2p/conn/secret_connection_test.go index c81b3b285..54c453a7c 100644 --- a/p2p/conn/secret_connection_test.go +++ b/p2p/conn/secret_connection_test.go @@ -1,9 +1,12 @@ package conn import ( + "fmt" "io" "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" crypto "github.com/tendermint/go-crypto" cmn "github.com/tendermint/tmlibs/common" ) @@ -36,33 +39,41 @@ func makeSecretConnPair(tb testing.TB) (fooSecConn, barSecConn *SecretConnection barPrvKey := crypto.GenPrivKeyEd25519().Wrap() barPubKey := barPrvKey.PubKey() - cmn.Parallel( - func() { - var err error + var trs, ok = cmn.Parallel( + func(_ int) (val interface{}, err error, abort bool) { fooSecConn, err = MakeSecretConnection(fooConn, fooPrvKey) if err != nil { tb.Errorf("Failed to establish SecretConnection for foo: %v", err) - return + return nil, err, true } remotePubBytes := fooSecConn.RemotePubKey() if !remotePubBytes.Equals(barPubKey) { - tb.Errorf("Unexpected fooSecConn.RemotePubKey. Expected %v, got %v", + err = fmt.Errorf("Unexpected fooSecConn.RemotePubKey. Expected %v, got %v", barPubKey, fooSecConn.RemotePubKey()) + tb.Error(err) + return nil, err, false } + return nil, nil, false }, - func() { - var err error + func(_ int) (val interface{}, err error, abort bool) { barSecConn, err = MakeSecretConnection(barConn, barPrvKey) if barSecConn == nil { tb.Errorf("Failed to establish SecretConnection for bar: %v", err) - return + return nil, err, true } remotePubBytes := barSecConn.RemotePubKey() if !remotePubBytes.Equals(fooPubKey) { - tb.Errorf("Unexpected barSecConn.RemotePubKey. Expected %v, got %v", + err = fmt.Errorf("Unexpected barSecConn.RemotePubKey. Expected %v, got %v", fooPubKey, barSecConn.RemotePubKey()) + tb.Error(err) + return nil, nil, false } - }) + return nil, nil, false + }, + ) + + require.Nil(tb, trs.FirstError()) + require.True(tb, ok, "Unexpected task abortion") return } @@ -89,59 +100,76 @@ func TestSecretConnectionReadWrite(t *testing.T) { } // A helper that will run with (fooConn, fooWrites, fooReads) and vice versa - genNodeRunner := func(nodeConn kvstoreConn, nodeWrites []string, nodeReads *[]string) func() { - return func() { + genNodeRunner := func(nodeConn kvstoreConn, nodeWrites []string, nodeReads *[]string) cmn.Task { + return func(_ int) (interface{}, error, bool) { // Node handskae nodePrvKey := crypto.GenPrivKeyEd25519().Wrap() nodeSecretConn, err := MakeSecretConnection(nodeConn, nodePrvKey) if err != nil { t.Errorf("Failed to establish SecretConnection for node: %v", err) - return + return nil, err, true } - // In parallel, handle reads and writes - cmn.Parallel( - func() { + // In parallel, handle some reads and writes. + var trs, ok = cmn.Parallel( + func(_ int) (interface{}, error, bool) { // Node writes for _, nodeWrite := range nodeWrites { n, err := nodeSecretConn.Write([]byte(nodeWrite)) if err != nil { t.Errorf("Failed to write to nodeSecretConn: %v", err) - return + return nil, err, true } if n != len(nodeWrite) { - t.Errorf("Failed to write all bytes. Expected %v, wrote %v", len(nodeWrite), n) - return + err = fmt.Errorf("Failed to write all bytes. Expected %v, wrote %v", len(nodeWrite), n) + t.Error(err) + return nil, err, true } } if err := nodeConn.PipeWriter.Close(); err != nil { t.Error(err) + return nil, err, true } + return nil, nil, false }, - func() { + func(_ int) (interface{}, error, bool) { // Node reads readBuffer := make([]byte, dataMaxSize) for { n, err := nodeSecretConn.Read(readBuffer) if err == io.EOF { - return + return nil, nil, false } else if err != nil { t.Errorf("Failed to read from nodeSecretConn: %v", err) - return + return nil, err, true } *nodeReads = append(*nodeReads, string(readBuffer[:n])) } if err := nodeConn.PipeReader.Close(); err != nil { t.Error(err) + return nil, err, true } - }) + return nil, nil, false + }, + ) + assert.True(t, ok, "Unexpected task abortion") + + // If error: + if trs.FirstError() != nil { + return nil, trs.FirstError(), true + } + + // Otherwise: + return nil, nil, false } } // Run foo & bar in parallel - cmn.Parallel( + var trs, ok = cmn.Parallel( genNodeRunner(fooConn, fooWrites, &fooReads), genNodeRunner(barConn, barWrites, &barReads), ) + require.Nil(t, trs.FirstError()) + require.True(t, ok, "unexpected task abortion") // A helper to ensure that the writes and reads match. // Additionally, small writes (<= dataMaxSize) must be atomically read. diff --git a/p2p/peer.go b/p2p/peer.go index 4af6eeaae..0fa7ca034 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -293,22 +293,20 @@ func (pc *peerConn) HandshakeTimeout(ourNodeInfo NodeInfo, timeout time.Duration return peerNodeInfo, errors.Wrap(err, "Error setting deadline") } - var err1 error - var err2 error - cmn.Parallel( - func() { + var trs, _ = cmn.Parallel( + func(_ int) (val interface{}, err error, abort bool) { var n int - wire.WriteBinary(&ourNodeInfo, pc.conn, &n, &err1) + wire.WriteBinary(&ourNodeInfo, pc.conn, &n, &err) + return }, - func() { + func(_ int) (val interface{}, err error, abort bool) { var n int - wire.ReadBinary(&peerNodeInfo, pc.conn, MaxNodeInfoSize(), &n, &err2) - }) - if err1 != nil { - return peerNodeInfo, errors.Wrap(err1, "Error during handshake/write") - } - if err2 != nil { - return peerNodeInfo, errors.Wrap(err2, "Error during handshake/read") + wire.ReadBinary(&peerNodeInfo, pc.conn, MaxNodeInfoSize(), &n, &err) + return + }, + ) + if err := trs.FirstError(); err != nil { + return peerNodeInfo, errors.Wrap(err, "Error during handshake") } // Remove deadline From ead9daf1ba95a1d275845a5ab205eea99252e983 Mon Sep 17 00:00:00 2001 From: Tomoya Ishizaki Date: Thu, 29 Mar 2018 02:40:47 +0900 Subject: [PATCH 11/54] Fix code style (#1362) * cfg: Uniform style for method args and var names --- config/config.go | 150 +++++++++++++++++++++++------------------------ 1 file changed, 75 insertions(+), 75 deletions(-) diff --git a/config/config.go b/config/config.go index 6b3f87b57..0844fbd51 100644 --- a/config/config.go +++ b/config/config.go @@ -137,10 +137,6 @@ type BaseConfig struct { DBPath string `mapstructure:"db_dir"` } -func (c BaseConfig) ChainID() string { - return c.chainID -} - // DefaultBaseConfig returns a default base configuration for a Tendermint node func DefaultBaseConfig() BaseConfig { return BaseConfig{ @@ -161,12 +157,16 @@ func DefaultBaseConfig() BaseConfig { // TestBaseConfig returns a base configuration for testing a Tendermint node func TestBaseConfig() BaseConfig { - conf := DefaultBaseConfig() - conf.chainID = "tendermint_test" - conf.ProxyApp = "kvstore" - conf.FastSync = false - conf.DBBackend = "memdb" - return conf + cfg := DefaultBaseConfig() + cfg.chainID = "tendermint_test" + cfg.ProxyApp = "kvstore" + cfg.FastSync = false + cfg.DBBackend = "memdb" + return cfg +} + +func (b BaseConfig) ChainID() string { + return b.chainID } // GenesisFile returns the full path to the genesis.json file @@ -229,11 +229,11 @@ func DefaultRPCConfig() *RPCConfig { // TestRPCConfig returns a configuration for testing the RPC server func TestRPCConfig() *RPCConfig { - conf := DefaultRPCConfig() - conf.ListenAddress = "tcp://0.0.0.0:36657" - conf.GRPCListenAddress = "tcp://0.0.0.0:36658" - conf.Unsafe = true - return conf + cfg := DefaultRPCConfig() + cfg.ListenAddress = "tcp://0.0.0.0:36657" + cfg.GRPCListenAddress = "tcp://0.0.0.0:36658" + cfg.Unsafe = true + return cfg } //----------------------------------------------------------------------------- @@ -313,11 +313,11 @@ func DefaultP2PConfig() *P2PConfig { // TestP2PConfig returns a configuration for testing the peer-to-peer layer func TestP2PConfig() *P2PConfig { - conf := DefaultP2PConfig() - conf.ListenAddress = "tcp://0.0.0.0:36656" - conf.SkipUPNP = true - conf.FlushThrottleTimeout = 10 - return conf + cfg := DefaultP2PConfig() + cfg.ListenAddress = "tcp://0.0.0.0:36656" + cfg.SkipUPNP = true + cfg.FlushThrottleTimeout = 10 + return cfg } // AddrBookFile returns the full path to the address book @@ -351,9 +351,9 @@ func DefaultMempoolConfig() *MempoolConfig { // TestMempoolConfig returns a configuration for testing the Tendermint mempool func TestMempoolConfig() *MempoolConfig { - config := DefaultMempoolConfig() - config.CacheSize = 1000 - return config + cfg := DefaultMempoolConfig() + cfg.CacheSize = 1000 + return cfg } // WalDir returns the full path to the mempool's write-ahead log @@ -397,46 +397,6 @@ type ConsensusConfig struct { PeerQueryMaj23SleepDuration int `mapstructure:"peer_query_maj23_sleep_duration"` } -// WaitForTxs returns true if the consensus should wait for transactions before entering the propose step -func (cfg *ConsensusConfig) WaitForTxs() bool { - return !cfg.CreateEmptyBlocks || cfg.CreateEmptyBlocksInterval > 0 -} - -// EmptyBlocks returns the amount of time to wait before proposing an empty block or starting the propose timer if there are no txs available -func (cfg *ConsensusConfig) EmptyBlocksInterval() time.Duration { - return time.Duration(cfg.CreateEmptyBlocksInterval) * time.Second -} - -// Propose returns the amount of time to wait for a proposal -func (cfg *ConsensusConfig) Propose(round int) time.Duration { - return time.Duration(cfg.TimeoutPropose+cfg.TimeoutProposeDelta*round) * time.Millisecond -} - -// Prevote returns the amount of time to wait for straggler votes after receiving any +2/3 prevotes -func (cfg *ConsensusConfig) Prevote(round int) time.Duration { - return time.Duration(cfg.TimeoutPrevote+cfg.TimeoutPrevoteDelta*round) * time.Millisecond -} - -// Precommit returns the amount of time to wait for straggler votes after receiving any +2/3 precommits -func (cfg *ConsensusConfig) Precommit(round int) time.Duration { - return time.Duration(cfg.TimeoutPrecommit+cfg.TimeoutPrecommitDelta*round) * time.Millisecond -} - -// Commit returns the amount of time to wait for straggler votes after receiving +2/3 precommits for a single block (ie. a commit). -func (cfg *ConsensusConfig) Commit(t time.Time) time.Time { - return t.Add(time.Duration(cfg.TimeoutCommit) * time.Millisecond) -} - -// PeerGossipSleep returns the amount of time to sleep if there is nothing to send from the ConsensusReactor -func (cfg *ConsensusConfig) PeerGossipSleep() time.Duration { - return time.Duration(cfg.PeerGossipSleepDuration) * time.Millisecond -} - -// PeerQueryMaj23Sleep returns the amount of time to sleep after each VoteSetMaj23Message is sent in the ConsensusReactor -func (cfg *ConsensusConfig) PeerQueryMaj23Sleep() time.Duration { - return time.Duration(cfg.PeerQueryMaj23SleepDuration) * time.Millisecond -} - // DefaultConsensusConfig returns a default configuration for the consensus service func DefaultConsensusConfig() *ConsensusConfig { return &ConsensusConfig{ @@ -461,18 +421,58 @@ func DefaultConsensusConfig() *ConsensusConfig { // TestConsensusConfig returns a configuration for testing the consensus service func TestConsensusConfig() *ConsensusConfig { - config := DefaultConsensusConfig() - config.TimeoutPropose = 100 - config.TimeoutProposeDelta = 1 - config.TimeoutPrevote = 10 - config.TimeoutPrevoteDelta = 1 - config.TimeoutPrecommit = 10 - config.TimeoutPrecommitDelta = 1 - config.TimeoutCommit = 10 - config.SkipTimeoutCommit = true - config.PeerGossipSleepDuration = 5 - config.PeerQueryMaj23SleepDuration = 250 - return config + cfg := DefaultConsensusConfig() + cfg.TimeoutPropose = 100 + cfg.TimeoutProposeDelta = 1 + cfg.TimeoutPrevote = 10 + cfg.TimeoutPrevoteDelta = 1 + cfg.TimeoutPrecommit = 10 + cfg.TimeoutPrecommitDelta = 1 + cfg.TimeoutCommit = 10 + cfg.SkipTimeoutCommit = true + cfg.PeerGossipSleepDuration = 5 + cfg.PeerQueryMaj23SleepDuration = 250 + return cfg +} + +// WaitForTxs returns true if the consensus should wait for transactions before entering the propose step +func (c *ConsensusConfig) WaitForTxs() bool { + return !c.CreateEmptyBlocks || c.CreateEmptyBlocksInterval > 0 +} + +// EmptyBlocks returns the amount of time to wait before proposing an empty block or starting the propose timer if there are no txs available +func (c *ConsensusConfig) EmptyBlocksInterval() time.Duration { + return time.Duration(c.CreateEmptyBlocksInterval) * time.Second +} + +// Propose returns the amount of time to wait for a proposal +func (c *ConsensusConfig) Propose(round int) time.Duration { + return time.Duration(c.TimeoutPropose+c.TimeoutProposeDelta*round) * time.Millisecond +} + +// Prevote returns the amount of time to wait for straggler votes after receiving any +2/3 prevotes +func (c *ConsensusConfig) Prevote(round int) time.Duration { + return time.Duration(c.TimeoutPrevote+c.TimeoutPrevoteDelta*round) * time.Millisecond +} + +// Precommit returns the amount of time to wait for straggler votes after receiving any +2/3 precommits +func (c *ConsensusConfig) Precommit(round int) time.Duration { + return time.Duration(c.TimeoutPrecommit+c.TimeoutPrecommitDelta*round) * time.Millisecond +} + +// Commit returns the amount of time to wait for straggler votes after receiving +2/3 precommits for a single block (ie. a commit). +func (c *ConsensusConfig) Commit(t time.Time) time.Time { + return t.Add(time.Duration(c.TimeoutCommit) * time.Millisecond) +} + +// PeerGossipSleep returns the amount of time to sleep if there is nothing to send from the ConsensusReactor +func (c *ConsensusConfig) PeerGossipSleep() time.Duration { + return time.Duration(c.PeerGossipSleepDuration) * time.Millisecond +} + +// PeerQueryMaj23Sleep returns the amount of time to sleep after each VoteSetMaj23Message is sent in the ConsensusReactor +func (c *ConsensusConfig) PeerQueryMaj23Sleep() time.Duration { + return time.Duration(c.PeerQueryMaj23SleepDuration) * time.Millisecond } // WalFile returns the full path to the write-ahead log file From 575a46d9d4340a4c96fd498c8e28eb5fd5c7ee1f Mon Sep 17 00:00:00 2001 From: Alex Hernandez Date: Thu, 29 Mar 2018 17:28:29 +0800 Subject: [PATCH 12/54] fix typo on block header (#1387) --- types/block.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/block.go b/types/block.go index 970ca36f4..31443b1a2 100644 --- a/types/block.go +++ b/types/block.go @@ -215,7 +215,7 @@ func (h *Header) StringIndented(indent string) string { %s Data: %v %s Validators: %v %s App: %v -%s Conensus: %v +%s Consensus: %v %s Results: %v %s Evidence: %v %s}#%v`, From eaee98ee1f4b72a3a4b3a64766b09070b65e2bb6 Mon Sep 17 00:00:00 2001 From: Greg Szabo <16846635+greg-szabo@users.noreply.github.com> Date: Sun, 1 Apr 2018 13:54:48 -0400 Subject: [PATCH 13/54] CGO_ENABLED=0 added for static linking (#1396) --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 851151eeb..c19ce11a6 100644 --- a/Makefile +++ b/Makefile @@ -14,13 +14,13 @@ check: check_tools ensure_deps ### Build build: - go build $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' -o build/tendermint ./cmd/tendermint/ + CGO_ENABLED=0 go build $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' -o build/tendermint ./cmd/tendermint/ build_race: - go build -race $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' -o build/tendermint ./cmd/tendermint + CGO_ENABLED=0 go build -race $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' -o build/tendermint ./cmd/tendermint install: - go install $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' ./cmd/tendermint + CGO_ENABLED=0 go install $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' ./cmd/tendermint ######################################## ### Distribution From 2644a529f0ddfb3fa928e8223ed4f8a7e91bf05b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Corbi=C3=A8re?= Date: Mon, 2 Apr 2018 10:21:17 +0200 Subject: [PATCH 14/54] Fix lint errors (#1390) * use increment and decrement operators. * remove unnecessary else branches. * fix package comment with leading space. * fix receiver names. * fix error strings. * remove omittable code. * remove redundant return statement. * Revert changes (code is generated.) * use cfg as receiver name for all config-related types. * use lsi as the receiver name for the LastSignedInfo type. --- cmd/tendermint/commands/run_node.go | 3 +- config/config.go | 72 ++++++++--------- consensus/common_test.go | 4 +- consensus/mempool_test.go | 11 ++- consensus/reactor.go | 10 +-- consensus/reactor_test.go | 2 +- consensus/replay.go | 12 +-- consensus/replay_file.go | 17 ++-- consensus/replay_test.go | 6 +- consensus/state.go | 19 +++-- node/node.go | 13 ++- p2p/base_reactor.go | 8 +- p2p/conn/connection.go | 3 +- p2p/conn/secret_connection.go | 7 +- p2p/conn/secret_connection_test.go | 2 +- p2p/key.go | 5 +- p2p/netaddress.go | 4 +- p2p/peer_set.go | 3 +- p2p/peer_set_test.go | 2 +- p2p/pex/known_address.go | 2 +- p2p/pex/pex_reactor.go | 3 +- p2p/test_util.go | 2 +- p2p/upnp/upnp.go | 2 +- rpc/core/tx.go | 4 +- rpc/lib/client/ws_client.go | 26 +++--- rpc/lib/types/types.go | 3 +- scripts/wal2json/main.go | 1 + state/state_test.go | 4 +- state/txindex/kv/kv.go | 3 +- state/txindex/kv/kv_test.go | 2 +- types/block.go | 3 +- types/block_test.go | 2 +- types/evidence.go | 2 +- types/genesis.go | 3 +- types/part_set.go | 30 +++---- types/priv_validator.go | 120 ++++++++++++++-------------- types/priv_validator/sign_info.go | 46 +++++------ types/validator_set.go | 49 +++++------- types/validator_set_test.go | 2 +- types/vote_set.go | 49 +++++------- 40 files changed, 262 insertions(+), 299 deletions(-) diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index 6ad92441b..0d50f9e4b 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -57,9 +57,8 @@ func NewRunNodeCmd(nodeProvider nm.NodeProvider) *cobra.Command { if err := n.Start(); err != nil { return fmt.Errorf("Failed to start node: %v", err) - } else { - logger.Info("Started node", "nodeInfo", n.Switch().NodeInfo()) } + logger.Info("Started node", "nodeInfo", n.Switch().NodeInfo()) // Trap signal, run forever. n.RunForever() diff --git a/config/config.go b/config/config.go index 0844fbd51..c87d56b3d 100644 --- a/config/config.go +++ b/config/config.go @@ -165,28 +165,28 @@ func TestBaseConfig() BaseConfig { return cfg } -func (b BaseConfig) ChainID() string { - return b.chainID +func (cfg BaseConfig) ChainID() string { + return cfg.chainID } // GenesisFile returns the full path to the genesis.json file -func (b BaseConfig) GenesisFile() string { - return rootify(b.Genesis, b.RootDir) +func (cfg BaseConfig) GenesisFile() string { + return rootify(cfg.Genesis, cfg.RootDir) } // PrivValidatorFile returns the full path to the priv_validator.json file -func (b BaseConfig) PrivValidatorFile() string { - return rootify(b.PrivValidator, b.RootDir) +func (cfg BaseConfig) PrivValidatorFile() string { + return rootify(cfg.PrivValidator, cfg.RootDir) } // NodeKeyFile returns the full path to the node_key.json file -func (b BaseConfig) NodeKeyFile() string { - return rootify(b.NodeKey, b.RootDir) +func (cfg BaseConfig) NodeKeyFile() string { + return rootify(cfg.NodeKey, cfg.RootDir) } // DBDir returns the full path to the database directory -func (b BaseConfig) DBDir() string { - return rootify(b.DBPath, b.RootDir) +func (cfg BaseConfig) DBDir() string { + return rootify(cfg.DBPath, cfg.RootDir) } // DefaultLogLevel returns a default log level of "error" @@ -321,8 +321,8 @@ func TestP2PConfig() *P2PConfig { } // AddrBookFile returns the full path to the address book -func (p *P2PConfig) AddrBookFile() string { - return rootify(p.AddrBook, p.RootDir) +func (cfg *P2PConfig) AddrBookFile() string { + return rootify(cfg.AddrBook, cfg.RootDir) } //----------------------------------------------------------------------------- @@ -357,8 +357,8 @@ func TestMempoolConfig() *MempoolConfig { } // WalDir returns the full path to the mempool's write-ahead log -func (m *MempoolConfig) WalDir() string { - return rootify(m.WalPath, m.RootDir) +func (cfg *MempoolConfig) WalDir() string { + return rootify(cfg.WalPath, cfg.RootDir) } //----------------------------------------------------------------------------- @@ -436,56 +436,56 @@ func TestConsensusConfig() *ConsensusConfig { } // WaitForTxs returns true if the consensus should wait for transactions before entering the propose step -func (c *ConsensusConfig) WaitForTxs() bool { - return !c.CreateEmptyBlocks || c.CreateEmptyBlocksInterval > 0 +func (cfg *ConsensusConfig) WaitForTxs() bool { + return !cfg.CreateEmptyBlocks || cfg.CreateEmptyBlocksInterval > 0 } // EmptyBlocks returns the amount of time to wait before proposing an empty block or starting the propose timer if there are no txs available -func (c *ConsensusConfig) EmptyBlocksInterval() time.Duration { - return time.Duration(c.CreateEmptyBlocksInterval) * time.Second +func (cfg *ConsensusConfig) EmptyBlocksInterval() time.Duration { + return time.Duration(cfg.CreateEmptyBlocksInterval) * time.Second } // Propose returns the amount of time to wait for a proposal -func (c *ConsensusConfig) Propose(round int) time.Duration { - return time.Duration(c.TimeoutPropose+c.TimeoutProposeDelta*round) * time.Millisecond +func (cfg *ConsensusConfig) Propose(round int) time.Duration { + return time.Duration(cfg.TimeoutPropose+cfg.TimeoutProposeDelta*round) * time.Millisecond } // Prevote returns the amount of time to wait for straggler votes after receiving any +2/3 prevotes -func (c *ConsensusConfig) Prevote(round int) time.Duration { - return time.Duration(c.TimeoutPrevote+c.TimeoutPrevoteDelta*round) * time.Millisecond +func (cfg *ConsensusConfig) Prevote(round int) time.Duration { + return time.Duration(cfg.TimeoutPrevote+cfg.TimeoutPrevoteDelta*round) * time.Millisecond } // Precommit returns the amount of time to wait for straggler votes after receiving any +2/3 precommits -func (c *ConsensusConfig) Precommit(round int) time.Duration { - return time.Duration(c.TimeoutPrecommit+c.TimeoutPrecommitDelta*round) * time.Millisecond +func (cfg *ConsensusConfig) Precommit(round int) time.Duration { + return time.Duration(cfg.TimeoutPrecommit+cfg.TimeoutPrecommitDelta*round) * time.Millisecond } // Commit returns the amount of time to wait for straggler votes after receiving +2/3 precommits for a single block (ie. a commit). -func (c *ConsensusConfig) Commit(t time.Time) time.Time { - return t.Add(time.Duration(c.TimeoutCommit) * time.Millisecond) +func (cfg *ConsensusConfig) Commit(t time.Time) time.Time { + return t.Add(time.Duration(cfg.TimeoutCommit) * time.Millisecond) } // PeerGossipSleep returns the amount of time to sleep if there is nothing to send from the ConsensusReactor -func (c *ConsensusConfig) PeerGossipSleep() time.Duration { - return time.Duration(c.PeerGossipSleepDuration) * time.Millisecond +func (cfg *ConsensusConfig) PeerGossipSleep() time.Duration { + return time.Duration(cfg.PeerGossipSleepDuration) * time.Millisecond } // PeerQueryMaj23Sleep returns the amount of time to sleep after each VoteSetMaj23Message is sent in the ConsensusReactor -func (c *ConsensusConfig) PeerQueryMaj23Sleep() time.Duration { - return time.Duration(c.PeerQueryMaj23SleepDuration) * time.Millisecond +func (cfg *ConsensusConfig) PeerQueryMaj23Sleep() time.Duration { + return time.Duration(cfg.PeerQueryMaj23SleepDuration) * time.Millisecond } // WalFile returns the full path to the write-ahead log file -func (c *ConsensusConfig) WalFile() string { - if c.walFile != "" { - return c.walFile +func (cfg *ConsensusConfig) WalFile() string { + if cfg.walFile != "" { + return cfg.walFile } - return rootify(c.WalPath, c.RootDir) + return rootify(cfg.WalPath, cfg.RootDir) } // SetWalFile sets the path to the write-ahead log file -func (c *ConsensusConfig) SetWalFile(walFile string) { - c.walFile = walFile +func (cfg *ConsensusConfig) SetWalFile(walFile string) { + cfg.walFile = walFile } //----------------------------------------------------------------------------- diff --git a/consensus/common_test.go b/consensus/common_test.go index 6d16958ca..287852e04 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -101,13 +101,13 @@ func signVotes(voteType byte, hash []byte, header types.PartSetHeader, vss ...*v func incrementHeight(vss ...*validatorStub) { for _, vs := range vss { - vs.Height += 1 + vs.Height++ } } func incrementRound(vss ...*validatorStub) { for _, vs := range vss { - vs.Round += 1 + vs.Round++ } } diff --git a/consensus/mempool_test.go b/consensus/mempool_test.go index d283ff4ba..31e4e1e62 100644 --- a/consensus/mempool_test.go +++ b/consensus/mempool_test.go @@ -200,7 +200,7 @@ func (app *CounterApplication) DeliverTx(tx []byte) abci.ResponseDeliverTx { Code: code.CodeTypeBadNonce, Log: fmt.Sprintf("Invalid nonce. Expected %v, got %v", app.txCount, txValue)} } - app.txCount += 1 + app.txCount++ return abci.ResponseDeliverTx{Code: code.CodeTypeOK} } @@ -211,7 +211,7 @@ func (app *CounterApplication) CheckTx(tx []byte) abci.ResponseCheckTx { Code: code.CodeTypeBadNonce, Log: fmt.Sprintf("Invalid nonce. Expected %v, got %v", app.mempoolTxCount, txValue)} } - app.mempoolTxCount += 1 + app.mempoolTxCount++ return abci.ResponseCheckTx{Code: code.CodeTypeOK} } @@ -225,9 +225,8 @@ func (app *CounterApplication) Commit() abci.ResponseCommit { app.mempoolTxCount = app.txCount if app.txCount == 0 { return abci.ResponseCommit{} - } else { - hash := make([]byte, 8) - binary.BigEndian.PutUint64(hash, uint64(app.txCount)) - return abci.ResponseCommit{Data: hash} } + hash := make([]byte, 8) + binary.BigEndian.PutUint64(hash, uint64(app.txCount)) + return abci.ResponseCommit{Data: hash} } diff --git a/consensus/reactor.go b/consensus/reactor.go index a265f76c0..b13be0e87 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -602,11 +602,9 @@ func (conR *ConsensusReactor) gossipDataForCatchup(logger log.Logger, rs *cstype logger.Debug("Sending block part for catchup failed") } return - } else { - //logger.Info("No parts to send in catch-up, sleeping") - time.Sleep(conR.conS.config.PeerGossipSleep()) - return } + //logger.Info("No parts to send in catch-up, sleeping") + time.Sleep(conR.conS.config.PeerGossipSleep()) } func (conR *ConsensusReactor) gossipVotesRoutine(peer p2p.Peer, ps *PeerState) { @@ -1087,7 +1085,7 @@ func (ps *PeerState) RecordVote(vote *types.Vote) int { return ps.stats.votes } ps.stats.lastVoteHeight = vote.Height - ps.stats.votes += 1 + ps.stats.votes++ return ps.stats.votes } @@ -1106,7 +1104,7 @@ func (ps *PeerState) RecordBlockPart(bp *BlockPartMessage) int { } ps.stats.lastBlockPartHeight = bp.Height - ps.stats.blockParts += 1 + ps.stats.blockParts++ return ps.stats.blockParts } diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index 26fc7e171..8448caf63 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -441,7 +441,7 @@ func waitForAndValidateBlockWithTx(t *testing.T, n int, activeVals map[string]st // but they should be in order. for _, tx := range newBlock.Data.Txs { assert.EqualValues(t, txs[ntxs], tx) - ntxs += 1 + ntxs++ } if ntxs == len(txs) { diff --git a/consensus/replay.go b/consensus/replay.go index 39dd592a4..5b5a48425 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -112,7 +112,7 @@ func (cs *ConsensusState) catchupReplay(csHeight int64) error { } } if found { - return fmt.Errorf("WAL should not contain #ENDHEIGHT %d.", csHeight) + return fmt.Errorf("WAL should not contain #ENDHEIGHT %d", csHeight) } // Search for last height marker @@ -125,7 +125,7 @@ func (cs *ConsensusState) catchupReplay(csHeight int64) error { return err } if !found { - return fmt.Errorf("Cannot replay height %d. WAL does not contain #ENDHEIGHT for %d.", csHeight, csHeight-1) + return fmt.Errorf("Cannot replay height %d. WAL does not contain #ENDHEIGHT for %d", csHeight, csHeight-1) } defer gr.Close() // nolint: errcheck @@ -352,7 +352,7 @@ func (h *Handshaker) replayBlocks(state sm.State, proxyApp proxy.AppConns, appBl var err error finalBlock := storeBlockHeight if mutateState { - finalBlock -= 1 + finalBlock-- } for i := appBlockHeight + 1; i <= finalBlock; i++ { h.logger.Info("Applying block", "height", i) @@ -362,7 +362,7 @@ func (h *Handshaker) replayBlocks(state sm.State, proxyApp proxy.AppConns, appBl return nil, err } - h.nBlocks += 1 + h.nBlocks++ } if mutateState { @@ -390,7 +390,7 @@ func (h *Handshaker) replayBlock(state sm.State, height int64, proxyApp proxy.Ap return sm.State{}, err } - h.nBlocks += 1 + h.nBlocks++ return state, nil } @@ -429,7 +429,7 @@ type mockProxyApp struct { func (mock *mockProxyApp) DeliverTx(tx []byte) abci.ResponseDeliverTx { r := mock.abciResponses.DeliverTx[mock.txCount] - mock.txCount += 1 + mock.txCount++ return *r } diff --git a/consensus/replay_file.go b/consensus/replay_file.go index 2fcd9b472..1fd4f415d 100644 --- a/consensus/replay_file.go +++ b/consensus/replay_file.go @@ -87,9 +87,9 @@ func (cs *ConsensusState) ReplayFile(file string, console bool) error { } if nextN > 0 { - nextN -= 1 + nextN-- } - pb.count += 1 + pb.count++ } return nil } @@ -153,7 +153,7 @@ func (pb *playback) replayReset(count int, newStepCh chan interface{}) error { if err := pb.cs.readReplayMessage(msg, newStepCh); err != nil { return err } - pb.count += 1 + pb.count++ } return nil } @@ -197,13 +197,12 @@ func (pb *playback) replayConsoleLoop() int { if len(tokens) == 1 { return 0 + } + i, err := strconv.Atoi(tokens[1]) + if err != nil { + fmt.Println("next takes an integer argument") } else { - i, err := strconv.Atoi(tokens[1]) - if err != nil { - fmt.Println("next takes an integer argument") - } else { - return i - } + return i } case "back": diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 9d7bbc8e4..606df8260 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -382,9 +382,9 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) { expectedBlocksToSync := NUM_BLOCKS - nBlocks if nBlocks == NUM_BLOCKS && mode > 0 { - expectedBlocksToSync += 1 + expectedBlocksToSync++ } else if nBlocks > 0 && mode == 1 { - expectedBlocksToSync += 1 + expectedBlocksToSync++ } if handshaker.NBlocks() != expectedBlocksToSync { @@ -533,7 +533,7 @@ func makeBlockchainFromWAL(wal WAL) ([]*types.Block, []*types.Commit, error) { } blocks = append(blocks, block) commits = append(commits, thisBlockCommit) - height += 1 + height++ } case *types.PartSetHeader: thisBlockParts = types.NewPartSetFromHeader(*p) diff --git a/consensus/state.go b/consensus/state.go index b59baf1ee..6acad698b 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -494,7 +494,7 @@ func (cs *ConsensusState) updateToState(state sm.State) { func (cs *ConsensusState) newStep() { rs := cs.RoundStateEvent() cs.wal.Save(rs) - cs.nSteps += 1 + cs.nSteps++ // newStep is called by updateToStep in NewConsensusState before the eventBus is set! if cs.eventBus != nil { cs.eventBus.PublishEventNewRoundStep(rs) @@ -741,7 +741,7 @@ func (cs *ConsensusState) proposalHeartbeat(height int64, round int) { } cs.privValidator.SignHeartbeat(chainID, heartbeat) cs.eventBus.PublishEventProposalHeartbeat(types.EventDataProposalHeartbeat{heartbeat}) - counter += 1 + counter++ time.Sleep(proposalHeartbeatIntervalSeconds * time.Second) } } @@ -852,10 +852,10 @@ func (cs *ConsensusState) isProposalComplete() bool { // make sure we have the prevotes from it too if cs.Proposal.POLRound < 0 { return true - } else { - // if this is false the proposer is lying or we haven't received the POL yet - return cs.Votes.Prevotes(cs.Proposal.POLRound).HasTwoThirdsMajority() } + // if this is false the proposer is lying or we haven't received the POL yet + return cs.Votes.Prevotes(cs.Proposal.POLRound).HasTwoThirdsMajority() + } // Create the next block to propose and return it. @@ -1498,12 +1498,11 @@ func (cs *ConsensusState) signAddVote(type_ byte, hash []byte, header types.Part cs.sendInternalMessage(msgInfo{&VoteMessage{vote}, ""}) cs.Logger.Info("Signed and pushed vote", "height", cs.Height, "round", cs.Round, "vote", vote, "err", err) return vote - } else { - //if !cs.replayMode { - cs.Logger.Error("Error signing vote", "height", cs.Height, "round", cs.Round, "vote", vote, "err", err) - //} - return nil } + //if !cs.replayMode { + cs.Logger.Error("Error signing vote", "height", cs.Height, "round", cs.Round, "vote", vote, "err", err) + //} + return nil } //--------------------------------------------------------- diff --git a/node/node.go b/node/node.go index 1f745bf00..3a59d2f06 100644 --- a/node/node.go +++ b/node/node.go @@ -643,14 +643,13 @@ func loadGenesisDoc(db dbm.DB) (*types.GenesisDoc, error) { bytes := db.Get(genesisDocKey) if len(bytes) == 0 { return nil, errors.New("Genesis doc not found") - } else { - var genDoc *types.GenesisDoc - err := json.Unmarshal(bytes, &genDoc) - if err != nil { - cmn.PanicCrisis(fmt.Sprintf("Failed to load genesis doc due to unmarshaling error: %v (bytes: %X)", err, bytes)) - } - return genDoc, nil } + var genDoc *types.GenesisDoc + err := json.Unmarshal(bytes, &genDoc) + if err != nil { + cmn.PanicCrisis(fmt.Sprintf("Failed to load genesis doc due to unmarshaling error: %v (bytes: %X)", err, bytes)) + } + return genDoc, nil } // panics if failed to marshal the given genesis document diff --git a/p2p/base_reactor.go b/p2p/base_reactor.go index f0dd14147..83c8efa4b 100644 --- a/p2p/base_reactor.go +++ b/p2p/base_reactor.go @@ -47,7 +47,7 @@ func NewBaseReactor(name string, impl Reactor) *BaseReactor { func (br *BaseReactor) SetSwitch(sw *Switch) { br.Switch = sw } -func (_ *BaseReactor) GetChannels() []*conn.ChannelDescriptor { return nil } -func (_ *BaseReactor) AddPeer(peer Peer) {} -func (_ *BaseReactor) RemovePeer(peer Peer, reason interface{}) {} -func (_ *BaseReactor) Receive(chID byte, peer Peer, msgBytes []byte) {} +func (*BaseReactor) GetChannels() []*conn.ChannelDescriptor { return nil } +func (*BaseReactor) AddPeer(peer Peer) {} +func (*BaseReactor) RemovePeer(peer Peer, reason interface{}) {} +func (*BaseReactor) Receive(chID byte, peer Peer, msgBytes []byte) {} diff --git a/p2p/conn/connection.go b/p2p/conn/connection.go index e4ae4df80..9a3360f2d 100644 --- a/p2p/conn/connection.go +++ b/p2p/conn/connection.go @@ -422,9 +422,8 @@ func (c *MConnection) sendMsgPacket() bool { // Nothing to send? if leastChannel == nil { return true - } else { - // c.Logger.Info("Found a msgPacket to send") } + // c.Logger.Info("Found a msgPacket to send") // Make & send a msgPacket from this channel n, err := leastChannel.writeMsgPacketTo(c.bufWriter) diff --git a/p2p/conn/secret_connection.go b/p2p/conn/secret_connection.go index 3495853e0..bc67abf3a 100644 --- a/p2p/conn/secret_connection.go +++ b/p2p/conn/secret_connection.go @@ -113,7 +113,7 @@ func (sc *SecretConnection) RemotePubKey() crypto.PubKey { // CONTRACT: data smaller than dataMaxSize is read atomically. func (sc *SecretConnection) Write(data []byte) (n int, err error) { for 0 < len(data) { - var frame []byte = make([]byte, totalFrameSize) + var frame = make([]byte, totalFrameSize) var chunk []byte if dataMaxSize < len(data) { chunk = data[:dataMaxSize] @@ -136,9 +136,8 @@ func (sc *SecretConnection) Write(data []byte) (n int, err error) { _, err := sc.conn.Write(sealedFrame) if err != nil { return n, err - } else { - n += len(chunk) } + n += len(chunk) } return } @@ -347,7 +346,7 @@ func incr2Nonce(nonce *[24]byte) { // increment nonce big-endian by 1 with wraparound. func incrNonce(nonce *[24]byte) { for i := 23; 0 <= i; i-- { - nonce[i] += 1 + nonce[i]++ if nonce[i] != 0 { return } diff --git a/p2p/conn/secret_connection_test.go b/p2p/conn/secret_connection_test.go index 54c453a7c..edb0de04d 100644 --- a/p2p/conn/secret_connection_test.go +++ b/p2p/conn/secret_connection_test.go @@ -180,7 +180,7 @@ func TestSecretConnectionReadWrite(t *testing.T) { var readCount = 0 for _, readChunk := range reads { read += readChunk - readCount += 1 + readCount++ if len(write) <= len(read) { break } diff --git a/p2p/key.go b/p2p/key.go index ea0f0b071..6d0f28586 100644 --- a/p2p/key.go +++ b/p2p/key.go @@ -53,9 +53,8 @@ func LoadOrGenNodeKey(filePath string) (*NodeKey, error) { return nil, err } return nodeKey, nil - } else { - return genNodeKey(filePath) } + return genNodeKey(filePath) } func loadNodeKey(filePath string) (*NodeKey, error) { @@ -66,7 +65,7 @@ func loadNodeKey(filePath string) (*NodeKey, error) { nodeKey := new(NodeKey) err = json.Unmarshal(jsonBytes, nodeKey) if err != nil { - return nil, fmt.Errorf("Error reading NodeKey from %v: %v\n", filePath, err) + return nil, fmt.Errorf("Error reading NodeKey from %v: %v", filePath, err) } return nodeKey, nil } diff --git a/p2p/netaddress.go b/p2p/netaddress.go index 333d16e5d..c7773a9f8 100644 --- a/p2p/netaddress.go +++ b/p2p/netaddress.go @@ -294,7 +294,7 @@ func (na *NetAddress) RFC6145() bool { return rfc6145.Contains(na.IP) } func removeProtocolIfDefined(addr string) string { if strings.Contains(addr, "://") { return strings.Split(addr, "://")[1] - } else { - return addr } + return addr + } diff --git a/p2p/peer_set.go b/p2p/peer_set.go index dc53174a1..a4565ea1d 100644 --- a/p2p/peer_set.go +++ b/p2p/peer_set.go @@ -68,9 +68,8 @@ func (ps *PeerSet) Get(peerKey ID) Peer { item, ok := ps.lookup[peerKey] if ok { return item.peer - } else { - return nil } + return nil } // Remove discards peer by its Key, if the peer was previously memoized. diff --git a/p2p/peer_set_test.go b/p2p/peer_set_test.go index 174e73c9c..c8d1d64d2 100644 --- a/p2p/peer_set_test.go +++ b/p2p/peer_set_test.go @@ -115,7 +115,7 @@ func TestPeerSetAddDuplicate(t *testing.T) { errsTally := make(map[error]int) for i := 0; i < n; i++ { err := <-errsChan - errsTally[err] += 1 + errsTally[err]++ } // Our next procedure is to ensure that only one addition diff --git a/p2p/pex/known_address.go b/p2p/pex/known_address.go index 085eb10fa..0261e4902 100644 --- a/p2p/pex/known_address.go +++ b/p2p/pex/known_address.go @@ -56,7 +56,7 @@ func (ka *knownAddress) isNew() bool { func (ka *knownAddress) markAttempt() { now := time.Now() ka.LastAttempt = now - ka.Attempts += 1 + ka.Attempts++ } func (ka *knownAddress) markGood() { diff --git a/p2p/pex/pex_reactor.go b/p2p/pex/pex_reactor.go index 45a05bdb7..45f689c05 100644 --- a/p2p/pex/pex_reactor.go +++ b/p2p/pex/pex_reactor.go @@ -467,9 +467,8 @@ func (r *PEXReactor) AttemptsToDial(addr *p2p.NetAddress) int { lAttempts, attempted := r.attemptsToDial.Load(addr.DialString()) if attempted { return lAttempts.(_attemptsToDial).number - } else { - return 0 } + return 0 } //---------------------------------------------------------- diff --git a/p2p/test_util.go b/p2p/test_util.go index 0d6427f43..535b0bd0b 100644 --- a/p2p/test_util.go +++ b/p2p/test_util.go @@ -143,7 +143,7 @@ func MakeSwitch(cfg *cfg.P2PConfig, i int, network, version string, initSwitch f Version: version, ListenAddr: cmn.Fmt("%v:%v", network, rand.Intn(64512)+1023), } - for ch, _ := range sw.reactorsByCh { + for ch := range sw.reactorsByCh { ni.Channels = append(ni.Channels, ch) } sw.SetNodeInfo(ni) diff --git a/p2p/upnp/upnp.go b/p2p/upnp/upnp.go index cac67a730..e98538aae 100644 --- a/p2p/upnp/upnp.go +++ b/p2p/upnp/upnp.go @@ -103,7 +103,7 @@ func Discover() (nat NAT, err error) { return } } - err = errors.New("UPnP port discovery failed.") + err = errors.New("UPnP port discovery failed") return } diff --git a/rpc/core/tx.go b/rpc/core/tx.go index 18120ded1..7ddc70809 100644 --- a/rpc/core/tx.go +++ b/rpc/core/tx.go @@ -73,7 +73,7 @@ func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { // if index is disabled, return error if _, ok := txIndexer.(*null.TxIndex); ok { - return nil, fmt.Errorf("Transaction indexing is disabled.") + return nil, fmt.Errorf("Transaction indexing is disabled") } r, err := txIndexer.Get(hash) @@ -169,7 +169,7 @@ func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { func TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error) { // if index is disabled, return error if _, ok := txIndexer.(*null.TxIndex); ok { - return nil, fmt.Errorf("Transaction indexing is disabled.") + return nil, fmt.Errorf("Transaction indexing is disabled") } q, err := tmquery.New(query) diff --git a/rpc/lib/client/ws_client.go b/rpc/lib/client/ws_client.go index fe15cda21..ab2e94d0b 100644 --- a/rpc/lib/client/ws_client.go +++ b/rpc/lib/client/ws_client.go @@ -318,21 +318,21 @@ func (c *WSClient) reconnectRoutine() { c.Logger.Error("failed to reconnect", "err", err, "original_err", originalError) c.Stop() return - } else { - // drain reconnectAfter - LOOP: - for { - select { - case <-c.reconnectAfter: - default: - break LOOP - } - } - err = c.processBacklog() - if err == nil { - c.startReadWriteRoutines() + } + // drain reconnectAfter + LOOP: + for { + select { + case <-c.reconnectAfter: + default: + break LOOP } } + err := c.processBacklog() + if err == nil { + c.startReadWriteRoutines() + } + case <-c.Quit(): return } diff --git a/rpc/lib/types/types.go b/rpc/lib/types/types.go index 37d451457..e4b02c58f 100644 --- a/rpc/lib/types/types.go +++ b/rpc/lib/types/types.go @@ -101,9 +101,8 @@ func NewRPCErrorResponse(id string, code int, msg string, data string) RPCRespon func (resp RPCResponse) String() string { if resp.Error == nil { return fmt.Sprintf("[%s %v]", resp.ID, resp.Result) - } else { - return fmt.Sprintf("[%s %s]", resp.ID, resp.Error) } + return fmt.Sprintf("[%s %s]", resp.ID, resp.Error) } func RPCParseError(id string, err error) RPCResponse { diff --git a/scripts/wal2json/main.go b/scripts/wal2json/main.go index e44ed4b17..f6ffea431 100644 --- a/scripts/wal2json/main.go +++ b/scripts/wal2json/main.go @@ -4,6 +4,7 @@ Usage: wal2json */ + package main import ( diff --git a/state/state_test.go b/state/state_test.go index 1c5a860d5..31f2065cf 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -222,7 +222,7 @@ func TestOneValidatorChangesSaveLoad(t *testing.T) { // use the next pubkey if changeIndex < len(changeHeights) && i == changeHeights[changeIndex] { changeIndex++ - power += 1 + power++ } header, blockID, responses := makeHeaderPartsResponsesValPowerChange(state, i, power) state, err = updateState(state, blockID, header, responses) @@ -240,7 +240,7 @@ func TestOneValidatorChangesSaveLoad(t *testing.T) { // use the next pubkey (note our counter starts at 0 this time) if changeIndex < len(changeHeights) && i == changeHeights[changeIndex]+1 { changeIndex++ - power += 1 + power++ } testCases[i-1] = power } diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index 1f544e58b..74bf4843b 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -148,9 +148,8 @@ func (txi *TxIndex) Search(q *query.Query) ([]*types.TxResult, error) { res, err := txi.Get(hash) if res == nil { return []*types.TxResult{}, nil - } else { - return []*types.TxResult{res}, errors.Wrap(err, "error while retrieving the result") } + return []*types.TxResult{res}, errors.Wrap(err, "error while retrieving the result") } // conditions to skip because they're handled before "everything else" diff --git a/state/txindex/kv/kv_test.go b/state/txindex/kv/kv_test.go index 810da23bd..97024b30b 100644 --- a/state/txindex/kv/kv_test.go +++ b/state/txindex/kv/kv_test.go @@ -196,7 +196,7 @@ func benchmarkTxIndex(txsCount int, b *testing.B) { if err := batch.Add(txResult); err != nil { b.Fatal(err) } - txResult.Index += 1 + txResult.Index++ } b.ResetTimer() diff --git a/types/block.go b/types/block.go index 31443b1a2..0c1d38e55 100644 --- a/types/block.go +++ b/types/block.go @@ -141,9 +141,8 @@ func (b *Block) StringIndented(indent string) string { func (b *Block) StringShort() string { if b == nil { return "nil-Block" - } else { - return fmt.Sprintf("Block#%v", b.Hash()) } + return fmt.Sprintf("Block#%v", b.Hash()) } //----------------------------------------------------------------------------- diff --git a/types/block_test.go b/types/block_test.go index e3e22743c..4aaa16a97 100644 --- a/types/block_test.go +++ b/types/block_test.go @@ -29,7 +29,7 @@ func TestValidateBlock(t *testing.T) { // tamper with NumTxs block = MakeBlock(h, txs, commit) - block.NumTxs += 1 + block.NumTxs++ err = block.ValidateBasic() require.Error(t, err) diff --git a/types/evidence.go b/types/evidence.go index 9e1f6af0e..0b3496045 100644 --- a/types/evidence.go +++ b/types/evidence.go @@ -144,7 +144,7 @@ func (dve *DuplicateVoteEvidence) Verify(chainID string) error { // BlockIDs must be different if dve.VoteA.BlockID.Equals(dve.VoteB.BlockID) { - return fmt.Errorf("DuplicateVoteEvidence Error: BlockIDs are the same (%v) - not a real duplicate vote!", dve.VoteA.BlockID) + return fmt.Errorf("DuplicateVoteEvidence Error: BlockIDs are the same (%v) - not a real duplicate vote", dve.VoteA.BlockID) } // Signatures must be valid diff --git a/types/genesis.go b/types/genesis.go index 8d885db97..21c61806a 100644 --- a/types/genesis.go +++ b/types/genesis.go @@ -37,9 +37,8 @@ type GenesisDoc struct { func (genDoc *GenesisDoc) AppState() json.RawMessage { if len(genDoc.AppOptions) > 0 { return genDoc.AppOptions - } else { - return genDoc.AppStateJSON } + return genDoc.AppStateJSON } // SaveAs is a utility method for saving GenensisDoc as a JSON file. diff --git a/types/part_set.go b/types/part_set.go index 5c43b1518..749943291 100644 --- a/types/part_set.go +++ b/types/part_set.go @@ -30,12 +30,11 @@ type Part struct { func (part *Part) Hash() []byte { if part.hash != nil { return part.hash - } else { - hasher := ripemd160.New() - hasher.Write(part.Bytes) // nolint: errcheck, gas - part.hash = hasher.Sum(nil) - return part.hash } + hasher := ripemd160.New() + hasher.Write(part.Bytes) // nolint: errcheck, gas + part.hash = hasher.Sum(nil) + return part.hash } func (part *Part) String() string { @@ -129,20 +128,18 @@ func NewPartSetFromHeader(header PartSetHeader) *PartSet { func (ps *PartSet) Header() PartSetHeader { if ps == nil { return PartSetHeader{} - } else { - return PartSetHeader{ - Total: ps.total, - Hash: ps.hash, - } + } + return PartSetHeader{ + Total: ps.total, + Hash: ps.hash, } } func (ps *PartSet) HasHeader(header PartSetHeader) bool { if ps == nil { return false - } else { - return ps.Header().Equals(header) } + return ps.Header().Equals(header) } func (ps *PartSet) BitArray() *cmn.BitArray { @@ -251,7 +248,7 @@ func (psr *PartSetReader) Read(p []byte) (n int, err error) { return n1 + n2, err } - psr.i += 1 + psr.i++ if psr.i >= len(psr.parts) { return 0, io.EOF } @@ -262,9 +259,8 @@ func (psr *PartSetReader) Read(p []byte) (n int, err error) { func (ps *PartSet) StringShort() string { if ps == nil { return "nil-PartSet" - } else { - ps.mtx.Lock() - defer ps.mtx.Unlock() - return fmt.Sprintf("(%v of %v)", ps.Count(), ps.Total()) } + ps.mtx.Lock() + defer ps.mtx.Unlock() + return fmt.Sprintf("(%v of %v)", ps.Count(), ps.Total()) } diff --git a/types/priv_validator.go b/types/priv_validator.go index daa456bc0..052ace978 100644 --- a/types/priv_validator.go +++ b/types/priv_validator.go @@ -211,18 +211,18 @@ func LoadPrivValidatorFSWithSigner(filePath string, signerFunc func(PrivValidato } // Save persists the PrivValidatorFS to disk. -func (privVal *PrivValidatorFS) Save() { - privVal.mtx.Lock() - defer privVal.mtx.Unlock() - privVal.save() +func (pv *PrivValidatorFS) Save() { + pv.mtx.Lock() + defer pv.mtx.Unlock() + pv.save() } -func (privVal *PrivValidatorFS) save() { - outFile := privVal.filePath +func (pv *PrivValidatorFS) save() { + outFile := pv.filePath if outFile == "" { panic("Cannot save PrivValidator: filePath not set") } - jsonBytes, err := json.Marshal(privVal) + jsonBytes, err := json.Marshal(pv) if err != nil { panic(err) } @@ -234,22 +234,22 @@ func (privVal *PrivValidatorFS) save() { // Reset resets all fields in the PrivValidatorFS. // NOTE: Unsafe! -func (privVal *PrivValidatorFS) Reset() { +func (pv *PrivValidatorFS) Reset() { var sig crypto.Signature - privVal.LastHeight = 0 - privVal.LastRound = 0 - privVal.LastStep = 0 - privVal.LastSignature = sig - privVal.LastSignBytes = nil - privVal.Save() + pv.LastHeight = 0 + pv.LastRound = 0 + pv.LastStep = 0 + pv.LastSignature = sig + pv.LastSignBytes = nil + pv.Save() } // SignVote signs a canonical representation of the vote, along with the // chainID. Implements PrivValidator. -func (privVal *PrivValidatorFS) SignVote(chainID string, vote *Vote) error { - privVal.mtx.Lock() - defer privVal.mtx.Unlock() - if err := privVal.signVote(chainID, vote); err != nil { +func (pv *PrivValidatorFS) SignVote(chainID string, vote *Vote) error { + pv.mtx.Lock() + defer pv.mtx.Unlock() + if err := pv.signVote(chainID, vote); err != nil { return errors.New(cmn.Fmt("Error signing vote: %v", err)) } return nil @@ -257,32 +257,32 @@ func (privVal *PrivValidatorFS) SignVote(chainID string, vote *Vote) error { // SignProposal signs a canonical representation of the proposal, along with // the chainID. Implements PrivValidator. -func (privVal *PrivValidatorFS) SignProposal(chainID string, proposal *Proposal) error { - privVal.mtx.Lock() - defer privVal.mtx.Unlock() - if err := privVal.signProposal(chainID, proposal); err != nil { +func (pv *PrivValidatorFS) SignProposal(chainID string, proposal *Proposal) error { + pv.mtx.Lock() + defer pv.mtx.Unlock() + if err := pv.signProposal(chainID, proposal); err != nil { return fmt.Errorf("Error signing proposal: %v", err) } return nil } // returns error if HRS regression or no LastSignBytes. returns true if HRS is unchanged -func (privVal *PrivValidatorFS) checkHRS(height int64, round int, step int8) (bool, error) { - if privVal.LastHeight > height { +func (pv *PrivValidatorFS) checkHRS(height int64, round int, step int8) (bool, error) { + if pv.LastHeight > height { return false, errors.New("Height regression") } - if privVal.LastHeight == height { - if privVal.LastRound > round { + if pv.LastHeight == height { + if pv.LastRound > round { return false, errors.New("Round regression") } - if privVal.LastRound == round { - if privVal.LastStep > step { + if pv.LastRound == round { + if pv.LastStep > step { return false, errors.New("Step regression") - } else if privVal.LastStep == step { - if privVal.LastSignBytes != nil { - if privVal.LastSignature.Empty() { + } else if pv.LastStep == step { + if pv.LastSignBytes != nil { + if pv.LastSignature.Empty() { panic("privVal: LastSignature is nil but LastSignBytes is not!") } return true, nil @@ -297,11 +297,11 @@ func (privVal *PrivValidatorFS) checkHRS(height int64, round int, step int8) (bo // signVote checks if the vote is good to sign and sets the vote signature. // It may need to set the timestamp as well if the vote is otherwise the same as // a previously signed vote (ie. we crashed after signing but before the vote hit the WAL). -func (privVal *PrivValidatorFS) signVote(chainID string, vote *Vote) error { +func (pv *PrivValidatorFS) signVote(chainID string, vote *Vote) error { height, round, step := vote.Height, vote.Round, voteToStep(vote) signBytes := vote.SignBytes(chainID) - sameHRS, err := privVal.checkHRS(height, round, step) + sameHRS, err := pv.checkHRS(height, round, step) if err != nil { return err } @@ -312,11 +312,11 @@ func (privVal *PrivValidatorFS) signVote(chainID string, vote *Vote) error { // If they only differ by timestamp, use last timestamp and signature // Otherwise, return error if sameHRS { - if bytes.Equal(signBytes, privVal.LastSignBytes) { - vote.Signature = privVal.LastSignature - } else if timestamp, ok := checkVotesOnlyDifferByTimestamp(privVal.LastSignBytes, signBytes); ok { + if bytes.Equal(signBytes, pv.LastSignBytes) { + vote.Signature = pv.LastSignature + } else if timestamp, ok := checkVotesOnlyDifferByTimestamp(pv.LastSignBytes, signBytes); ok { vote.Timestamp = timestamp - vote.Signature = privVal.LastSignature + vote.Signature = pv.LastSignature } else { err = fmt.Errorf("Conflicting data") } @@ -324,11 +324,11 @@ func (privVal *PrivValidatorFS) signVote(chainID string, vote *Vote) error { } // It passed the checks. Sign the vote - sig, err := privVal.Sign(signBytes) + sig, err := pv.Sign(signBytes) if err != nil { return err } - privVal.saveSigned(height, round, step, signBytes, sig) + pv.saveSigned(height, round, step, signBytes, sig) vote.Signature = sig return nil } @@ -336,11 +336,11 @@ func (privVal *PrivValidatorFS) signVote(chainID string, vote *Vote) error { // signProposal checks if the proposal is good to sign and sets the proposal signature. // It may need to set the timestamp as well if the proposal is otherwise the same as // a previously signed proposal ie. we crashed after signing but before the proposal hit the WAL). -func (privVal *PrivValidatorFS) signProposal(chainID string, proposal *Proposal) error { +func (pv *PrivValidatorFS) signProposal(chainID string, proposal *Proposal) error { height, round, step := proposal.Height, proposal.Round, stepPropose signBytes := proposal.SignBytes(chainID) - sameHRS, err := privVal.checkHRS(height, round, step) + sameHRS, err := pv.checkHRS(height, round, step) if err != nil { return err } @@ -351,11 +351,11 @@ func (privVal *PrivValidatorFS) signProposal(chainID string, proposal *Proposal) // If they only differ by timestamp, use last timestamp and signature // Otherwise, return error if sameHRS { - if bytes.Equal(signBytes, privVal.LastSignBytes) { - proposal.Signature = privVal.LastSignature - } else if timestamp, ok := checkProposalsOnlyDifferByTimestamp(privVal.LastSignBytes, signBytes); ok { + if bytes.Equal(signBytes, pv.LastSignBytes) { + proposal.Signature = pv.LastSignature + } else if timestamp, ok := checkProposalsOnlyDifferByTimestamp(pv.LastSignBytes, signBytes); ok { proposal.Timestamp = timestamp - proposal.Signature = privVal.LastSignature + proposal.Signature = pv.LastSignature } else { err = fmt.Errorf("Conflicting data") } @@ -363,40 +363,40 @@ func (privVal *PrivValidatorFS) signProposal(chainID string, proposal *Proposal) } // It passed the checks. Sign the proposal - sig, err := privVal.Sign(signBytes) + sig, err := pv.Sign(signBytes) if err != nil { return err } - privVal.saveSigned(height, round, step, signBytes, sig) + pv.saveSigned(height, round, step, signBytes, sig) proposal.Signature = sig return nil } // Persist height/round/step and signature -func (privVal *PrivValidatorFS) saveSigned(height int64, round int, step int8, +func (pv *PrivValidatorFS) saveSigned(height int64, round int, step int8, signBytes []byte, sig crypto.Signature) { - privVal.LastHeight = height - privVal.LastRound = round - privVal.LastStep = step - privVal.LastSignature = sig - privVal.LastSignBytes = signBytes - privVal.save() + pv.LastHeight = height + pv.LastRound = round + pv.LastStep = step + pv.LastSignature = sig + pv.LastSignBytes = signBytes + pv.save() } // SignHeartbeat signs a canonical representation of the heartbeat, along with the chainID. // Implements PrivValidator. -func (privVal *PrivValidatorFS) SignHeartbeat(chainID string, heartbeat *Heartbeat) error { - privVal.mtx.Lock() - defer privVal.mtx.Unlock() +func (pv *PrivValidatorFS) SignHeartbeat(chainID string, heartbeat *Heartbeat) error { + pv.mtx.Lock() + defer pv.mtx.Unlock() var err error - heartbeat.Signature, err = privVal.Sign(heartbeat.SignBytes(chainID)) + heartbeat.Signature, err = pv.Sign(heartbeat.SignBytes(chainID)) return err } // String returns a string representation of the PrivValidatorFS. -func (privVal *PrivValidatorFS) String() string { - return fmt.Sprintf("PrivValidator{%v LH:%v, LR:%v, LS:%v}", privVal.GetAddress(), privVal.LastHeight, privVal.LastRound, privVal.LastStep) +func (pv *PrivValidatorFS) String() string { + return fmt.Sprintf("PrivValidator{%v LH:%v, LR:%v, LS:%v}", pv.GetAddress(), pv.LastHeight, pv.LastRound, pv.LastStep) } //------------------------------------- diff --git a/types/priv_validator/sign_info.go b/types/priv_validator/sign_info.go index 746131a96..8b135df63 100644 --- a/types/priv_validator/sign_info.go +++ b/types/priv_validator/sign_info.go @@ -49,30 +49,30 @@ func NewLastSignedInfo() *LastSignedInfo { } } -func (info *LastSignedInfo) String() string { - return fmt.Sprintf("LH:%v, LR:%v, LS:%v", info.Height, info.Round, info.Step) +func (lsi *LastSignedInfo) String() string { + return fmt.Sprintf("LH:%v, LR:%v, LS:%v", lsi.Height, lsi.Round, lsi.Step) } // Verify returns an error if there is a height/round/step regression // or if the HRS matches but there are no LastSignBytes. // It returns true if HRS matches exactly and the LastSignature exists. // It panics if the HRS matches, the LastSignBytes are not empty, but the LastSignature is empty. -func (info LastSignedInfo) Verify(height int64, round int, step int8) (bool, error) { - if info.Height > height { +func (lsi LastSignedInfo) Verify(height int64, round int, step int8) (bool, error) { + if lsi.Height > height { return false, errors.New("Height regression") } - if info.Height == height { - if info.Round > round { + if lsi.Height == height { + if lsi.Round > round { return false, errors.New("Round regression") } - if info.Round == round { - if info.Step > step { + if lsi.Round == round { + if lsi.Step > step { return false, errors.New("Step regression") - } else if info.Step == step { - if info.SignBytes != nil { - if info.Signature.Empty() { + } else if lsi.Step == step { + if lsi.SignBytes != nil { + if lsi.Signature.Empty() { panic("info: LastSignature is nil but LastSignBytes is not!") } return true, nil @@ -85,24 +85,24 @@ func (info LastSignedInfo) Verify(height int64, round int, step int8) (bool, err } // Set height/round/step and signature on the info -func (info *LastSignedInfo) Set(height int64, round int, step int8, +func (lsi *LastSignedInfo) Set(height int64, round int, step int8, signBytes []byte, sig crypto.Signature) { - info.Height = height - info.Round = round - info.Step = step - info.Signature = sig - info.SignBytes = signBytes + lsi.Height = height + lsi.Round = round + lsi.Step = step + lsi.Signature = sig + lsi.SignBytes = signBytes } // Reset resets all the values. // XXX: Unsafe. -func (info *LastSignedInfo) Reset() { - info.Height = 0 - info.Round = 0 - info.Step = 0 - info.Signature = crypto.Signature{} - info.SignBytes = nil +func (lsi *LastSignedInfo) Reset() { + lsi.Height = 0 + lsi.Round = 0 + lsi.Step = 0 + lsi.Signature = crypto.Signature{} + lsi.SignBytes = nil } // SignVote checks the height/round/step (HRS) are greater than the latest state of the LastSignedInfo. diff --git a/types/validator_set.go b/types/validator_set.go index 83d066ec1..dc6b66d68 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -96,9 +96,8 @@ func (valSet *ValidatorSet) GetByAddress(address []byte) (index int, val *Valida }) if idx != len(valSet.Validators) && bytes.Equal(valSet.Validators[idx].Address, address) { return idx, valSet.Validators[idx].Copy() - } else { - return 0, nil } + return 0, nil } // GetByIndex returns the validator by index. @@ -187,13 +186,12 @@ func (valSet *ValidatorSet) Update(val *Validator) (updated bool) { index, sameVal := valSet.GetByAddress(val.Address) if sameVal == nil { return false - } else { - valSet.Validators[index] = val.Copy() - // Invalidate cache - valSet.Proposer = nil - valSet.totalVotingPower = 0 - return true } + valSet.Validators[index] = val.Copy() + // Invalidate cache + valSet.Proposer = nil + valSet.totalVotingPower = 0 + return true } func (valSet *ValidatorSet) Remove(address []byte) (val *Validator, removed bool) { @@ -202,18 +200,17 @@ func (valSet *ValidatorSet) Remove(address []byte) (val *Validator, removed bool }) if idx == len(valSet.Validators) || !bytes.Equal(valSet.Validators[idx].Address, address) { return nil, false - } else { - removedVal := valSet.Validators[idx] - newValidators := valSet.Validators[:idx] - if idx+1 < len(valSet.Validators) { - newValidators = append(newValidators, valSet.Validators[idx+1:]...) - } - valSet.Validators = newValidators - // Invalidate cache - valSet.Proposer = nil - valSet.totalVotingPower = 0 - return removedVal, true } + removedVal := valSet.Validators[idx] + newValidators := valSet.Validators[:idx] + if idx+1 < len(valSet.Validators) { + newValidators = append(newValidators, valSet.Validators[idx+1:]...) + } + valSet.Validators = newValidators + // Invalidate cache + valSet.Proposer = nil + valSet.totalVotingPower = 0 + return removedVal, true } func (valSet *ValidatorSet) Iterate(fn func(index int, val *Validator) bool) { @@ -266,10 +263,9 @@ func (valSet *ValidatorSet) VerifyCommit(chainID string, blockID BlockID, height if talliedVotingPower > valSet.TotalVotingPower()*2/3 { return nil - } else { - return fmt.Errorf("Invalid commit -- insufficient voting power: got %v, needed %v", - talliedVotingPower, (valSet.TotalVotingPower()*2/3 + 1)) } + return fmt.Errorf("Invalid commit -- insufficient voting power: got %v, needed %v", + talliedVotingPower, (valSet.TotalVotingPower()*2/3 + 1)) } // VerifyCommitAny will check to see if the set would @@ -472,9 +468,8 @@ func safeMulClip(a, b int64) int64 { if overflow { if (a < 0 || b < 0) && !(a < 0 && b < 0) { return math.MinInt64 - } else { - return math.MaxInt64 } + return math.MaxInt64 } return c } @@ -484,9 +479,8 @@ func safeAddClip(a, b int64) int64 { if overflow { if b < 0 { return math.MinInt64 - } else { - return math.MaxInt64 } + return math.MaxInt64 } return c } @@ -496,9 +490,8 @@ func safeSubClip(a, b int64) int64 { if overflow { if b > 0 { return math.MinInt64 - } else { - return math.MaxInt64 } + return math.MaxInt64 } return c } diff --git a/types/validator_set_test.go b/types/validator_set_test.go index b346be1be..8769ac5ff 100644 --- a/types/validator_set_test.go +++ b/types/validator_set_test.go @@ -127,7 +127,7 @@ func TestProposerSelection2(t *testing.T) { for i := 0; i < 120*N; i++ { prop := vals.GetProposer() ii := prop.Address[19] - propCount[ii] += 1 + propCount[ii]++ vals.IncrementAccum(1) } diff --git a/types/vote_set.go b/types/vote_set.go index 37f26f4a5..e255488d6 100644 --- a/types/vote_set.go +++ b/types/vote_set.go @@ -94,33 +94,29 @@ func (voteSet *VoteSet) ChainID() string { func (voteSet *VoteSet) Height() int64 { if voteSet == nil { return 0 - } else { - return voteSet.height } + return voteSet.height } func (voteSet *VoteSet) Round() int { if voteSet == nil { return -1 - } else { - return voteSet.round } + return voteSet.round } func (voteSet *VoteSet) Type() byte { if voteSet == nil { return 0x00 - } else { - return voteSet.type_ } + return voteSet.type_ } func (voteSet *VoteSet) Size() int { if voteSet == nil { return 0 - } else { - return voteSet.valSet.Size() } + return voteSet.valSet.Size() } // Returns added=true if vote is valid and new. @@ -185,9 +181,8 @@ func (voteSet *VoteSet) addVote(vote *Vote) (added bool, err error) { if existing, ok := voteSet.getVote(valIndex, blockKey); ok { if existing.Signature.Equals(vote.Signature) { return false, nil // duplicate - } else { - return false, errors.Wrapf(ErrVoteNonDeterministicSignature, "Existing vote: %v; New vote: %v", existing, vote) } + return false, errors.Wrapf(ErrVoteNonDeterministicSignature, "Existing vote: %v; New vote: %v", existing, vote) } // Check signature. @@ -199,13 +194,11 @@ func (voteSet *VoteSet) addVote(vote *Vote) (added bool, err error) { added, conflicting := voteSet.addVerifiedVote(vote, blockKey, val.VotingPower) if conflicting != nil { return added, NewConflictingVoteError(val, conflicting, vote) - } else { - if !added { - cmn.PanicSanity("Expected to add non-conflicting vote") - } - return added, nil } - + if !added { + cmn.PanicSanity("Expected to add non-conflicting vote") + } + return added, nil } // Returns (vote, true) if vote exists for valIndex and blockKey @@ -257,13 +250,12 @@ func (voteSet *VoteSet) addVerifiedVote(vote *Vote, blockKey string, votingPower // ... and there's a conflicting vote. // We're not even tracking this blockKey, so just forget it. return false, conflicting - } else { - // ... and there's no conflicting vote. - // Start tracking this blockKey - votesByBlock = newBlockVotes(false, voteSet.valSet.Size()) - voteSet.votesByBlock[blockKey] = votesByBlock - // We'll add the vote in a bit. } + // ... and there's no conflicting vote. + // Start tracking this blockKey + votesByBlock = newBlockVotes(false, voteSet.valSet.Size()) + voteSet.votesByBlock[blockKey] = votesByBlock + // We'll add the vote in a bit. } // Before adding to votesByBlock, see if we'll exceed quorum @@ -309,10 +301,9 @@ func (voteSet *VoteSet) SetPeerMaj23(peerID P2PID, blockID BlockID) error { if existing, ok := voteSet.peerMaj23s[peerID]; ok { if existing.Equals(blockID) { return nil // Nothing to do - } else { - return fmt.Errorf("SetPeerMaj23: Received conflicting blockID from peer %v. Got %v, expected %v", - peerID, blockID, existing) } + return fmt.Errorf("SetPeerMaj23: Received conflicting blockID from peer %v. Got %v, expected %v", + peerID, blockID, existing) } voteSet.peerMaj23s[peerID] = blockID @@ -321,10 +312,9 @@ func (voteSet *VoteSet) SetPeerMaj23(peerID P2PID, blockID BlockID) error { if ok { if votesByBlock.peerMaj23 { return nil // Nothing to do - } else { - votesByBlock.peerMaj23 = true - // No need to copy votes, already there. } + votesByBlock.peerMaj23 = true + // No need to copy votes, already there. } else { votesByBlock = newBlockVotes(true, voteSet.valSet.Size()) voteSet.votesByBlock[blockKey] = votesByBlock @@ -422,9 +412,8 @@ func (voteSet *VoteSet) TwoThirdsMajority() (blockID BlockID, ok bool) { defer voteSet.mtx.Unlock() if voteSet.maj23 != nil { return *voteSet.maj23, true - } else { - return BlockID{}, false } + return BlockID{}, false } func (voteSet *VoteSet) String() string { From 70e7454c218bccebb6c5e125e34288efd4d09626 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 27 Mar 2018 11:44:53 -0400 Subject: [PATCH 15/54] comment out test_libs because of gcc dep in tmlibs --- .circleci/config.yml | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cb2577908..11ad5f33f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -130,18 +130,18 @@ jobs: paths: - "profiles/*" - test_libs: - <<: *defaults - steps: - - attach_workspace: - at: /tmp/workspace - - restore_cache: - key: v1-pkg-cache - - restore_cache: - key: v1-tree-{{ .Environment.CIRCLE_SHA1 }} - - run: - name: Run tests - command: bash test/test_libs.sh + #test_libs: + # <<: *defaults + # steps: + # - attach_workspace: + # at: /tmp/workspace + # - restore_cache: + # key: v1-pkg-cache + # - restore_cache: + # key: v1-tree-{{ .Environment.CIRCLE_SHA1 }} + # - run: + # name: Run tests + # command: bash test/test_libs.sh test_persistence: <<: *defaults @@ -205,14 +205,14 @@ workflows: - test_cover: requires: - setup_dependencies - - test_libs: - filters: - branches: - only: - - develop - - master - requires: - - setup_dependencies + #- test_libs: + #- filters: + #- branches: + #- only: + #- - develop + #- - master + #- requires: + #- - setup_dependencies - test_persistence: requires: - setup_abci From 5243e54641dc2437a6aac0f379d4f234948bcbf8 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 2 Apr 2018 11:23:18 +0200 Subject: [PATCH 16/54] [codecov] ignore docs, scripts and DOCKER dirs --- codecov.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/codecov.yml b/codecov.yml index 6db0f15d5..b190853de 100644 --- a/codecov.yml +++ b/codecov.yml @@ -16,3 +16,8 @@ comment: require_changes: no require_base: no require_head: yes + +ignore: + - "docs" + - "DOCKER" + - "scripts" From 9c757108ca8bc849a04c23e6800e53155863e933 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 2 Apr 2018 11:29:03 +0200 Subject: [PATCH 17/54] [test] remove test_libs Reasons: 1) all deps we're using should be passing tests (including external) 2) deps can require complicated setup for testing 3) the person responsible for releasing Tendermint should be cautious when updating a dep --- .circleci/config.yml | 21 --------------------- Makefile | 7 +------ scripts/dep_utils/checkout.sh | 35 ----------------------------------- test/README.md | 4 ---- test/test_libs.sh | 34 ---------------------------------- 5 files changed, 1 insertion(+), 100 deletions(-) delete mode 100644 scripts/dep_utils/checkout.sh delete mode 100644 test/test_libs.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 11ad5f33f..8dac95a8e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -130,19 +130,6 @@ jobs: paths: - "profiles/*" - #test_libs: - # <<: *defaults - # steps: - # - attach_workspace: - # at: /tmp/workspace - # - restore_cache: - # key: v1-pkg-cache - # - restore_cache: - # key: v1-tree-{{ .Environment.CIRCLE_SHA1 }} - # - run: - # name: Run tests - # command: bash test/test_libs.sh - test_persistence: <<: *defaults steps: @@ -205,14 +192,6 @@ workflows: - test_cover: requires: - setup_dependencies - #- test_libs: - #- filters: - #- branches: - #- only: - #- - develop - #- - master - #- requires: - #- - setup_dependencies - test_persistence: requires: - setup_abci diff --git a/Makefile b/Makefile index c19ce11a6..3e51c834e 100644 --- a/Makefile +++ b/Makefile @@ -119,11 +119,6 @@ test_integrations: make test_persistence make test_p2p -test_libs: - # checkout every github.com/tendermint dir and run its tests - # NOTE: on release-* or master branches only (set by Jenkins) - docker run --name run_libs -t tester bash test/test_libs.sh - test_release: @go test -tags release $(PACKAGES) @@ -186,4 +181,4 @@ metalinter_all: # To avoid unintended conflicts with file names, always add to .PHONY # unless there is a reason not to. # https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html -.PHONY: check build build_race dist install check_tools get_tools update_tools get_vendor_deps draw_deps test_cover test_apps test_persistence test_p2p test test_race test_libs test_integrations test_release test100 vagrant_test fmt +.PHONY: check build build_race dist install check_tools get_tools update_tools get_vendor_deps draw_deps test_cover test_apps test_persistence test_p2p test test_race test_integrations test_release test100 vagrant_test fmt diff --git a/scripts/dep_utils/checkout.sh b/scripts/dep_utils/checkout.sh deleted file mode 100644 index 1d35e97ab..000000000 --- a/scripts/dep_utils/checkout.sh +++ /dev/null @@ -1,35 +0,0 @@ -#! /bin/bash - -set -ex - -set +u -if [[ "$DEP" == "" ]]; then - DEP=$GOPATH/src/github.com/tendermint/tendermint/Gopkg.lock -fi -set -u - - -set -u - -function getVendoredVersion() { - grep -A100 "$LIB" "$DEP" | grep revision | head -n1 | grep -o '"[^"]\+"' | cut -d '"' -f 2 -} - - -# fetch and checkout vendored dep - -lib=$1 - -echo "----------------------------------" -echo "Getting $lib ..." -go get -t "github.com/tendermint/$lib/..." - -VENDORED=$(getVendoredVersion "$lib") -cd "$GOPATH/src/github.com/tendermint/$lib" || exit -MASTER=$(git rev-parse origin/master) - -if [[ "$VENDORED" != "$MASTER" ]]; then - echo "... VENDORED != MASTER ($VENDORED != $MASTER)" - echo "... Checking out commit $VENDORED" - git checkout "$VENDORED" &> /dev/null -fi diff --git a/test/README.md b/test/README.md index d9f09fef7..ee53905bf 100644 --- a/test/README.md +++ b/test/README.md @@ -19,7 +19,3 @@ and run the following tests in docker containers: - send a tx on each node and ensure the state root is updated on all of them - crash and restart nodes one at a time and ensure they can sync back up (via fastsync) - crash and restart all nodes at once and ensure they can sync back up - -If on a `release-x.x.x` branch, we also run - -- `go test` for all our dependency libs (test/test_libs.sh) diff --git a/test/test_libs.sh b/test/test_libs.sh deleted file mode 100644 index 893bf35dc..000000000 --- a/test/test_libs.sh +++ /dev/null @@ -1,34 +0,0 @@ -#! /bin/bash -set -ex - -export PATH="$GOBIN:$PATH" - -# Get the parent directory of where this script is. -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done -DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )" - -#################### -# libs we depend on -#################### - -# All libs should define `make test` and `make get_vendor_deps` -LIBS=(tmlibs go-wire go-crypto abci) -for lib in "${LIBS[@]}"; do - # checkout vendored version of lib - bash scripts/dep_utils/checkout.sh "$lib" - - echo "Testing $lib ..." - cd "$GOPATH/src/github.com/tendermint/$lib" - make get_tools - make get_vendor_deps - make test - if [[ "$?" != 0 ]]; then - echo "FAIL" - exit 1 - fi - cd "$DIR" -done - -echo "" -echo "PASS" From ecdc1b9bb07329b31ff999dce5424c3c406caeb0 Mon Sep 17 00:00:00 2001 From: Zaki Manian Date: Mon, 2 Apr 2018 02:36:09 -0700 Subject: [PATCH 18/54] Add a method for creating an https server (#1403) --- rpc/lib/server/http_server.go | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/rpc/lib/server/http_server.go b/rpc/lib/server/http_server.go index 515baf5dd..ce7673644 100644 --- a/rpc/lib/server/http_server.go +++ b/rpc/lib/server/http_server.go @@ -48,6 +48,39 @@ func StartHTTPServer(listenAddr string, handler http.Handler, logger log.Logger) return listener, nil } +func StartHTTPAndTLSServer(listenAddr string, handler http.Handler, cert_path string, key_path string, logger log.Logger) (listener net.Listener, err error) { + // listenAddr should be fully formed including tcp:// or unix:// prefix + var proto, addr string + parts := strings.SplitN(listenAddr, "://", 2) + if len(parts) != 2 { + logger.Error("WARNING (tendermint/rpc/lib): Please use fully formed listening addresses, including the tcp:// or unix:// prefix") + // we used to allow addrs without tcp/unix prefix by checking for a colon + // TODO: Deprecate + proto = types.SocketType(listenAddr) + addr = listenAddr + // return nil, errors.Errorf("Invalid listener address %s", lisenAddr) + } else { + proto, addr = parts[0], parts[1] + } + + logger.Info(fmt.Sprintf("Starting RPC HTTPS server on %s socket %v", proto, addr)) + listener, err = net.Listen(proto, addr) + if err != nil { + return nil, errors.Errorf("Failed to listen to %v: %v", listenAddr, err) + } + + go func() { + res := http.ServeTLS( + listener, + RecoverAndLogHandler(handler, logger), + cert_path, + key_path, + ) + logger.Error("RPC HTTPS server stopped", "result", res) + }() + return listener, nil +} + func WriteRPCResponseHTTPError(w http.ResponseWriter, httpCode int, res types.RPCResponse) { jsonBytes, err := json.MarshalIndent(res, "", " ") if err != nil { From d694d47d2245c10ef1788abacb2a206b7a32157d Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 2 Apr 2018 11:40:47 +0200 Subject: [PATCH 19/54] [rpc/lib] rename vars according to Go conventions --- rpc/lib/server/http_server.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rpc/lib/server/http_server.go b/rpc/lib/server/http_server.go index ce7673644..df4ab8a10 100644 --- a/rpc/lib/server/http_server.go +++ b/rpc/lib/server/http_server.go @@ -48,7 +48,7 @@ func StartHTTPServer(listenAddr string, handler http.Handler, logger log.Logger) return listener, nil } -func StartHTTPAndTLSServer(listenAddr string, handler http.Handler, cert_path string, key_path string, logger log.Logger) (listener net.Listener, err error) { +func StartHTTPAndTLSServer(listenAddr string, handler http.Handler, certFile, keyFile string, logger log.Logger) (listener net.Listener, err error) { // listenAddr should be fully formed including tcp:// or unix:// prefix var proto, addr string parts := strings.SplitN(listenAddr, "://", 2) @@ -73,8 +73,8 @@ func StartHTTPAndTLSServer(listenAddr string, handler http.Handler, cert_path st res := http.ServeTLS( listener, RecoverAndLogHandler(handler, logger), - cert_path, - key_path, + certFile, + keyFile, ) logger.Error("RPC HTTPS server stopped", "result", res) }() From 5ef8a6e887580b60a34217816fb165e37fd7857b Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 2 Apr 2018 11:47:04 +0200 Subject: [PATCH 20/54] deprecate not fully formed addresses --- rpc/lib/server/http_server.go | 34 ++++++++++------------------------ 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/rpc/lib/server/http_server.go b/rpc/lib/server/http_server.go index df4ab8a10..01ad0809d 100644 --- a/rpc/lib/server/http_server.go +++ b/rpc/lib/server/http_server.go @@ -18,65 +18,51 @@ import ( ) func StartHTTPServer(listenAddr string, handler http.Handler, logger log.Logger) (listener net.Listener, err error) { - // listenAddr should be fully formed including tcp:// or unix:// prefix var proto, addr string parts := strings.SplitN(listenAddr, "://", 2) if len(parts) != 2 { - logger.Error("WARNING (tendermint/rpc/lib): Please use fully formed listening addresses, including the tcp:// or unix:// prefix") - // we used to allow addrs without tcp/unix prefix by checking for a colon - // TODO: Deprecate - proto = types.SocketType(listenAddr) - addr = listenAddr - // return nil, errors.Errorf("Invalid listener address %s", lisenAddr) - } else { - proto, addr = parts[0], parts[1] + return nil, errors.Errorf("Invalid listening address %s (use fully formed addresses, including the tcp:// or unix:// prefix)", listenAddr) } + proto, addr = parts[0], parts[1] logger.Info(fmt.Sprintf("Starting RPC HTTP server on %s socket %v", proto, addr)) listener, err = net.Listen(proto, addr) if err != nil { - return nil, errors.Errorf("Failed to listen to %v: %v", listenAddr, err) + return nil, errors.Errorf("Failed to listen on %v: %v", listenAddr, err) } go func() { - res := http.Serve( + err := http.Serve( listener, RecoverAndLogHandler(handler, logger), ) - logger.Error("RPC HTTP server stopped", "result", res) + logger.Error("RPC HTTP server stopped", "err", err) }() return listener, nil } func StartHTTPAndTLSServer(listenAddr string, handler http.Handler, certFile, keyFile string, logger log.Logger) (listener net.Listener, err error) { - // listenAddr should be fully formed including tcp:// or unix:// prefix var proto, addr string parts := strings.SplitN(listenAddr, "://", 2) if len(parts) != 2 { - logger.Error("WARNING (tendermint/rpc/lib): Please use fully formed listening addresses, including the tcp:// or unix:// prefix") - // we used to allow addrs without tcp/unix prefix by checking for a colon - // TODO: Deprecate - proto = types.SocketType(listenAddr) - addr = listenAddr - // return nil, errors.Errorf("Invalid listener address %s", lisenAddr) - } else { - proto, addr = parts[0], parts[1] + return nil, errors.Errorf("Invalid listening address %s (use fully formed addresses, including the tcp:// or unix:// prefix)", listenAddr) } + proto, addr = parts[0], parts[1] logger.Info(fmt.Sprintf("Starting RPC HTTPS server on %s socket %v", proto, addr)) listener, err = net.Listen(proto, addr) if err != nil { - return nil, errors.Errorf("Failed to listen to %v: %v", listenAddr, err) + return nil, errors.Errorf("Failed to listen on %v: %v", listenAddr, err) } go func() { - res := http.ServeTLS( + err := http.ServeTLS( listener, RecoverAndLogHandler(handler, logger), certFile, keyFile, ) - logger.Error("RPC HTTPS server stopped", "result", res) + logger.Error("RPC HTTPS server stopped", "err", err) }() return listener, nil } From 491c8ab4c1e867c160b2a550356cfb18f9e2f047 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 2 Apr 2018 11:55:55 +0200 Subject: [PATCH 21/54] [rpc/lib] log cert and key files in StartHTTPAndTLSServer --- rpc/lib/server/http_server.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rpc/lib/server/http_server.go b/rpc/lib/server/http_server.go index 01ad0809d..3f54c61ef 100644 --- a/rpc/lib/server/http_server.go +++ b/rpc/lib/server/http_server.go @@ -25,7 +25,7 @@ func StartHTTPServer(listenAddr string, handler http.Handler, logger log.Logger) } proto, addr = parts[0], parts[1] - logger.Info(fmt.Sprintf("Starting RPC HTTP server on %s socket %v", proto, addr)) + logger.Info(fmt.Sprintf("Starting RPC HTTP server on %s", listenAddr)) listener, err = net.Listen(proto, addr) if err != nil { return nil, errors.Errorf("Failed to listen on %v: %v", listenAddr, err) @@ -49,7 +49,7 @@ func StartHTTPAndTLSServer(listenAddr string, handler http.Handler, certFile, ke } proto, addr = parts[0], parts[1] - logger.Info(fmt.Sprintf("Starting RPC HTTPS server on %s socket %v", proto, addr)) + logger.Info(fmt.Sprintf("Starting RPC HTTPS server on %s (cert: %q, key: %q)", listenAddr, certFile, keyFile)) listener, err = net.Listen(proto, addr) if err != nil { return nil, errors.Errorf("Failed to listen on %v: %v", listenAddr, err) From 641476d40f6e8a51013b10809ff3fbc3edf0ec90 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 2 Apr 2018 16:55:43 +0200 Subject: [PATCH 22/54] update docker to use alpine 3.7 --- DOCKER/Dockerfile | 2 +- DOCKER/Dockerfile.develop | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/DOCKER/Dockerfile b/DOCKER/Dockerfile index c00318fba..9934ff722 100644 --- a/DOCKER/Dockerfile +++ b/DOCKER/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.6 +FROM alpine:3.7 # This is the release of tendermint to pull in. ENV TM_VERSION 0.15.0 diff --git a/DOCKER/Dockerfile.develop b/DOCKER/Dockerfile.develop index e6ee6607e..4f8ae0b2e 100644 --- a/DOCKER/Dockerfile.develop +++ b/DOCKER/Dockerfile.develop @@ -1,4 +1,4 @@ -FROM alpine:3.6 +FROM alpine:3.7 ENV DATA_ROOT /tendermint ENV TMHOME $DATA_ROOT From 208ac32fa266657bd6c304e84ec828aa252bb0b8 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 2 Apr 2018 16:56:07 +0200 Subject: [PATCH 23/54] update Dockerfile to point to 0.17.1 release --- DOCKER/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DOCKER/Dockerfile b/DOCKER/Dockerfile index 9934ff722..cf45ac2f1 100644 --- a/DOCKER/Dockerfile +++ b/DOCKER/Dockerfile @@ -1,8 +1,8 @@ FROM alpine:3.7 # This is the release of tendermint to pull in. -ENV TM_VERSION 0.15.0 -ENV TM_SHA256SUM 71cc271c67eca506ca492c8b90b090132f104bf5dbfe0af2702a50886e88de17 +ENV TM_VERSION 0.17.1 +ENV TM_SHA256SUM d57008c63d2d9176861137e38ed203da486febf20ae7d388fb810a75afff8f24 # Tendermint will be looking for genesis file in /tendermint (unless you change # `genesis_file` in config.toml). You can put your config.toml and private @@ -26,7 +26,7 @@ RUN mkdir -p $DATA_ROOT && \ RUN apk add --no-cache bash curl jq RUN apk add --no-cache openssl && \ - wget https://s3-us-west-2.amazonaws.com/tendermint/binaries/tendermint/v${TM_VERSION}/tendermint_${TM_VERSION}_linux_amd64.zip && \ + wget https://github.com/tendermint/tendermint/releases/download/v${TM_VERSION}/tendermint_${TM_VERSION}_linux_amd64.zip && \ echo "${TM_SHA256SUM} tendermint_${TM_VERSION}_linux_amd64.zip" | sha256sum -c && \ unzip -d /bin tendermint_${TM_VERSION}_linux_amd64.zip && \ apk del openssl && \ From b800b4ec1d389c9891ce65f2c0633a1fad6f2af1 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 2 Apr 2018 16:57:25 +0200 Subject: [PATCH 24/54] update docker readme --- DOCKER/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DOCKER/README.md b/DOCKER/README.md index 24cd38a51..2fe3db866 100644 --- a/DOCKER/README.md +++ b/DOCKER/README.md @@ -1,6 +1,7 @@ # Supported tags and respective `Dockerfile` links -- `0.15.0`, `latest` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/170777300ea92dc21a8aec1abc16cb51812513a4/DOCKER/Dockerfile) +- `0.17.1`, `latest` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/208ac32fa266657bd6c304e84ec828aa252bb0b8/DOCKER/Dockerfile) +- `0.15.0` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/170777300ea92dc21a8aec1abc16cb51812513a4/DOCKER/Dockerfile) - `0.13.0` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/a28b3fff49dce2fb31f90abb2fc693834e0029c2/DOCKER/Dockerfile) - `0.12.1` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/457c688346b565e90735431619ca3ca597ef9007/DOCKER/Dockerfile) - `0.12.0` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/70d8afa6e952e24c573ece345560a5971bf2cc0e/DOCKER/Dockerfile) From 315c475b791c9c48a35f52f79fef6afc2208138d Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Tue, 3 Apr 2018 04:48:40 -0700 Subject: [PATCH 25/54] docs: build updates ref: https://github.com/tendermint/tools/pull/79 --- .gitignore | 1 + docs/conf.py | 5 ++--- docs/index.rst | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 7b7b746fb..e76fb1fc5 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ test/logs coverage.txt docs/_build docs/tools +docs/abci-spec.rst *.log scripts/wal2json/wal2json diff --git a/docs/conf.py b/docs/conf.py index 6122e90a3..9d4f6b017 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -196,9 +196,8 @@ urllib.urlretrieve(tools_repo+tools_branch+'/mintnet-kubernetes/assets/statefuls urllib.urlretrieve(tools_repo+tools_branch+'/mintnet-kubernetes/assets/t_plus_k.png', filename=assets_dir+'/t_plus_k.png') urllib.urlretrieve(tools_repo+tools_branch+'/terraform-digitalocean/README.rst', filename=tools_dir+'/terraform-digitalocean.rst') -urllib.urlretrieve(tools_repo+tools_branch+'/tm-bench/README.rst', filename=tools_dir+'/benchmarking-and-monitoring.rst') -# the readme for below is included in tm-bench -# urllib.urlretrieve('https://raw.githubusercontent.com/tendermint/tools/master/tm-monitor/README.rst', filename='tools/tm-monitor.rst') +urllib.urlretrieve(tools_repo+tools_branch+'/tm-bench/README.rst', filename=tools_dir+'/benchmarking.rst') +urllib.urlretrieve('https://raw.githubusercontent.com/tendermint/tools/master/tm-monitor/README.rst', filename='tools/monitoring.rst') #### abci spec ################################# diff --git a/docs/index.rst b/docs/index.rst index f2f70d194..a89adb296 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -44,7 +44,8 @@ Tendermint Tools tools/docker.rst tools/mintnet-kubernetes.rst tools/terraform-digitalocean.rst - tools/benchmarking-and-monitoring.rst + tools/benchmarking.rst + tools/monitoring.rst Tendermint 102 -------------- From 47b8bd1728c0f88eff57249223a87ddcda5fbaf4 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 27 Mar 2018 18:18:24 +0200 Subject: [PATCH 26/54] wrote a test for EventBus#PublishEventTx Refs #1369 --- types/event_bus_test.go | 49 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/types/event_bus_test.go b/types/event_bus_test.go index aa97092f6..c1b4c1b0f 100644 --- a/types/event_bus_test.go +++ b/types/event_bus_test.go @@ -7,9 +7,58 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + abci "github.com/tendermint/abci/types" + cmn "github.com/tendermint/tmlibs/common" tmpubsub "github.com/tendermint/tmlibs/pubsub" + tmquery "github.com/tendermint/tmlibs/pubsub/query" ) +func TestEventBusPublishEventTx(t *testing.T) { + eventBus := NewEventBus() + err := eventBus.Start() + require.NoError(t, err) + defer eventBus.Stop() + + tx := Tx("foo") + result := abci.ResponseDeliverTx{Data: []byte("bar"), Tags: []cmn.KVPair{}, Fee: cmn.KI64Pair{Key: []uint8{}, Value: 0}} + + txEventsCh := make(chan interface{}) + + // PublishEventTx adds all these 3 tags, so the query below should work + query := fmt.Sprintf("tm.event='Tx' AND tx.height=1 AND tx.hash='%X'", tx.Hash()) + err = eventBus.Subscribe(context.Background(), "test", tmquery.MustParse(query), txEventsCh) + require.NoError(t, err) + + done := make(chan struct{}) + go func() { + for e := range txEventsCh { + edt := e.(TMEventData).Unwrap().(EventDataTx) + assert.Equal(t, int64(1), edt.Height) + assert.Equal(t, uint32(0), edt.Index) + assert.Equal(t, tx, edt.Tx) + assert.Equal(t, result, edt.Result) + close(done) + } + }() + + err = eventBus.PublishEventTx(EventDataTx{TxResult{ + Height: 1, + Index: 0, + Tx: tx, + Result: result, + }}) + assert.NoError(t, err) + + select { + case <-done: + case <-time.After(1 * time.Second): + t.Fatal("did not receive a transaction after 1 sec.") + } +} + func BenchmarkEventBus(b *testing.B) { benchmarks := []struct { name string From 8462493cbfefbc0222dc26be892eb39e79072691 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 27 Mar 2018 19:00:24 +0200 Subject: [PATCH 27/54] [rpc] fix subscribing using an abci.ResponseDeliverTx tag Refs #1369 --- CHANGELOG.md | 4 +++- types/event_bus.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47b39b23a..e7c61489c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,13 +27,15 @@ BUG FIXES: ## 0.17.2 (TBD) +BUG FIXES: +- [rpc] fix subscribing using an abci.ResponseDeliverTx tag + IMPROVEMENTS: - [rpc] `/tx` and `/tx_search` responses now include the transaction hash ## 0.17.1 (March 27th, 2018) BUG FIXES: - - [types] Actually support `app_state` in genesis as `AppStateJSON` ## 0.17.0 (March 27th, 2018) diff --git a/types/event_bus.go b/types/event_bus.go index 4edaea588..37bd5619d 100644 --- a/types/event_bus.go +++ b/types/event_bus.go @@ -101,7 +101,7 @@ func (b *EventBus) PublishEventTx(event EventDataTx) error { b.Logger.Info("Got tag with an empty key (skipping)", "tag", tag, "tx", event.Tx) continue } - tags[string(tag.Key)] = tag.Value + tags[string(tag.Key)] = string(tag.Value) } // add predefined tags From 39ff4d22e93495e11df6f9f75b0e1335590ef744 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 3 Apr 2018 22:34:18 +0300 Subject: [PATCH 28/54] minor cleanup --- lite/proxy/validate_test.go | 55 +++++++++++++++++-------------------- types/block.go | 4 ++- 2 files changed, 28 insertions(+), 31 deletions(-) diff --git a/lite/proxy/validate_test.go b/lite/proxy/validate_test.go index bd57994d7..782a6aabb 100644 --- a/lite/proxy/validate_test.go +++ b/lite/proxy/validate_test.go @@ -12,14 +12,15 @@ import ( ) var ( - deadBeefTxs = types.Txs{[]byte("DE"), []byte("AD"), []byte("BE"), []byte("EF")} - - deadBeefRipEmd160Hash = deadBeefTxs.Hash() + deadBeefTxs = types.Txs{[]byte("DE"), []byte("AD"), []byte("BE"), []byte("EF")} + deadBeefHash = deadBeefTxs.Hash() + testTime1 = time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC) + testTime2 = time.Date(2017, 1, 2, 1, 1, 1, 1, time.UTC) ) -var hdrHeight11Tendermint = &types.Header{ +var hdrHeight11 = &types.Header{ Height: 11, - Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), + Time: testTime1, ValidatorsHash: []byte("Tendermint"), } @@ -54,14 +55,14 @@ func TestValidateBlock(t *testing.T) { // Start Header.Hash mismatch test { - block: &types.Block{Header: hdrHeight11Tendermint}, + block: &types.Block{Header: hdrHeight11}, commit: lite.Commit{Header: &types.Header{Height: 11}}, wantErr: "Headers don't match", }, { - block: &types.Block{Header: hdrHeight11Tendermint}, - commit: lite.Commit{Header: hdrHeight11Tendermint}, + block: &types.Block{Header: hdrHeight11}, + commit: lite.Commit{Header: hdrHeight11}, }, // End Header.Hash mismatch test @@ -79,7 +80,7 @@ func TestValidateBlock(t *testing.T) { }, { block: &types.Block{ - Header: &types.Header{Height: 11, DataHash: deadBeefRipEmd160Hash}, + Header: &types.Header{Height: 11, DataHash: deadBeefHash}, Data: &types.Data{Txs: deadBeefTxs}, }, commit: lite.Commit{ @@ -136,33 +137,27 @@ func TestValidateBlockMeta(t *testing.T) { // Start Headers don't match test { - meta: &types.BlockMeta{Header: hdrHeight11Tendermint}, + meta: &types.BlockMeta{Header: hdrHeight11}, commit: lite.Commit{Header: &types.Header{Height: 11}}, wantErr: "Headers don't match", }, { - meta: &types.BlockMeta{Header: hdrHeight11Tendermint}, - commit: lite.Commit{Header: hdrHeight11Tendermint}, + meta: &types.BlockMeta{Header: hdrHeight11}, + commit: lite.Commit{Header: hdrHeight11}, }, { meta: &types.BlockMeta{ Header: &types.Header{ - Height: 11, - // TODO: (@odeke-em) inquire why ValidatorsHash has to be non-blank - // for the Header to be hashed. Perhaps this is a security hole because - // an aggressor could perhaps pass in headers that don't have - // ValidatorsHash set and we won't be able to validate blocks. + Height: 11, ValidatorsHash: []byte("lite-test"), - // TODO: (@odeke-em) file an issue with Tendermint to get them to update - // to the latest go-wire, then no more need for this value fill to avoid - // the time zero value of less than 1970. - Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC), + // TODO: should be able to use empty time after Amino upgrade + Time: testTime1, }, }, commit: lite.Commit{ - Header: &types.Header{Height: 11, DataHash: deadBeefRipEmd160Hash}, + Header: &types.Header{Height: 11, DataHash: deadBeefHash}, }, wantErr: "Headers don't match", }, @@ -170,16 +165,16 @@ func TestValidateBlockMeta(t *testing.T) { { meta: &types.BlockMeta{ Header: &types.Header{ - Height: 11, DataHash: deadBeefRipEmd160Hash, + Height: 11, DataHash: deadBeefHash, ValidatorsHash: []byte("Tendermint"), - Time: time.Date(2017, 1, 2, 1, 1, 1, 1, time.UTC), + Time: testTime1, }, }, commit: lite.Commit{ Header: &types.Header{ - Height: 11, DataHash: deadBeefRipEmd160Hash, + Height: 11, DataHash: deadBeefHash, ValidatorsHash: []byte("Tendermint"), - Time: time.Date(2018, 1, 2, 1, 1, 1, 1, time.UTC), + Time: testTime2, }, Commit: &types.Commit{BlockID: types.BlockID{Hash: []byte("DEADBEEF")}}, }, @@ -189,16 +184,16 @@ func TestValidateBlockMeta(t *testing.T) { { meta: &types.BlockMeta{ Header: &types.Header{ - Height: 11, DataHash: deadBeefRipEmd160Hash, + Height: 11, DataHash: deadBeefHash, ValidatorsHash: []byte("Tendermint"), - Time: time.Date(2017, 1, 2, 1, 1, 1, 1, time.UTC), + Time: testTime2, }, }, commit: lite.Commit{ Header: &types.Header{ - Height: 11, DataHash: deadBeefRipEmd160Hash, + Height: 11, DataHash: deadBeefHash, ValidatorsHash: []byte("Tendermint-x"), - Time: time.Date(2017, 1, 2, 1, 1, 1, 1, time.UTC), + Time: testTime2, }, Commit: &types.Commit{BlockID: types.BlockID{Hash: []byte("DEADBEEF")}}, }, diff --git a/types/block.go b/types/block.go index 970ca36f4..774c69f32 100644 --- a/types/block.go +++ b/types/block.go @@ -177,7 +177,9 @@ type Header struct { } // Hash returns the hash of the header. -// Returns nil if ValidatorHash is missing. +// Returns nil if ValidatorHash is missing, +// since a Header is not valid unless there is +// a ValidaotrsHash (corresponding to the validator set). func (h *Header) Hash() cmn.HexBytes { if h == nil || len(h.ValidatorsHash) == 0 { return nil From 5f548c7679dbc4c9ab0ff6ead0dacc77bbf719db Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 3 Apr 2018 22:56:46 +0300 Subject: [PATCH 29/54] consensus: close pubsub channels. fixes #1372 --- consensus/reactor.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/consensus/reactor.go b/consensus/reactor.go index b13be0e87..c71f9e0c4 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -371,19 +371,21 @@ func (conR *ConsensusReactor) startBroadcastRoutine() error { } go func() { + var data interface{} + var ok bool for { select { - case data, ok := <-stepsCh: + case data, ok = <-stepsCh: if ok { // a receive from a closed channel returns the zero value immediately edrs := data.(types.TMEventData).Unwrap().(types.EventDataRoundState) conR.broadcastNewRoundStep(edrs.RoundState.(*cstypes.RoundState)) } - case data, ok := <-votesCh: + case data, ok = <-votesCh: if ok { edv := data.(types.TMEventData).Unwrap().(types.EventDataVote) conR.broadcastHasVoteMessage(edv.Vote) } - case data, ok := <-heartbeatsCh: + case data, ok = <-heartbeatsCh: if ok { edph := data.(types.TMEventData).Unwrap().(types.EventDataProposalHeartbeat) conR.broadcastProposalHeartbeatMessage(edph) @@ -392,6 +394,10 @@ func (conR *ConsensusReactor) startBroadcastRoutine() error { conR.eventBus.UnsubscribeAll(ctx, subscriber) return } + if !ok { + conR.eventBus.UnsubscribeAll(ctx, subscriber) + return + } } }() From 37ce6b195a2c26748bb7cc61466116748a589da0 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 29 Mar 2018 13:32:14 +0200 Subject: [PATCH 30/54] ValidatorSet#GetByAddress: return -1 if no validator was found --- consensus/state.go | 6 +----- types/validator_set.go | 5 ++++- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/consensus/state.go b/consensus/state.go index 6acad698b..57c7b32f0 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -720,11 +720,7 @@ func (cs *ConsensusState) needProofBlock(height int64) bool { func (cs *ConsensusState) proposalHeartbeat(height int64, round int) { counter := 0 addr := cs.privValidator.GetAddress() - valIndex, v := cs.Validators.GetByAddress(addr) - if v == nil { - // not a validator - valIndex = -1 - } + valIndex, _ := cs.Validators.GetByAddress(addr) chainID := cs.state.ChainID for { rs := cs.GetRoundState() diff --git a/types/validator_set.go b/types/validator_set.go index dc6b66d68..ce32772da 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -90,14 +90,17 @@ func (valSet *ValidatorSet) HasAddress(address []byte) bool { return idx != len(valSet.Validators) && bytes.Equal(valSet.Validators[idx].Address, address) } +// GetByAddress returns an index of the validator with address and validator +// itself if found. Otherwise, -1 and nil are returned. func (valSet *ValidatorSet) GetByAddress(address []byte) (index int, val *Validator) { idx := sort.Search(len(valSet.Validators), func(i int) bool { return bytes.Compare(address, valSet.Validators[i].Address) <= 0 }) if idx != len(valSet.Validators) && bytes.Equal(valSet.Validators[idx].Address, address) { return idx, valSet.Validators[idx].Copy() + } else { + return -1, nil } - return 0, nil } // GetByIndex returns the validator by index. From 39a496378203bbc36e6e9e3cac057e21d1174c33 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 29 Mar 2018 13:35:15 +0200 Subject: [PATCH 31/54] document funcs in validator_set.go --- types/validator_set.go | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/types/validator_set.go b/types/validator_set.go index ce32772da..10183beca 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -83,6 +83,8 @@ func (valSet *ValidatorSet) Copy() *ValidatorSet { } } +// HasAddress returns true if address given is in the validator set, false - +// otherwise. func (valSet *ValidatorSet) HasAddress(address []byte) bool { idx := sort.Search(len(valSet.Validators), func(i int) bool { return bytes.Compare(address, valSet.Validators[i].Address) <= 0 @@ -96,16 +98,16 @@ func (valSet *ValidatorSet) GetByAddress(address []byte) (index int, val *Valida idx := sort.Search(len(valSet.Validators), func(i int) bool { return bytes.Compare(address, valSet.Validators[i].Address) <= 0 }) - if idx != len(valSet.Validators) && bytes.Equal(valSet.Validators[idx].Address, address) { + if idx < len(valSet.Validators) && bytes.Equal(valSet.Validators[idx].Address, address) { return idx, valSet.Validators[idx].Copy() } else { return -1, nil } } -// GetByIndex returns the validator by index. -// It returns nil values if index < 0 or -// index >= len(ValidatorSet.Validators) +// GetByIndex returns the validator's address and validator itself by index. +// It returns nil values if index is less than 0 or greater or equal to +// len(ValidatorSet.Validators). func (valSet *ValidatorSet) GetByIndex(index int) (address []byte, val *Validator) { if index < 0 || index >= len(valSet.Validators) { return nil, nil @@ -114,10 +116,12 @@ func (valSet *ValidatorSet) GetByIndex(index int) (address []byte, val *Validato return val.Address, val.Copy() } +// Size returns the length of the validator set. func (valSet *ValidatorSet) Size() int { return len(valSet.Validators) } +// TotalVotingPower returns the sum of the voting powers of all validators. func (valSet *ValidatorSet) TotalVotingPower() int64 { if valSet.totalVotingPower == 0 { for _, val := range valSet.Validators { @@ -128,6 +132,8 @@ func (valSet *ValidatorSet) TotalVotingPower() int64 { return valSet.totalVotingPower } +// GetProposer returns the current proposer. If the validator set is empty, nil +// is returned. func (valSet *ValidatorSet) GetProposer() (proposer *Validator) { if len(valSet.Validators) == 0 { return nil @@ -148,6 +154,8 @@ func (valSet *ValidatorSet) findProposer() *Validator { return proposer } +// Hash returns the Merkle root hash build using validators (as leaves) in the +// set. func (valSet *ValidatorSet) Hash() []byte { if len(valSet.Validators) == 0 { return nil @@ -159,6 +167,8 @@ func (valSet *ValidatorSet) Hash() []byte { return merkle.SimpleHashFromHashers(hashers) } +// Add adds val to the validator set and returns true. It returns false if val +// is already in the set. func (valSet *ValidatorSet) Add(val *Validator) (added bool) { val = val.Copy() idx := sort.Search(len(valSet.Validators), func(i int) bool { @@ -185,6 +195,8 @@ func (valSet *ValidatorSet) Add(val *Validator) (added bool) { } } +// Update updates val and returns true. It returns false if val is not present +// in the set. func (valSet *ValidatorSet) Update(val *Validator) (updated bool) { index, sameVal := valSet.GetByAddress(val.Address) if sameVal == nil { @@ -197,6 +209,8 @@ func (valSet *ValidatorSet) Update(val *Validator) (updated bool) { return true } +// Remove deletes the validator with address. It returns the validator removed +// and true. If returns nil and false if validator is not present in the set. func (valSet *ValidatorSet) Remove(address []byte) (val *Validator, removed bool) { idx := sort.Search(len(valSet.Validators), func(i int) bool { return bytes.Compare(address, valSet.Validators[i].Address) <= 0 @@ -216,6 +230,7 @@ func (valSet *ValidatorSet) Remove(address []byte) (val *Validator, removed bool return removedVal, true } +// Iterate will run the given function over the set. func (valSet *ValidatorSet) Iterate(fn func(index int, val *Validator) bool) { for i, val := range valSet.Validators { stop := fn(i, val.Copy()) From 0732526465b45e7db85e0ce1b992837d48b4f777 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 29 Mar 2018 13:37:12 +0200 Subject: [PATCH 32/54] use more relaxing < and >= ops instead of != an example of Search from godocs: ``` package main import ( "fmt" "sort" ) func main() { a := []int{1, 3, 6, 10, 15, 21, 28, 36, 45, 55} x := 6 i := sort.Search(len(a), func(i int) bool { return a[i] >= x }) if i < len(a) && a[i] == x { fmt.Printf("found %d at index %d in %v\n", x, i, a) } else { fmt.Printf("%d not found in %v\n", x, a) } } ``` --- types/validator_set.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/types/validator_set.go b/types/validator_set.go index 10183beca..887580d69 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -89,7 +89,7 @@ func (valSet *ValidatorSet) HasAddress(address []byte) bool { idx := sort.Search(len(valSet.Validators), func(i int) bool { return bytes.Compare(address, valSet.Validators[i].Address) <= 0 }) - return idx != len(valSet.Validators) && bytes.Equal(valSet.Validators[idx].Address, address) + return idx < len(valSet.Validators) && bytes.Equal(valSet.Validators[idx].Address, address) } // GetByAddress returns an index of the validator with address and validator @@ -174,7 +174,7 @@ func (valSet *ValidatorSet) Add(val *Validator) (added bool) { idx := sort.Search(len(valSet.Validators), func(i int) bool { return bytes.Compare(val.Address, valSet.Validators[i].Address) <= 0 }) - if idx == len(valSet.Validators) { + if idx >= len(valSet.Validators) { valSet.Validators = append(valSet.Validators, val) // Invalidate cache valSet.Proposer = nil @@ -215,7 +215,7 @@ func (valSet *ValidatorSet) Remove(address []byte) (val *Validator, removed bool idx := sort.Search(len(valSet.Validators), func(i int) bool { return bytes.Compare(address, valSet.Validators[i].Address) <= 0 }) - if idx == len(valSet.Validators) || !bytes.Equal(valSet.Validators[idx].Address, address) { + if idx >= len(valSet.Validators) || !bytes.Equal(valSet.Validators[idx].Address, address) { return nil, false } removedVal := valSet.Validators[idx] From ed782e7508f430c2ece4fb2ebce82e2d4c2a6623 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 29 Mar 2018 13:39:15 +0200 Subject: [PATCH 33/54] include validator's voting power in /status Refs #581 --- CHANGELOG.md | 1 + rpc/core/status.go | 49 ++++++++++++++++++++++++++++++++++--- rpc/core/types/responses.go | 19 ++++++++------ types/validator_set.go | 3 +-- 4 files changed, 60 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7c61489c..cf01bdbb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ BUG FIXES: IMPROVEMENTS: - [rpc] `/tx` and `/tx_search` responses now include the transaction hash +- [rpc] include validator power in `/status` ## 0.17.1 (March 27th, 2018) diff --git a/rpc/core/status.go b/rpc/core/status.go index a8771c8f7..b4543a61c 100644 --- a/rpc/core/status.go +++ b/rpc/core/status.go @@ -1,9 +1,11 @@ package core import ( + "bytes" "time" ctypes "github.com/tendermint/tendermint/rpc/core/types" + sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" cmn "github.com/tendermint/tmlibs/common" ) @@ -48,7 +50,10 @@ import ( // "remote_addr": "", // "network": "test-chain-qhVCa2", // "moniker": "vagrant-ubuntu-trusty-64", -// "pub_key": "844981FE99ABB19F7816F2D5E94E8A74276AB1153760A7799E925C75401856C6" +// "pub_key": "844981FE99ABB19F7816F2D5E94E8A74276AB1153760A7799E925C75401856C6", +// "validator_status": { +// "voting_power": 10 +// } // } // }, // "id": "", @@ -72,12 +77,50 @@ func Status() (*ctypes.ResultStatus, error) { latestBlockTime := time.Unix(0, latestBlockTimeNano) - return &ctypes.ResultStatus{ + result := &ctypes.ResultStatus{ NodeInfo: p2pSwitch.NodeInfo(), PubKey: pubKey, LatestBlockHash: latestBlockHash, LatestAppHash: latestAppHash, LatestBlockHeight: latestHeight, LatestBlockTime: latestBlockTime, - Syncing: consensusReactor.FastSync()}, nil + Syncing: consensusReactor.FastSync(), + } + + // add ValidatorStatus if node is a validator + if val := validatorAtHeight(latestHeight); val != nil { + result.ValidatorStatus = ctypes.ValidatorStatus{ + VotingPower: val.VotingPower, + } + } + + return result, nil +} + +func validatorAtHeight(h int64) *types.Validator { + lastBlockHeight, vals := consensusState.GetValidators() + + privValAddress := pubKey.Address() + + // if we're still at height h, search in the current validator set + if lastBlockHeight == h { + for _, val := range vals { + if bytes.Equal(val.Address, privValAddress) { + return val + } + } + } + + // if we've moved to the next height, retrieve the validator set from DB + if lastBlockHeight > h { + vals, err := sm.LoadValidators(stateDB, h) + if err != nil { + // should not happen + return nil + } + _, val := vals.GetByAddress(privValAddress) + return val + } + + return nil } diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index 5b49e1af6..48f8723d6 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -54,14 +54,19 @@ func NewResultCommit(header *types.Header, commit *types.Commit, } } +type ValidatorStatus struct { + VotingPower int64 `json:"voting_power"` +} + type ResultStatus struct { - NodeInfo p2p.NodeInfo `json:"node_info"` - PubKey crypto.PubKey `json:"pub_key"` - LatestBlockHash cmn.HexBytes `json:"latest_block_hash"` - LatestAppHash cmn.HexBytes `json:"latest_app_hash"` - LatestBlockHeight int64 `json:"latest_block_height"` - LatestBlockTime time.Time `json:"latest_block_time"` - Syncing bool `json:"syncing"` + NodeInfo p2p.NodeInfo `json:"node_info"` + PubKey crypto.PubKey `json:"pub_key"` + LatestBlockHash cmn.HexBytes `json:"latest_block_hash"` + LatestAppHash cmn.HexBytes `json:"latest_app_hash"` + LatestBlockHeight int64 `json:"latest_block_height"` + LatestBlockTime time.Time `json:"latest_block_time"` + Syncing bool `json:"syncing"` + ValidatorStatus ValidatorStatus `json:"validator_status,omitempty"` } func (s *ResultStatus) TxIndexEnabled() bool { diff --git a/types/validator_set.go b/types/validator_set.go index 887580d69..4b84f85d7 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -100,9 +100,8 @@ func (valSet *ValidatorSet) GetByAddress(address []byte) (index int, val *Valida }) if idx < len(valSet.Validators) && bytes.Equal(valSet.Validators[idx].Address, address) { return idx, valSet.Validators[idx].Copy() - } else { - return -1, nil } + return -1, nil } // GetByIndex returns the validator's address and validator itself by index. From f907113c197a296e448d3734d26e8a06ec1cba6b Mon Sep 17 00:00:00 2001 From: Zaki Manian Date: Thu, 5 Apr 2018 02:02:23 -0700 Subject: [PATCH 34/54] Net_info should print the ID of peers (#1312) --- rpc/core/net.go | 1 + rpc/core/types/responses.go | 1 + 2 files changed, 2 insertions(+) diff --git a/rpc/core/net.go b/rpc/core/net.go index 9b04926ab..1918abf11 100644 --- a/rpc/core/net.go +++ b/rpc/core/net.go @@ -43,6 +43,7 @@ func NetInfo() (*ctypes.ResultNetInfo, error) { for _, peer := range p2pSwitch.Peers().List() { peers = append(peers, ctypes.Peer{ NodeInfo: peer.NodeInfo(), + ID: peer.ID(), IsOutbound: peer.IsOutbound(), ConnectionStatus: peer.Status(), }) diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index 48f8723d6..8a6fff63c 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -98,6 +98,7 @@ type ResultDialPeers struct { type Peer struct { p2p.NodeInfo `json:"node_info"` + p2p.ID `json:"node_id"` IsOutbound bool `json:"is_outbound"` ConnectionStatus p2p.ConnectionStatus `json:"connection_status"` } From a506cf47add829f72224531f173526e84af4826b Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 5 Apr 2018 11:42:45 +0200 Subject: [PATCH 35/54] protect Record* peerStateStats functions by mutex Fixes #1414 DATA RACE: ``` Read at 0x00c4214ee940 by goroutine 146: github.com/tendermint/tendermint/consensus.(*peerStateStats).String() :1 +0x57 fmt.(*pp).handleMethods() /usr/local/go/src/fmt/print.go:596 +0x3f4 fmt.(*pp).printArg() /usr/local/go/src/fmt/print.go:679 +0x11f fmt.(*pp).doPrintf() /usr/local/go/src/fmt/print.go:996 +0x319 fmt.Sprintf() /usr/local/go/src/fmt/print.go:196 +0x73 github.com/tendermint/tendermint/consensus.(*PeerState).StringIndented() github.com/tendermint/tendermint/consensus/_test/_obj_test/reactor.go:1426 +0x573 github.com/tendermint/tendermint/consensus.(*PeerState).String() github.com/tendermint/tendermint/consensus/_test/_obj_test/reactor.go:1419 +0x66 github.com/go-logfmt/logfmt.safeString() /home/ubuntu/go/src/github.com/go-logfmt/logfmt/encode.go:299 +0x9d github.com/go-logfmt/logfmt.writeValue() /home/ubuntu/go/src/github.com/go-logfmt/logfmt/encode.go:217 +0x5a0 github.com/go-logfmt/logfmt.(*Encoder).EncodeKeyval() /home/ubuntu/go/src/github.com/go-logfmt/logfmt/encode.go:61 +0x1dd github.com/tendermint/tmlibs/log.tmfmtLogger.Log() /home/ubuntu/go/src/github.com/tendermint/tmlibs/log/tmfmt_logger.go:107 +0x1001 github.com/tendermint/tmlibs/log.(*tmfmtLogger).Log() :1 +0x93 github.com/go-kit/kit/log.(*context).Log() /home/ubuntu/go/src/github.com/go-kit/kit/log/log.go:124 +0x248 github.com/tendermint/tmlibs/log.(*tmLogger).Debug() /home/ubuntu/go/src/github.com/tendermint/tmlibs/log/tm_logger.go:64 +0x1d0 github.com/tendermint/tendermint/consensus.(*PeerState).PickSendVote() github.com/tendermint/tendermint/consensus/_test/_obj_test/reactor.go:1059 +0x242 github.com/tendermint/tendermint/consensus.(*ConsensusReactor).gossipVotesForHeight() github.com/tendermint/tendermint/consensus/_test/_obj_test/reactor.go:789 +0x6ef github.com/tendermint/tendermint/consensus.(*ConsensusReactor).gossipVotesRoutine() github.com/tendermint/tendermint/consensus/_test/_obj_test/reactor.go:723 +0x1039 Previous write at 0x00c4214ee940 by goroutine 21: github.com/tendermint/tendermint/consensus.(*PeerState).RecordVote() github.com/tendermint/tendermint/consensus/_test/_obj_test/reactor.go:1242 +0x15a github.com/tendermint/tendermint/consensus.(*ConsensusReactor).Receive() github.com/tendermint/tendermint/consensus/_test/_obj_test/reactor.go:309 +0x32e6 github.com/tendermint/tendermint/p2p.createMConnection.func1() /home/ubuntu/go/src/github.com/tendermint/tendermint/p2p/peer.go:365 +0xea github.com/tendermint/tendermint/p2p/conn.(*MConnection).recvRoutine() /home/ubuntu/go/src/github.com/tendermint/tendermint/p2p/conn/connection.go:531 +0x779 ``` --- consensus/reactor.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/consensus/reactor.go b/consensus/reactor.go index c71f9e0c4..70a79d86a 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -1087,6 +1087,9 @@ func (ps *PeerState) ensureVoteBitArrays(height int64, numValidators int) { // It returns the total number of votes (1 per block). This essentially means // the number of blocks for which peer has been sending us votes. func (ps *PeerState) RecordVote(vote *types.Vote) int { + ps.mtx.Lock() + defer ps.mtx.Unlock() + if ps.stats.lastVoteHeight >= vote.Height { return ps.stats.votes } @@ -1098,13 +1101,20 @@ func (ps *PeerState) RecordVote(vote *types.Vote) int { // VotesSent returns the number of blocks for which peer has been sending us // votes. func (ps *PeerState) VotesSent() int { + ps.mtx.Lock() + defer ps.mtx.Unlock() + return ps.stats.votes } -// RecordVote updates internal statistics for this peer by recording the block part. -// It returns the total number of block parts (1 per block). This essentially means -// the number of blocks for which peer has been sending us block parts. +// RecordBlockPart updates internal statistics for this peer by recording the +// block part. It returns the total number of block parts (1 per block). This +// essentially means the number of blocks for which peer has been sending us +// block parts. func (ps *PeerState) RecordBlockPart(bp *BlockPartMessage) int { + ps.mtx.Lock() + defer ps.mtx.Unlock() + if ps.stats.lastBlockPartHeight >= bp.Height { return ps.stats.blockParts } @@ -1117,6 +1127,9 @@ func (ps *PeerState) RecordBlockPart(bp *BlockPartMessage) int { // BlockPartsSent returns the number of blocks for which peer has been sending // us block parts. func (ps *PeerState) BlockPartsSent() int { + ps.mtx.Lock() + defer ps.mtx.Unlock() + return ps.stats.blockParts } From 904a3115a623ad0b7a30d87d3472339ae7d1b5dc Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 22 Mar 2018 11:27:10 +0100 Subject: [PATCH 36/54] require addresses to have an ID by default Refs #1228 --- CHANGELOG.md | 9 +++-- p2p/listener.go | 2 +- p2p/netaddress.go | 37 +++++++++++++-------- p2p/netaddress_test.go | 75 ++++++++++++++++++++++++++---------------- 4 files changed, 77 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf01bdbb1..68416cbeb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,15 +25,18 @@ BUG FIXES: - Graceful handling/recovery for apps that have non-determinism or fail to halt - Graceful handling/recovery for violations of safety, or liveness -## 0.17.2 (TBD) +## 0.18.0 (TBD) -BUG FIXES: -- [rpc] fix subscribing using an abci.ResponseDeliverTx tag +BREAKING: +- [p2p] require all addresses come with an ID no matter what IMPROVEMENTS: - [rpc] `/tx` and `/tx_search` responses now include the transaction hash - [rpc] include validator power in `/status` +BUG FIXES: +- [rpc] fix subscribing using an abci.ResponseDeliverTx tag + ## 0.17.1 (March 27th, 2018) BUG FIXES: diff --git a/p2p/listener.go b/p2p/listener.go index 884c45ee8..e698765cd 100644 --- a/p2p/listener.go +++ b/p2p/listener.go @@ -72,7 +72,7 @@ func NewDefaultListener(protocol string, lAddr string, skipUPNP bool, logger log // Determine internal address... var intAddr *NetAddress - intAddr, err = NewNetAddressString(lAddr) + intAddr, err = NewNetAddressStringWithOptionalID(lAddr) if err != nil { panic(err) } diff --git a/p2p/netaddress.go b/p2p/netaddress.go index c7773a9f8..a77090a78 100644 --- a/p2p/netaddress.go +++ b/p2p/netaddress.go @@ -49,33 +49,45 @@ func NewNetAddress(id ID, addr net.Addr) *NetAddress { } ip := tcpAddr.IP port := uint16(tcpAddr.Port) - netAddr := NewNetAddressIPPort(ip, port) - netAddr.ID = id - return netAddr + na := NewNetAddressIPPort(ip, port) + na.ID = id + return na } -// NewNetAddressString returns a new NetAddress using the provided -// address in the form of "ID@IP:Port", where the ID is optional. +// NewNetAddressString returns a new NetAddress using the provided address in +// the form of "ID@IP:Port". // Also resolves the host if host is not an IP. func NewNetAddressString(addr string) (*NetAddress, error) { - addr = removeProtocolIfDefined(addr) + spl := strings.Split(addr, "@") + if len(spl) < 2 { + return nil, fmt.Errorf("Address (%s) does not contain ID", addr) + } + return NewNetAddressStringWithOptionalID(addr) +} + +// NewNetAddressStringWithOptionalID returns a new NetAddress using the +// provided address in the form of "ID@IP:Port", where the ID is optional. +// Also resolves the host if host is not an IP. +func NewNetAddressStringWithOptionalID(addr string) (*NetAddress, error) { + addrWithoutProtocol := removeProtocolIfDefined(addr) var id ID - spl := strings.Split(addr, "@") + spl := strings.Split(addrWithoutProtocol, "@") if len(spl) == 2 { idStr := spl[0] idBytes, err := hex.DecodeString(idStr) if err != nil { - return nil, errors.Wrap(err, fmt.Sprintf("Address (%s) contains invalid ID", addr)) + return nil, errors.Wrapf(err, "Address (%s) contains invalid ID", addrWithoutProtocol) } if len(idBytes) != IDByteLength { return nil, fmt.Errorf("Address (%s) contains ID of invalid length (%d). Should be %d hex-encoded bytes", - addr, len(idBytes), IDByteLength) + addrWithoutProtocol, len(idBytes), IDByteLength) } - id, addr = ID(idStr), spl[1] + + id, addrWithoutProtocol = ID(idStr), spl[1] } - host, portStr, err := net.SplitHostPort(addr) + host, portStr, err := net.SplitHostPort(addrWithoutProtocol) if err != nil { return nil, err } @@ -120,11 +132,10 @@ func NewNetAddressStrings(addrs []string) ([]*NetAddress, []error) { // NewNetAddressIPPort returns a new NetAddress using the provided IP // and port number. func NewNetAddressIPPort(ip net.IP, port uint16) *NetAddress { - na := &NetAddress{ + return &NetAddress{ IP: ip, Port: port, } - return na } // Equals reports whether na and other are the same addresses, diff --git a/p2p/netaddress_test.go b/p2p/netaddress_test.go index 6c1930a2f..653b436a6 100644 --- a/p2p/netaddress_test.go +++ b/p2p/netaddress_test.go @@ -9,20 +9,18 @@ import ( ) func TestNewNetAddress(t *testing.T) { - assert, require := assert.New(t), require.New(t) - tcpAddr, err := net.ResolveTCPAddr("tcp", "127.0.0.1:8080") - require.Nil(err) + require.Nil(t, err) addr := NewNetAddress("", tcpAddr) - assert.Equal("127.0.0.1:8080", addr.String()) + assert.Equal(t, "127.0.0.1:8080", addr.String()) - assert.NotPanics(func() { + assert.NotPanics(t, func() { NewNetAddress("", &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 8000}) }, "Calling NewNetAddress with UDPAddr should not panic in testing") } -func TestNewNetAddressString(t *testing.T) { +func TestNewNetAddressStringWithOptionalID(t *testing.T) { testCases := []struct { addr string expected string @@ -57,6 +55,28 @@ func TestNewNetAddressString(t *testing.T) { {" @ ", "", false}, } + for _, tc := range testCases { + addr, err := NewNetAddressStringWithOptionalID(tc.addr) + if tc.correct { + if assert.Nil(t, err, tc.addr) { + assert.Equal(t, tc.expected, addr.String()) + } + } else { + assert.NotNil(t, err, tc.addr) + } + } +} + +func TestNewNetAddressString(t *testing.T) { + testCases := []struct { + addr string + expected string + correct bool + }{ + {"127.0.0.1:8080", "127.0.0.1:8080", false}, + {"deadbeefdeadbeefdeadbeefdeadbeefdeadbeef@127.0.0.1:8080", "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef@127.0.0.1:8080", true}, + } + for _, tc := range testCases { addr, err := NewNetAddressString(tc.addr) if tc.correct { @@ -70,23 +90,22 @@ func TestNewNetAddressString(t *testing.T) { } func TestNewNetAddressStrings(t *testing.T) { - addrs, errs := NewNetAddressStrings([]string{"127.0.0.1:8080", "127.0.0.2:8080"}) - assert.Len(t, errs, 0) + addrs, errs := NewNetAddressStrings([]string{ + "127.0.0.1:8080", + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef@127.0.0.1:8080", + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeed@127.0.0.2:8080"}) + assert.Len(t, errs, 1) assert.Equal(t, 2, len(addrs)) } func TestNewNetAddressIPPort(t *testing.T) { - assert := assert.New(t) addr := NewNetAddressIPPort(net.ParseIP("127.0.0.1"), 8080) - - assert.Equal("127.0.0.1:8080", addr.String()) + assert.Equal(t, "127.0.0.1:8080", addr.String()) } func TestNetAddressProperties(t *testing.T) { - assert, require := assert.New(t), require.New(t) - // TODO add more test cases - tests := []struct { + testCases := []struct { addr string valid bool local bool @@ -96,21 +115,19 @@ func TestNetAddressProperties(t *testing.T) { {"ya.ru:80", true, false, true}, } - for _, t := range tests { - addr, err := NewNetAddressString(t.addr) - require.Nil(err) + for _, tc := range testCases { + addr, err := NewNetAddressStringWithOptionalID(tc.addr) + require.Nil(t, err) - assert.Equal(t.valid, addr.Valid()) - assert.Equal(t.local, addr.Local()) - assert.Equal(t.routable, addr.Routable()) + assert.Equal(t, tc.valid, addr.Valid()) + assert.Equal(t, tc.local, addr.Local()) + assert.Equal(t, tc.routable, addr.Routable()) } } func TestNetAddressReachabilityTo(t *testing.T) { - assert, require := assert.New(t), require.New(t) - // TODO add more test cases - tests := []struct { + testCases := []struct { addr string other string reachability int @@ -119,13 +136,13 @@ func TestNetAddressReachabilityTo(t *testing.T) { {"ya.ru:80", "127.0.0.1:8080", 1}, } - for _, t := range tests { - addr, err := NewNetAddressString(t.addr) - require.Nil(err) + for _, tc := range testCases { + addr, err := NewNetAddressStringWithOptionalID(tc.addr) + require.Nil(t, err) - other, err := NewNetAddressString(t.other) - require.Nil(err) + other, err := NewNetAddressStringWithOptionalID(tc.other) + require.Nil(t, err) - assert.Equal(t.reachability, addr.ReachabilityTo(other)) + assert.Equal(t, tc.reachability, addr.ReachabilityTo(other)) } } From 8e699c2bfdeed06a9994e6ca8568457789132a7c Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Sat, 24 Mar 2018 17:04:32 +0100 Subject: [PATCH 37/54] defaultSeedDisconnectWaitPeriod should be at least as long as we expect it to take for a peer to become MarkGood Refs #1130 --- p2p/pex/pex_reactor.go | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/p2p/pex/pex_reactor.go b/p2p/pex/pex_reactor.go index 45f689c05..b6378fd0c 100644 --- a/p2p/pex/pex_reactor.go +++ b/p2p/pex/pex_reactor.go @@ -30,15 +30,17 @@ const ( defaultMinNumOutboundPeers = 10 // Seed/Crawler constants - // TODO: - // We want seeds to only advertise good peers. - // Peers are marked by external mechanisms. - // We need a config value that can be set to be - // on the order of how long it would take before a good - // peer is marked good. - defaultSeedDisconnectWaitPeriod = 2 * time.Minute // disconnect after this - defaultCrawlPeerInterval = 2 * time.Minute // dont redial for this. TODO: back-off - defaultCrawlPeersPeriod = 30 * time.Second // check some peers every this + + // We want seeds to only advertise good peers. Therefore they should wait at + // least as long as we expect it to take for a peer to become good before + // disconnecting. + // see consensus/reactor.go: blocksToContributeToBecomeGoodPeer + // 10000 blocks assuming 1s blocks ~ 2.7 hours. + defaultSeedDisconnectWaitPeriod = 3 * time.Hour + + defaultCrawlPeerInterval = 2 * time.Minute // don't redial for this. TODO: back-off. what for? + + defaultCrawlPeersPeriod = 30 * time.Second // check some peers every this maxAttemptsToDial = 16 // ~ 35h in total (last attempt - 18h) ) @@ -578,8 +580,7 @@ func (r *PEXReactor) crawlPeers() { // attemptDisconnects checks if we've been with each peer long enough to disconnect func (r *PEXReactor) attemptDisconnects() { for _, peer := range r.Switch.Peers().List() { - status := peer.Status() - if status.Duration < defaultSeedDisconnectWaitPeriod { + if peer.Status().Duration < defaultSeedDisconnectWaitPeriod { continue } if peer.IsPersistent() { From 1585152341cabf966381dcb5cd984e3a1c579633 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Sat, 24 Mar 2018 17:04:59 +0100 Subject: [PATCH 38/54] https://github.com/tendermint/tendermint/pull/1128#discussion_r162799294 Refs #1130 --- p2p/pex/pex_reactor.go | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/p2p/pex/pex_reactor.go b/p2p/pex/pex_reactor.go index b6378fd0c..c56181de3 100644 --- a/p2p/pex/pex_reactor.go +++ b/p2p/pex/pex_reactor.go @@ -564,16 +564,9 @@ func (r *PEXReactor) crawlPeers() { r.book.MarkAttempt(pi.Addr) continue } - } - // Crawl the connected peers asking for more addresses - for _, pi := range peerInfos { - // We will wait a minimum period of time before crawling peers again - if now.Sub(pi.LastAttempt) >= defaultCrawlPeerInterval { - peer := r.Switch.Peers().Get(pi.Addr.ID) - if peer != nil { - r.RequestAddrs(peer) - } - } + // Ask for more addresses + peer := r.Switch.Peers().Get(pi.Addr.ID) + r.RequestAddrs(peer) } } From cee7b5cb5411b3ee94f82d7d0be5fde42df5bf36 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Sat, 24 Mar 2018 17:49:42 +0100 Subject: [PATCH 39/54] GetSelectionWithBias Refs #1130 --- CHANGELOG.md | 1 + p2p/pex/addrbook.go | 124 ++++++++++++++++++++++++++++++++++++--- p2p/pex/addrbook_test.go | 109 ++++++++++++++++++++++++++++++++++ p2p/pex/pex_reactor.go | 8 ++- 4 files changed, 231 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf01bdbb1..d88f3ff9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ BUG FIXES: - [rpc] fix subscribing using an abci.ResponseDeliverTx tag IMPROVEMENTS: +- [p2p] seeds respond with a bias towards good peers - [rpc] `/tx` and `/tx_search` responses now include the transaction hash - [rpc] include validator power in `/status` diff --git a/p2p/pex/addrbook.go b/p2p/pex/addrbook.go index 3a3920486..efb48f6da 100644 --- a/p2p/pex/addrbook.go +++ b/p2p/pex/addrbook.go @@ -42,15 +42,19 @@ type AddrBook interface { NeedMoreAddrs() bool // Pick an address to dial - PickAddress(newBias int) *p2p.NetAddress + PickAddress(biasTowardsNewAddrs int) *p2p.NetAddress // Mark address MarkGood(*p2p.NetAddress) MarkAttempt(*p2p.NetAddress) MarkBad(*p2p.NetAddress) + IsGood(*p2p.NetAddress) bool + // Send a selection of addresses to peers GetSelection() []*p2p.NetAddress + // Send a selection of addresses with bias + GetSelectionWithBias(biasTowardsNewAddrs int) []*p2p.NetAddress // TODO: remove ListOfKnownAddresses() []*knownAddress @@ -173,6 +177,14 @@ func (a *addrBook) RemoveAddress(addr *p2p.NetAddress) { a.removeFromAllBuckets(ka) } +// IsGood returns true if peer was ever marked as good and haven't +// done anything wrong since then. +func (a *addrBook) IsGood(addr *p2p.NetAddress) bool { + a.mtx.Lock() + defer a.mtx.Unlock() + return a.addrLookup[addr.ID].isOld() +} + // NeedMoreAddrs implements AddrBook - returns true if there are not have enough addresses in the book. func (a *addrBook) NeedMoreAddrs() bool { return a.Size() < needAddressThreshold @@ -180,27 +192,27 @@ func (a *addrBook) NeedMoreAddrs() bool { // PickAddress implements AddrBook. It picks an address to connect to. // The address is picked randomly from an old or new bucket according -// to the newBias argument, which must be between [0, 100] (or else is truncated to that range) +// to the biasTowardsNewAddrs argument, which must be between [0, 100] (or else is truncated to that range) // and determines how biased we are to pick an address from a new bucket. // PickAddress returns nil if the AddrBook is empty or if we try to pick // from an empty bucket. -func (a *addrBook) PickAddress(newBias int) *p2p.NetAddress { +func (a *addrBook) PickAddress(biasTowardsNewAddrs int) *p2p.NetAddress { a.mtx.Lock() defer a.mtx.Unlock() if a.size() == 0 { return nil } - if newBias > 100 { - newBias = 100 + if biasTowardsNewAddrs > 100 { + biasTowardsNewAddrs = 100 } - if newBias < 0 { - newBias = 0 + if biasTowardsNewAddrs < 0 { + biasTowardsNewAddrs = 0 } // Bias between new and old addresses. - oldCorrelation := math.Sqrt(float64(a.nOld)) * (100.0 - float64(newBias)) - newCorrelation := math.Sqrt(float64(a.nNew)) * float64(newBias) + oldCorrelation := math.Sqrt(float64(a.nOld)) * (100.0 - float64(biasTowardsNewAddrs)) + newCorrelation := math.Sqrt(float64(a.nNew)) * float64(biasTowardsNewAddrs) // pick a random peer from a random bucket var bucket map[string]*knownAddress @@ -295,6 +307,100 @@ func (a *addrBook) GetSelection() []*p2p.NetAddress { return allAddr[:numAddresses] } +// GetSelectionWithBias implements AddrBook. +// It randomly selects some addresses (old & new). Suitable for peer-exchange protocols. +// +// Each address is picked randomly from an old or new bucket according to the +// biasTowardsNewAddrs argument, which must be between [0, 100] (or else is truncated to +// that range) and determines how biased we are to pick an address from a new +// bucket. +func (a *addrBook) GetSelectionWithBias(biasTowardsNewAddrs int) []*p2p.NetAddress { + a.mtx.Lock() + defer a.mtx.Unlock() + + if a.size() == 0 { + return nil + } + + if biasTowardsNewAddrs > 100 { + biasTowardsNewAddrs = 100 + } + if biasTowardsNewAddrs < 0 { + biasTowardsNewAddrs = 0 + } + + numAddresses := cmn.MaxInt( + cmn.MinInt(minGetSelection, a.size()), + a.size()*getSelectionPercent/100) + numAddresses = cmn.MinInt(maxGetSelection, numAddresses) + + selection := make([]*p2p.NetAddress, numAddresses) + + oldBucketToAddrsMap := make(map[int]map[string]struct{}) + var oldIndex int + newBucketToAddrsMap := make(map[int]map[string]struct{}) + var newIndex int + + selectionIndex := 0 +ADDRS_LOOP: + for selectionIndex < numAddresses { + pickFromOldBucket := int((float64(selectionIndex)/float64(numAddresses))*100) >= biasTowardsNewAddrs + pickFromOldBucket = (pickFromOldBucket && a.nOld > 0) || a.nNew == 0 + bucket := make(map[string]*knownAddress) + + // loop until we pick a random non-empty bucket + for len(bucket) == 0 { + if pickFromOldBucket { + oldIndex = a.rand.Intn(len(a.bucketsOld)) + bucket = a.bucketsOld[oldIndex] + } else { + newIndex = a.rand.Intn(len(a.bucketsNew)) + bucket = a.bucketsNew[newIndex] + } + } + + // pick a random index + randIndex := a.rand.Intn(len(bucket)) + + // loop over the map to return that index + var selectedAddr *p2p.NetAddress + for _, ka := range bucket { + if randIndex == 0 { + selectedAddr = ka.Addr + break + } + randIndex-- + } + + // if we have selected the address before, restart the loop + // otherwise, record it and continue + if pickFromOldBucket { + if addrsMap, ok := oldBucketToAddrsMap[oldIndex]; ok { + if _, ok = addrsMap[selectedAddr.String()]; ok { + continue ADDRS_LOOP + } + } else { + oldBucketToAddrsMap[oldIndex] = make(map[string]struct{}) + } + oldBucketToAddrsMap[oldIndex][selectedAddr.String()] = struct{}{} + } else { + if addrsMap, ok := newBucketToAddrsMap[newIndex]; ok { + if _, ok = addrsMap[selectedAddr.String()]; ok { + continue ADDRS_LOOP + } + } else { + newBucketToAddrsMap[newIndex] = make(map[string]struct{}) + } + newBucketToAddrsMap[newIndex][selectedAddr.String()] = struct{}{} + } + + selection[selectionIndex] = selectedAddr + selectionIndex++ + } + + return selection +} + // ListOfKnownAddresses returns the new and old addresses. func (a *addrBook) ListOfKnownAddresses() []*knownAddress { a.mtx.Lock() diff --git a/p2p/pex/addrbook_test.go b/p2p/pex/addrbook_test.go index 4a8df7166..68edb188a 100644 --- a/p2p/pex/addrbook_test.go +++ b/p2p/pex/addrbook_test.go @@ -157,6 +157,13 @@ func TestAddrBookPromoteToOld(t *testing.T) { t.Errorf("selection could not be bigger than the book") } + selection = book.GetSelectionWithBias(30) + t.Logf("selection: %v", selection) + + if len(selection) > book.Size() { + t.Errorf("selection with bias could not be bigger than the book") + } + assert.Equal(t, book.Size(), 100, "expecting book size to be 100") } @@ -229,3 +236,105 @@ func TestAddrBookRemoveAddress(t *testing.T) { book.RemoveAddress(nonExistingAddr) assert.Equal(t, 0, book.Size()) } + +func TestAddrBookGetSelection(t *testing.T) { + fname := createTempFileName("addrbook_test") + defer deleteTempFile(fname) + + book := NewAddrBook(fname, true) + book.SetLogger(log.TestingLogger()) + + // 1) empty book + assert.Empty(t, book.GetSelection()) + + // 2) add one address + addr := randIPv4Address(t) + book.AddAddress(addr, addr) + + assert.Equal(t, 1, len(book.GetSelection())) + assert.Equal(t, addr, book.GetSelection()[0]) + + // 3) add a bunch of addresses + randAddrs := randNetAddressPairs(t, 100) + for _, addrSrc := range randAddrs { + book.AddAddress(addrSrc.addr, addrSrc.src) + } + + // check there is no duplicates + addrs := make(map[string]*p2p.NetAddress) + selection := book.GetSelection() + for _, addr := range selection { + if dup, ok := addrs[addr.String()]; ok { + t.Fatalf("selection %v contains duplicates %v", selection, dup) + } + addrs[addr.String()] = addr + } + + if len(selection) > book.Size() { + t.Errorf("selection %v could not be bigger than the book", selection) + } +} + +func TestAddrBookGetSelectionWithBias(t *testing.T) { + const biasTowardsNewAddrs = 30 + + fname := createTempFileName("addrbook_test") + defer deleteTempFile(fname) + + book := NewAddrBook(fname, true) + book.SetLogger(log.TestingLogger()) + + // 1) empty book + selection := book.GetSelectionWithBias(biasTowardsNewAddrs) + assert.Empty(t, selection) + + // 2) add one address + addr := randIPv4Address(t) + book.AddAddress(addr, addr) + + selection = book.GetSelectionWithBias(biasTowardsNewAddrs) + assert.Equal(t, 1, len(selection)) + assert.Equal(t, addr, selection[0]) + + // 3) add a bunch of addresses + randAddrs := randNetAddressPairs(t, 100) + for _, addrSrc := range randAddrs { + book.AddAddress(addrSrc.addr, addrSrc.src) + } + + // check there is no duplicates + addrs := make(map[string]*p2p.NetAddress) + selection = book.GetSelectionWithBias(biasTowardsNewAddrs) + for _, addr := range selection { + if dup, ok := addrs[addr.String()]; ok { + t.Fatalf("selection %v contains duplicates %v", selection, dup) + } + addrs[addr.String()] = addr + } + + if len(selection) > book.Size() { + t.Fatalf("selection %v could not be bigger than the book", selection) + } + + // 4) mark 80% of the addresses as good + randAddrsLen := len(randAddrs) + for i, addrSrc := range randAddrs { + if int((float64(i)/float64(randAddrsLen))*100) >= 20 { + book.MarkGood(addrSrc.addr) + } + } + + selection = book.GetSelectionWithBias(biasTowardsNewAddrs) + + // check that ~70% of addresses returned are good + good := 0 + for _, addr := range selection { + if book.IsGood(addr) { + good++ + } + } + got, expected := int((float64(good)/float64(len(selection)))*100), (100 - biasTowardsNewAddrs) + if got >= expected { + t.Fatalf("expected more good peers (%% got: %d, %% expected: %d, number of good addrs: %d, total: %d)", got, expected, good, len(selection)) + } +} diff --git a/p2p/pex/pex_reactor.go b/p2p/pex/pex_reactor.go index c56181de3..8c6ad5a8b 100644 --- a/p2p/pex/pex_reactor.go +++ b/p2p/pex/pex_reactor.go @@ -43,6 +43,11 @@ const ( defaultCrawlPeersPeriod = 30 * time.Second // check some peers every this maxAttemptsToDial = 16 // ~ 35h in total (last attempt - 18h) + + // if node connects to seed, it does not have any trusted peers. + // Especially in the beginning, node should have more trusted peers than + // untrusted. + biasToSelectNewPeers = 30 // 70 to select good peers ) // PEXReactor handles PEX (peer exchange) and ensures that an @@ -191,8 +196,7 @@ func (r *PEXReactor) Receive(chID byte, src Peer, msgBytes []byte) { // Seeds disconnect after sending a batch of addrs if r.config.SeedMode { - // TODO: should we be more selective ? - r.SendAddrs(src, r.book.GetSelection()) + r.SendAddrs(src, r.book.GetSelectionWithBias(biasToSelectNewPeers)) r.Switch.StopPeerGracefully(src) } else { r.SendAddrs(src, r.book.GetSelection()) From 59ca9bf480b167ba3a17b4e6e3ed3f3b52b14a8a Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 5 Apr 2018 16:16:36 +0300 Subject: [PATCH 40/54] update to tmlibs v0.8.1 --- Gopkg.lock | 30 +++++++++++++++--------------- Gopkg.toml | 4 ++-- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 09dab7a5b..9461b319b 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -105,7 +105,7 @@ "json/scanner", "json/token" ] - revision = "f40e974e75af4e271d97ce0fc917af5898ae7bda" + revision = "ef8a98b0bbce4a65b5aa4c368430a80ddc533168" [[projects]] name = "github.com/inconshreveable/mousetrap" @@ -167,8 +167,8 @@ ".", "mem" ] - revision = "bb8f1927f2a9d3ab41c9340aa034f6b803f4359c" - version = "v1.0.2" + revision = "63644898a8da0bc22138abf860edaf5277b6102e" + version = "v1.1.0" [[projects]] name = "github.com/spf13/cast" @@ -179,8 +179,8 @@ [[projects]] name = "github.com/spf13/cobra" packages = ["."] - revision = "7b2c5ac9fc04fc5efafb60700713d4fa609b777b" - version = "v0.0.1" + revision = "a1f051bc3eba734da4772d60e2d677f47cf93ef4" + version = "v0.0.2" [[projects]] branch = "master" @@ -226,7 +226,7 @@ "leveldb/table", "leveldb/util" ] - revision = "169b1b37be738edb2813dab48c97a549bcf99bb5" + revision = "714f901b98fdb3aa954b4193d8cbd64a28d80cad" [[projects]] name = "github.com/tendermint/abci" @@ -268,7 +268,6 @@ version = "v0.7.3" [[projects]] - branch = "develop" name = "github.com/tendermint/tmlibs" packages = [ "autofile", @@ -284,7 +283,8 @@ "pubsub/query", "test" ] - revision = "0f2811441f4cf44b414df16ceae3c0931c74662e" + revision = "2e24b64fc121dcdf1cabceab8dc2f7257675483c" + version = "0.8.1" [[projects]] branch = "master" @@ -299,7 +299,7 @@ "ripemd160", "salsa20/salsa" ] - revision = "88942b9c40a4c9d203b82b3731787b672d6e809b" + revision = "b2aa35443fbc700ab74c586ae79b81c171851023" [[projects]] branch = "master" @@ -313,13 +313,13 @@ "lex/httplex", "trace" ] - revision = "6078986fec03a1dcc236c34816c71b0e05018fda" + revision = "b3c676e531a6dc479fa1b35ac961c13f5e2b4d2e" [[projects]] branch = "master" name = "golang.org/x/sys" packages = ["unix"] - revision = "91ee8cde435411ca3f1cd365e8f20131aed4d0a1" + revision = "1d206c9fa8975fb4cf00df1dc8bf3283dc24ba0e" [[projects]] name = "golang.org/x/text" @@ -346,7 +346,7 @@ branch = "master" name = "google.golang.org/genproto" packages = ["googleapis/rpc/status"] - revision = "f8c8703595236ae70fdf8789ecb656ea0bcdcf46" + revision = "35de2414665fc36f56b72d982c5af480d86de5ab" [[projects]] name = "google.golang.org/grpc" @@ -375,12 +375,12 @@ [[projects]] name = "gopkg.in/yaml.v2" packages = ["."] - revision = "7f97868eec74b32b0982dd158a51a446d1da7eb5" - version = "v2.1.1" + revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183" + version = "v2.2.1" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "88f227f33e785669706f12e131610a86fd3c00f5dfd6f22c62d46e0688049726" + inputs-digest = "b7c02a311569ec5fe2197614444fb231ea60f3e65a11a20e318421f1752054d7" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 8de0de9c2..f6846793d 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -85,8 +85,8 @@ [[override]] # [[constraint]] name = "github.com/tendermint/tmlibs" - # version = "~0.7.1" - branch = "develop" + version = "~0.8.1" + # branch = "develop" [[constraint]] name = "google.golang.org/grpc" From 9a57ef9cbf671dbbeafa7aad268a44bade6117e6 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 21 Mar 2018 19:41:56 +0100 Subject: [PATCH 41/54] do not dial ourselves (ok, maybe just once) Refs #1275 --- p2p/node_info.go | 8 ++++++++ p2p/switch.go | 11 +++++++++++ 2 files changed, 19 insertions(+) diff --git a/p2p/node_info.go b/p2p/node_info.go index 6f44b49cd..46c9a1629 100644 --- a/p2p/node_info.go +++ b/p2p/node_info.go @@ -2,6 +2,7 @@ package p2p import ( "fmt" + "net" "strings" crypto "github.com/tendermint/go-crypto" @@ -107,10 +108,17 @@ OUTER_LOOP: return nil } +// ID returns node's ID. func (info NodeInfo) ID() ID { return PubKeyToID(info.PubKey) } +// IP returns node listen addr's IP. +func (info NodeInfo) IP() net.IP { + hostPort := strings.SplitN(info.ListenAddr, ":", 2) + return net.ParseIP(hostPort[0]) +} + // NetAddress returns a NetAddress derived from the NodeInfo - // it includes the authenticated peer ID and the self-reported // ListenAddr. Note that the ListenAddr is not authenticated and diff --git a/p2p/switch.go b/p2p/switch.go index fa037a9b8..f50cab802 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -58,6 +58,7 @@ type Switch struct { dialing *cmn.CMap nodeInfo NodeInfo // our node info nodeKey *NodeKey // our node privkey + nodeIP net.IP // our IP addrBook AddrBook filterConnByAddr func(net.Addr) error @@ -148,6 +149,7 @@ func (sw *Switch) IsListening() bool { // NOTE: Not goroutine safe. func (sw *Switch) SetNodeInfo(nodeInfo NodeInfo) { sw.nodeInfo = nodeInfo + sw.nodeIP = nodeInfo.IP() } // NodeInfo returns the switch's NodeInfo. @@ -381,6 +383,11 @@ func (sw *Switch) DialPeersAsync(addrBook AddrBook, peers []string, persistent b // DialPeerWithAddress dials the given peer and runs sw.addPeer if it connects and authenticates successfully. // If `persistent == true`, the switch will always try to reconnect to this peer if the connection ever fails. func (sw *Switch) DialPeerWithAddress(addr *NetAddress, persistent bool) error { + // do not dial ourselves + if addr.IP == sw.nodeIP { + return ErrSwitchConnectToSelf + } + sw.dialing.Set(string(addr.ID), addr) defer sw.dialing.Delete(string(addr.ID)) return sw.addOutboundPeerWithConfig(addr, sw.peerConfig, persistent) @@ -522,6 +529,10 @@ func (sw *Switch) addPeer(pc peerConn) error { // Avoid self if sw.nodeKey.ID() == peerID { + // overwrite current IP to avoid dialing ourselves again + // it means original nodeIP was different from public one + sw.nodeIP = peerNodeInfo.IP() + return ErrSwitchConnectToSelf } From 5a2fa71b033c576177060845c49b995af820668a Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 21 Mar 2018 20:04:22 +0100 Subject: [PATCH 42/54] use combination of IP and port, not just IP --- p2p/node_info.go | 7 ------- p2p/switch.go | 34 +++++++++++++++++----------------- 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/p2p/node_info.go b/p2p/node_info.go index 46c9a1629..346de37d3 100644 --- a/p2p/node_info.go +++ b/p2p/node_info.go @@ -2,7 +2,6 @@ package p2p import ( "fmt" - "net" "strings" crypto "github.com/tendermint/go-crypto" @@ -113,12 +112,6 @@ func (info NodeInfo) ID() ID { return PubKeyToID(info.PubKey) } -// IP returns node listen addr's IP. -func (info NodeInfo) IP() net.IP { - hostPort := strings.SplitN(info.ListenAddr, ":", 2) - return net.ParseIP(hostPort[0]) -} - // NetAddress returns a NetAddress derived from the NodeInfo - // it includes the authenticated peer ID and the self-reported // ListenAddr. Note that the ListenAddr is not authenticated and diff --git a/p2p/switch.go b/p2p/switch.go index f50cab802..d0e6df721 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -48,18 +48,18 @@ type AddrBook interface { type Switch struct { cmn.BaseService - config *cfg.P2PConfig - peerConfig *PeerConfig - listeners []Listener - reactors map[string]Reactor - chDescs []*conn.ChannelDescriptor - reactorsByCh map[byte]Reactor - peers *PeerSet - dialing *cmn.CMap - nodeInfo NodeInfo // our node info - nodeKey *NodeKey // our node privkey - nodeIP net.IP // our IP - addrBook AddrBook + config *cfg.P2PConfig + peerConfig *PeerConfig + listeners []Listener + reactors map[string]Reactor + chDescs []*conn.ChannelDescriptor + reactorsByCh map[byte]Reactor + peers *PeerSet + dialing *cmn.CMap + nodeInfo NodeInfo // our node info + nodeKey *NodeKey // our node privkey + nodePublicAddr *NetAddress + addrBook AddrBook filterConnByAddr func(net.Addr) error filterConnByID func(ID) error @@ -149,7 +149,7 @@ func (sw *Switch) IsListening() bool { // NOTE: Not goroutine safe. func (sw *Switch) SetNodeInfo(nodeInfo NodeInfo) { sw.nodeInfo = nodeInfo - sw.nodeIP = nodeInfo.IP() + sw.nodePublicAddr = nodeInfo.NetAddress() } // NodeInfo returns the switch's NodeInfo. @@ -384,7 +384,7 @@ func (sw *Switch) DialPeersAsync(addrBook AddrBook, peers []string, persistent b // If `persistent == true`, the switch will always try to reconnect to this peer if the connection ever fails. func (sw *Switch) DialPeerWithAddress(addr *NetAddress, persistent bool) error { // do not dial ourselves - if addr.IP == sw.nodeIP { + if addr.Same(sw.nodePublicAddr) { return ErrSwitchConnectToSelf } @@ -529,9 +529,9 @@ func (sw *Switch) addPeer(pc peerConn) error { // Avoid self if sw.nodeKey.ID() == peerID { - // overwrite current IP to avoid dialing ourselves again - // it means original nodeIP was different from public one - sw.nodeIP = peerNodeInfo.IP() + // overwrite current addr to avoid dialing ourselves again + // it means original nodePublicAddr was different from public one + sw.nodePublicAddr = peerNodeInfo.NetAddress() return ErrSwitchConnectToSelf } From 4b8e342309e25c801b6295313bb234c2a9c356d2 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 21 Mar 2018 20:08:43 +0100 Subject: [PATCH 43/54] fix panic: lookup testing on 10.0.2.3:53: no such host --- p2p/switch.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/p2p/switch.go b/p2p/switch.go index d0e6df721..fec448265 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -58,7 +58,7 @@ type Switch struct { dialing *cmn.CMap nodeInfo NodeInfo // our node info nodeKey *NodeKey // our node privkey - nodePublicAddr *NetAddress + nodePublicAddr string addrBook AddrBook filterConnByAddr func(net.Addr) error @@ -149,7 +149,7 @@ func (sw *Switch) IsListening() bool { // NOTE: Not goroutine safe. func (sw *Switch) SetNodeInfo(nodeInfo NodeInfo) { sw.nodeInfo = nodeInfo - sw.nodePublicAddr = nodeInfo.NetAddress() + sw.nodePublicAddr = nodeInfo.ListenAddr } // NodeInfo returns the switch's NodeInfo. @@ -384,7 +384,7 @@ func (sw *Switch) DialPeersAsync(addrBook AddrBook, peers []string, persistent b // If `persistent == true`, the switch will always try to reconnect to this peer if the connection ever fails. func (sw *Switch) DialPeerWithAddress(addr *NetAddress, persistent bool) error { // do not dial ourselves - if addr.Same(sw.nodePublicAddr) { + if addr.DialString() == sw.nodePublicAddr { return ErrSwitchConnectToSelf } @@ -531,7 +531,7 @@ func (sw *Switch) addPeer(pc peerConn) error { if sw.nodeKey.ID() == peerID { // overwrite current addr to avoid dialing ourselves again // it means original nodePublicAddr was different from public one - sw.nodePublicAddr = peerNodeInfo.NetAddress() + sw.nodePublicAddr = peerNodeInfo.ListenAddr return ErrSwitchConnectToSelf } From 3a672cb2a904a491a2a257d0eb0b3bcab834d5f7 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 21 Mar 2018 20:23:22 +0100 Subject: [PATCH 44/54] update changelog [ci skip] --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79e866a2c..69c98bf7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ IMPROVEMENTS: - [p2p] seeds respond with a bias towards good peers - [rpc] `/tx` and `/tx_search` responses now include the transaction hash - [rpc] include validator power in `/status` +- [p2p] do not try to connect to ourselves (ok, maybe only once) BUG FIXES: - [rpc] fix subscribing using an abci.ResponseDeliverTx tag From fc9ffee2e3c0bb650b424e77462fa572734f6b89 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 22 Mar 2018 09:54:32 +0100 Subject: [PATCH 45/54] remove unused tracking because it leads to memory leaks in tests see https://blog.cosmos.network/debugging-the-memory-leak-in-tendermint-210186711420 --- p2p/switch_test.go | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/p2p/switch_test.go b/p2p/switch_test.go index 06e8b642e..de0c8d204 100644 --- a/p2p/switch_test.go +++ b/p2p/switch_test.go @@ -39,8 +39,6 @@ type TestReactor struct { mtx sync.Mutex channels []*conn.ChannelDescriptor - peersAdded []Peer - peersRemoved []Peer logMessages bool msgsCounter int msgsReceived map[byte][]PeerMessage @@ -61,17 +59,9 @@ func (tr *TestReactor) GetChannels() []*conn.ChannelDescriptor { return tr.channels } -func (tr *TestReactor) AddPeer(peer Peer) { - tr.mtx.Lock() - defer tr.mtx.Unlock() - tr.peersAdded = append(tr.peersAdded, peer) -} +func (tr *TestReactor) AddPeer(peer Peer) {} -func (tr *TestReactor) RemovePeer(peer Peer, reason interface{}) { - tr.mtx.Lock() - defer tr.mtx.Unlock() - tr.peersRemoved = append(tr.peersRemoved, peer) -} +func (tr *TestReactor) RemovePeer(peer Peer, reason interface{}) {} func (tr *TestReactor) Receive(chID byte, peer Peer, msgBytes []byte) { if tr.logMessages { From 3284a13feed2ee3cda84e6ddbabc2366014449f7 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 22 Mar 2018 09:55:21 +0100 Subject: [PATCH 46/54] add test Refs #1275 --- p2p/switch_test.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/p2p/switch_test.go b/p2p/switch_test.go index de0c8d204..7ac72395c 100644 --- a/p2p/switch_test.go +++ b/p2p/switch_test.go @@ -175,6 +175,37 @@ func TestConnAddrFilter(t *testing.T) { assertNoPeersAfterTimeout(t, s2, 400*time.Millisecond) } +func TestSwitchFiltersOutItself(t *testing.T) { + s1 := MakeSwitch(config, 1, "127.0.0.2", "123.123.123", initSwitchFunc) + + // addr should be rejected immediately because of the same IP & port + addr := s1.NodeInfo().NetAddress() + err := s1.DialPeerWithAddress(addr, false) + if assert.Error(t, err) { + assert.Equal(t, ErrSwitchConnectToSelf, err) + } + + // simulate s1 having a public IP by creating a remote peer with the same ID + rp := &remotePeer{PrivKey: s1.nodeKey.PrivKey, Config: DefaultPeerConfig()} + rp.Start() + + // addr should be rejected in addPeer based on the same ID + err = s1.DialPeerWithAddress(rp.Addr(), false) + if assert.Error(t, err) { + assert.Equal(t, ErrSwitchConnectToSelf, err) + } + + // addr should be rejected immediately because during previous step we changed node's public IP + err = s1.DialPeerWithAddress(rp.Addr(), false) + if assert.Error(t, err) { + assert.Equal(t, ErrSwitchConnectToSelf, err) + } + + rp.Stop() + + assertNoPeersAfterTimeout(t, s1, 100*time.Millisecond) +} + func assertNoPeersAfterTimeout(t *testing.T, sw *Switch, timeout time.Duration) { time.Sleep(timeout) if sw.Peers().Size() != 0 { From 3b3f45d49b6ba255ed32152019864f95a7755e82 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 22 Mar 2018 16:58:57 +0100 Subject: [PATCH 47/54] use addrbook#AddOurAddress to store our address --- node/node.go | 9 ++++-- p2p/pex/addrbook.go | 17 +++++++++-- p2p/pex/pex_reactor_test.go | 57 +++++++++++++++++++++++++------------ p2p/switch.go | 34 +++++++++++----------- p2p/switch_test.go | 23 ++++++++++++++- 5 files changed, 99 insertions(+), 41 deletions(-) diff --git a/node/node.go b/node/node.go index 3a59d2f06..8b7ca1032 100644 --- a/node/node.go +++ b/node/node.go @@ -414,9 +414,14 @@ func (n *Node) OnStart() error { } n.Logger.Info("P2P Node ID", "ID", nodeKey.ID(), "file", n.config.NodeKeyFile()) - // Start the switch - n.sw.SetNodeInfo(n.makeNodeInfo(nodeKey.PubKey())) + nodeInfo := n.makeNodeInfo(nodeKey.PubKey()) + n.sw.SetNodeInfo(nodeInfo) n.sw.SetNodeKey(nodeKey) + + // Add ourselves to addrbook to prevent dialing ourselves + n.addrBook.AddOurAddress(nodeInfo.NetAddress()) + + // Start the switch err = n.sw.Start() if err != nil { return err diff --git a/p2p/pex/addrbook.go b/p2p/pex/addrbook.go index efb48f6da..adaba2fd6 100644 --- a/p2p/pex/addrbook.go +++ b/p2p/pex/addrbook.go @@ -34,6 +34,9 @@ type AddrBook interface { // Add our own addresses so we don't later add ourselves AddOurAddress(*p2p.NetAddress) + // Check if it is our address + OurAddress(*p2p.NetAddress) bool + // Add and remove an address AddAddress(addr *p2p.NetAddress, src *p2p.NetAddress) error RemoveAddress(addr *p2p.NetAddress) @@ -78,7 +81,7 @@ type addrBook struct { // accessed concurrently mtx sync.Mutex rand *rand.Rand - ourAddrs map[string]*p2p.NetAddress + ourAddrs map[string]struct{} addrLookup map[p2p.ID]*knownAddress // new & old bucketsOld []map[string]*knownAddress bucketsNew []map[string]*knownAddress @@ -93,7 +96,7 @@ type addrBook struct { func NewAddrBook(filePath string, routabilityStrict bool) *addrBook { am := &addrBook{ rand: rand.New(rand.NewSource(time.Now().UnixNano())), // TODO: seed from outside - ourAddrs: make(map[string]*p2p.NetAddress), + ourAddrs: make(map[string]struct{}), addrLookup: make(map[p2p.ID]*knownAddress), filePath: filePath, routabilityStrict: routabilityStrict, @@ -154,7 +157,15 @@ func (a *addrBook) AddOurAddress(addr *p2p.NetAddress) { a.mtx.Lock() defer a.mtx.Unlock() a.Logger.Info("Add our address to book", "addr", addr) - a.ourAddrs[addr.String()] = addr + a.ourAddrs[addr.String()] = struct{}{} +} + +// OurAddress returns true if it is our address. +func (a *addrBook) OurAddress(addr *p2p.NetAddress) bool { + a.mtx.Lock() + _, ok := a.ourAddrs[addr.String()] + a.mtx.Unlock() + return ok } // AddAddress implements AddrBook - adds the given address as received from the given source. diff --git a/p2p/pex/pex_reactor_test.go b/p2p/pex/pex_reactor_test.go index 20276ec86..329d17f37 100644 --- a/p2p/pex/pex_reactor_test.go +++ b/p2p/pex/pex_reactor_test.go @@ -63,35 +63,45 @@ func TestPEXReactorRunning(t *testing.T) { N := 3 switches := make([]*p2p.Switch, N) + // directory to store address books dir, err := ioutil.TempDir("", "pex_reactor") require.Nil(t, err) defer os.RemoveAll(dir) // nolint: errcheck - book := NewAddrBook(filepath.Join(dir, "addrbook.json"), false) - book.SetLogger(log.TestingLogger()) + + books := make([]*addrBook, N) + logger := log.TestingLogger() // create switches for i := 0; i < N; i++ { switches[i] = p2p.MakeSwitch(config, i, "127.0.0.1", "123.123.123", func(i int, sw *p2p.Switch) *p2p.Switch { - sw.SetLogger(log.TestingLogger().With("switch", i)) + books[i] = NewAddrBook(filepath.Join(dir, fmt.Sprintf("addrbook%d.json", i)), false) + books[i].SetLogger(logger.With("pex", i)) + sw.SetAddrBook(books[i]) - r := NewPEXReactor(book, &PEXReactorConfig{}) - r.SetLogger(log.TestingLogger()) + sw.SetLogger(logger.With("pex", i)) + + r := NewPEXReactor(books[i], &PEXReactorConfig{}) + r.SetLogger(logger.With("pex", i)) r.SetEnsurePeersPeriod(250 * time.Millisecond) sw.AddReactor("pex", r) + return sw }) } - // fill the address book and add listeners - for _, s := range switches { - addr := s.NodeInfo().NetAddress() - book.AddAddress(addr, addr) - s.AddListener(p2p.NewDefaultListener("tcp", s.NodeInfo().ListenAddr, true, log.TestingLogger())) + addOtherNodeAddrToAddrBook := func(switchIndex, otherSwitchIndex int) { + addr := switches[otherSwitchIndex].NodeInfo().NetAddress() + books[switchIndex].AddAddress(addr, addr) } - // start switches - for _, s := range switches { - err := s.Start() // start switch and reactors + addOtherNodeAddrToAddrBook(0, 1) + addOtherNodeAddrToAddrBook(1, 0) + addOtherNodeAddrToAddrBook(2, 1) + + for i, sw := range switches { + sw.AddListener(p2p.NewDefaultListener("tcp", sw.NodeInfo().ListenAddr, true, logger.With("pex", i))) + + err := sw.Start() // start switch and reactors require.Nil(t, err) } @@ -127,6 +137,7 @@ func TestPEXReactorRequestMessageAbuse(t *testing.T) { defer teardownReactor(book) sw := createSwitchAndAddReactors(r) + sw.SetAddrBook(book) peer := newMockPeer() p2p.AddPeerToSwitch(sw, peer) @@ -156,6 +167,7 @@ func TestPEXReactorAddrsMessageAbuse(t *testing.T) { defer teardownReactor(book) sw := createSwitchAndAddReactors(r) + sw.SetAddrBook(book) peer := newMockPeer() p2p.AddPeerToSwitch(sw, peer) @@ -182,13 +194,11 @@ func TestPEXReactorAddrsMessageAbuse(t *testing.T) { } func TestPEXReactorUsesSeedsIfNeeded(t *testing.T) { + // directory to store address books dir, err := ioutil.TempDir("", "pex_reactor") require.Nil(t, err) defer os.RemoveAll(dir) // nolint: errcheck - book := NewAddrBook(filepath.Join(dir, "addrbook.json"), false) - book.SetLogger(log.TestingLogger()) - // 1. create seed seed := p2p.MakeSwitch( config, @@ -196,6 +206,10 @@ func TestPEXReactorUsesSeedsIfNeeded(t *testing.T) { "127.0.0.1", "123.123.123", func(i int, sw *p2p.Switch) *p2p.Switch { + book := NewAddrBook(filepath.Join(dir, "addrbook0.json"), false) + book.SetLogger(log.TestingLogger()) + sw.SetAddrBook(book) + sw.SetLogger(log.TestingLogger()) r := NewPEXReactor(book, &PEXReactorConfig{}) @@ -222,6 +236,10 @@ func TestPEXReactorUsesSeedsIfNeeded(t *testing.T) { "127.0.0.1", "123.123.123", func(i int, sw *p2p.Switch) *p2p.Switch { + book := NewAddrBook(filepath.Join(dir, "addrbook1.json"), false) + book.SetLogger(log.TestingLogger()) + sw.SetAddrBook(book) + sw.SetLogger(log.TestingLogger()) r := NewPEXReactor( @@ -247,7 +265,8 @@ func TestPEXReactorCrawlStatus(t *testing.T) { defer teardownReactor(book) // Seed/Crawler mode uses data from the Switch - _ = createSwitchAndAddReactors(pexR) + sw := createSwitchAndAddReactors(pexR) + sw.SetAddrBook(book) // Create a peer, add it to the peer set and the addrbook. peer := p2p.CreateRandomPeer(false) @@ -291,7 +310,8 @@ func TestPEXReactorDialPeer(t *testing.T) { pexR, book := createReactor(&PEXReactorConfig{}) defer teardownReactor(book) - _ = createSwitchAndAddReactors(pexR) + sw := createSwitchAndAddReactors(pexR) + sw.SetAddrBook(book) peer := newMockPeer() addr := peer.NodeInfo().NetAddress() @@ -397,6 +417,7 @@ func assertPeersWithTimeout( } func createReactor(config *PEXReactorConfig) (r *PEXReactor, book *addrBook) { + // directory to store address book dir, err := ioutil.TempDir("", "pex_reactor") if err != nil { panic(err) diff --git a/p2p/switch.go b/p2p/switch.go index fec448265..a12052953 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -35,6 +35,8 @@ const ( type AddrBook interface { AddAddress(addr *NetAddress, src *NetAddress) error + AddOurAddress(*NetAddress) + OurAddress(*NetAddress) bool MarkGood(*NetAddress) Save() } @@ -48,18 +50,17 @@ type AddrBook interface { type Switch struct { cmn.BaseService - config *cfg.P2PConfig - peerConfig *PeerConfig - listeners []Listener - reactors map[string]Reactor - chDescs []*conn.ChannelDescriptor - reactorsByCh map[byte]Reactor - peers *PeerSet - dialing *cmn.CMap - nodeInfo NodeInfo // our node info - nodeKey *NodeKey // our node privkey - nodePublicAddr string - addrBook AddrBook + config *cfg.P2PConfig + peerConfig *PeerConfig + listeners []Listener + reactors map[string]Reactor + chDescs []*conn.ChannelDescriptor + reactorsByCh map[byte]Reactor + peers *PeerSet + dialing *cmn.CMap + nodeInfo NodeInfo // our node info + nodeKey *NodeKey // our node privkey + addrBook AddrBook filterConnByAddr func(net.Addr) error filterConnByID func(ID) error @@ -149,7 +150,6 @@ func (sw *Switch) IsListening() bool { // NOTE: Not goroutine safe. func (sw *Switch) SetNodeInfo(nodeInfo NodeInfo) { sw.nodeInfo = nodeInfo - sw.nodePublicAddr = nodeInfo.ListenAddr } // NodeInfo returns the switch's NodeInfo. @@ -384,7 +384,7 @@ func (sw *Switch) DialPeersAsync(addrBook AddrBook, peers []string, persistent b // If `persistent == true`, the switch will always try to reconnect to this peer if the connection ever fails. func (sw *Switch) DialPeerWithAddress(addr *NetAddress, persistent bool) error { // do not dial ourselves - if addr.DialString() == sw.nodePublicAddr { + if sw.addrBook.OurAddress(addr) { return ErrSwitchConnectToSelf } @@ -529,9 +529,9 @@ func (sw *Switch) addPeer(pc peerConn) error { // Avoid self if sw.nodeKey.ID() == peerID { - // overwrite current addr to avoid dialing ourselves again - // it means original nodePublicAddr was different from public one - sw.nodePublicAddr = peerNodeInfo.ListenAddr + // add given address to the address book to avoid dialing ourselves again + // this is our public address + sw.addrBook.AddOurAddress(peerNodeInfo.NetAddress()) return ErrSwitchConnectToSelf } diff --git a/p2p/switch_test.go b/p2p/switch_test.go index 7ac72395c..4995ad3f2 100644 --- a/p2p/switch_test.go +++ b/p2p/switch_test.go @@ -90,6 +90,8 @@ func MakeSwitchPair(t testing.TB, initSwitch func(int, *Switch) *Switch) (*Switc } func initSwitchFunc(i int, sw *Switch) *Switch { + sw.SetAddrBook(&addrBookMock{ourAddrs: make(map[string]struct{})}) + // Make two reactors of two channels each sw.AddReactor("foo", NewTestReactor([]*conn.ChannelDescriptor{ {ID: byte(0x00), Priority: 10}, @@ -99,6 +101,7 @@ func initSwitchFunc(i int, sw *Switch) *Switch { {ID: byte(0x02), Priority: 10}, {ID: byte(0x03), Priority: 10}, }, true)) + return sw } @@ -177,9 +180,12 @@ func TestConnAddrFilter(t *testing.T) { func TestSwitchFiltersOutItself(t *testing.T) { s1 := MakeSwitch(config, 1, "127.0.0.2", "123.123.123", initSwitchFunc) + addr := s1.NodeInfo().NetAddress() + + // add ourselves like we do in node.go#427 + s1.addrBook.AddOurAddress(addr) // addr should be rejected immediately because of the same IP & port - addr := s1.NodeInfo().NetAddress() err := s1.DialPeerWithAddress(addr, false) if assert.Error(t, err) { assert.Equal(t, ErrSwitchConnectToSelf, err) @@ -371,3 +377,18 @@ func BenchmarkSwitchBroadcast(b *testing.B) { b.Logf("success: %v, failure: %v", numSuccess, numFailure) } + +type addrBookMock struct { + ourAddrs map[string]struct{} +} + +var _ AddrBook = (*addrBookMock)(nil) + +func (book *addrBookMock) AddAddress(addr *NetAddress, src *NetAddress) error { return nil } +func (book *addrBookMock) AddOurAddress(addr *NetAddress) { book.ourAddrs[addr.String()] = struct{}{} } +func (book *addrBookMock) OurAddress(addr *NetAddress) bool { + _, ok := book.ourAddrs[addr.String()] + return ok +} +func (book *addrBookMock) MarkGood(*NetAddress) {} +func (book *addrBookMock) Save() {} From 34b77fcad452cf0d797967a46359f05e4841c574 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 5 Apr 2018 15:19:15 +0200 Subject: [PATCH 48/54] log error when we fail to add new address --- p2p/pex/pex_reactor.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/p2p/pex/pex_reactor.go b/p2p/pex/pex_reactor.go index 8c6ad5a8b..1bcc493dd 100644 --- a/p2p/pex/pex_reactor.go +++ b/p2p/pex/pex_reactor.go @@ -164,7 +164,10 @@ func (r *PEXReactor) AddPeer(p Peer) { // peers when we need - we don't trust inbound peers as much. addr := p.NodeInfo().NetAddress() if !isAddrPrivate(addr, r.config.PrivatePeerIDs) { - r.book.AddAddress(addr, addr) + err := r.book.AddAddress(addr, addr) + if err != nil { + r.Logger.Error("Failed to add new address", "err", err) + } } } } @@ -265,7 +268,10 @@ func (r *PEXReactor) ReceiveAddrs(addrs []*p2p.NetAddress, src Peer) error { srcAddr := src.NodeInfo().NetAddress() for _, netAddr := range addrs { if netAddr != nil && !isAddrPrivate(netAddr, r.config.PrivatePeerIDs) { - r.book.AddAddress(netAddr, srcAddr) + err := r.book.AddAddress(netAddr, srcAddr) + if err != nil { + r.Logger.Error("Failed to add new address", "err", err) + } } } return nil From 7f6ee7a46b69cb86bebcacbcd7bd86a122130698 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 5 Apr 2018 15:20:53 +0200 Subject: [PATCH 49/54] add a comment for NewSwitch --- p2p/switch.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/p2p/switch.go b/p2p/switch.go index a12052953..1abc6b14e 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -33,6 +33,8 @@ const ( //----------------------------------------------------------------------------- +// An AddrBook represents an address book from the pex package, which is used +// to store peer addresses. type AddrBook interface { AddAddress(addr *NetAddress, src *NetAddress) error AddOurAddress(*NetAddress) @@ -43,7 +45,7 @@ type AddrBook interface { //----------------------------------------------------------------------------- -// `Switch` handles peer connections and exposes an API to receive incoming messages +// Switch handles peer connections and exposes an API to receive incoming messages // on `Reactors`. Each `Reactor` is responsible for handling incoming messages of one // or more `Channels`. So while sending outgoing messages is typically performed on the peer, // incoming messages are received on the reactor. @@ -68,6 +70,7 @@ type Switch struct { rng *rand.Rand // seed for randomizing dial times and orders } +// NewSwitch creates a new Switch with the given config. func NewSwitch(config *cfg.P2PConfig) *Switch { sw := &Switch{ config: config, From d38a6cc7ea2bc536921c2cf8a5538004ede119e8 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 29 Mar 2018 12:06:15 +0200 Subject: [PATCH 50/54] trim whitespace from elements of lists (like `persistent_peers`) Refs #1380 --- CHANGELOG.md | 1 + node/node.go | 19 +++++-------------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79e866a2c..7f2b00ed1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ BREAKING: IMPROVEMENTS: - [p2p] seeds respond with a bias towards good peers +- [config] trim whitespace from elements of lists (like `persistent_peers`) - [rpc] `/tx` and `/tx_search` responses now include the transaction hash - [rpc] include validator power in `/status` diff --git a/node/node.go b/node/node.go index 3a59d2f06..dbda514b7 100644 --- a/node/node.go +++ b/node/node.go @@ -7,7 +7,6 @@ import ( "fmt" "net" "net/http" - "strings" abci "github.com/tendermint/abci/types" crypto "github.com/tendermint/go-crypto" @@ -277,19 +276,11 @@ func NewNode(config *cfg.Config, trustMetricStore = trust.NewTrustMetricStore(trustHistoryDB, trust.DefaultConfig()) trustMetricStore.SetLogger(p2pLogger) - var seeds []string - if config.P2P.Seeds != "" { - seeds = strings.Split(config.P2P.Seeds, ",") - } - var privatePeerIDs []string - if config.P2P.PrivatePeerIDs != "" { - privatePeerIDs = strings.Split(config.P2P.PrivatePeerIDs, ",") - } pexReactor := pex.NewPEXReactor(addrBook, &pex.PEXReactorConfig{ - Seeds: seeds, + Seeds: cmn.SplitAndTrim(config.P2P.Seeds, ",", " "), SeedMode: config.P2P.SeedMode, - PrivatePeerIDs: privatePeerIDs}) + PrivatePeerIDs: cmn.SplitAndTrim(config.P2P.PrivatePeerIDs, ",", " ")}) pexReactor.SetLogger(p2pLogger) sw.AddReactor("PEX", pexReactor) } @@ -339,7 +330,7 @@ func NewNode(config *cfg.Config, return nil, err } if config.TxIndex.IndexTags != "" { - txIndexer = kv.NewTxIndex(store, kv.IndexTags(strings.Split(config.TxIndex.IndexTags, ","))) + txIndexer = kv.NewTxIndex(store, kv.IndexTags(cmn.SplitAndTrim(config.TxIndex.IndexTags, ",", " "))) } else if config.TxIndex.IndexAllTags { txIndexer = kv.NewTxIndex(store, kv.IndexAllTags()) } else { @@ -424,7 +415,7 @@ func (n *Node) OnStart() error { // Always connect to persistent peers if n.config.P2P.PersistentPeers != "" { - err = n.sw.DialPeersAsync(n.addrBook, strings.Split(n.config.P2P.PersistentPeers, ","), true) + err = n.sw.DialPeersAsync(n.addrBook, cmn.SplitAndTrim(n.config.P2P.PersistentPeers, ",", " "), true) if err != nil { return err } @@ -495,7 +486,7 @@ func (n *Node) ConfigureRPC() { func (n *Node) startRPC() ([]net.Listener, error) { n.ConfigureRPC() - listenAddrs := strings.Split(n.config.RPC.ListenAddress, ",") + listenAddrs := cmn.SplitAndTrim(n.config.RPC.ListenAddress, ",", " ") if n.config.RPC.Unsafe { rpccore.AddUnsafeRoutes() From 6e39ec6e262790c9dc0080b043fa3c55ca76f364 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 5 Apr 2018 15:21:11 +0200 Subject: [PATCH 51/54] do not even try to dial ourselves also, remove address from the book (plus mark it as our address) and return an error if we fail to parse peers list --- p2p/pex/addrbook.go | 14 +++++++++++-- p2p/pex/addrbook_test.go | 16 +++++++++++++++ p2p/switch.go | 39 +++++++++++++++++++++++------------- p2p/switch_test.go | 43 ++++++++++++++++++++++------------------ 4 files changed, 77 insertions(+), 35 deletions(-) diff --git a/p2p/pex/addrbook.go b/p2p/pex/addrbook.go index adaba2fd6..a8462f375 100644 --- a/p2p/pex/addrbook.go +++ b/p2p/pex/addrbook.go @@ -33,13 +33,15 @@ type AddrBook interface { // Add our own addresses so we don't later add ourselves AddOurAddress(*p2p.NetAddress) - // Check if it is our address OurAddress(*p2p.NetAddress) bool // Add and remove an address AddAddress(addr *p2p.NetAddress, src *p2p.NetAddress) error - RemoveAddress(addr *p2p.NetAddress) + RemoveAddress(*p2p.NetAddress) + + // Check if the address is in the book + HasAddress(*p2p.NetAddress) bool // Do we need more peers? NeedMoreAddrs() bool @@ -196,6 +198,14 @@ func (a *addrBook) IsGood(addr *p2p.NetAddress) bool { return a.addrLookup[addr.ID].isOld() } +// HasAddress returns true if the address is in the book. +func (a *addrBook) HasAddress(addr *p2p.NetAddress) bool { + a.mtx.Lock() + defer a.mtx.Unlock() + ka := a.addrLookup[addr.ID] + return ka != nil +} + // NeedMoreAddrs implements AddrBook - returns true if there are not have enough addresses in the book. func (a *addrBook) NeedMoreAddrs() bool { return a.Size() < needAddressThreshold diff --git a/p2p/pex/addrbook_test.go b/p2p/pex/addrbook_test.go index 68edb188a..2e2604286 100644 --- a/p2p/pex/addrbook_test.go +++ b/p2p/pex/addrbook_test.go @@ -338,3 +338,19 @@ func TestAddrBookGetSelectionWithBias(t *testing.T) { t.Fatalf("expected more good peers (%% got: %d, %% expected: %d, number of good addrs: %d, total: %d)", got, expected, good, len(selection)) } } + +func TestAddrBookHasAddress(t *testing.T) { + fname := createTempFileName("addrbook_test") + defer deleteTempFile(fname) + + book := NewAddrBook(fname, true) + book.SetLogger(log.TestingLogger()) + addr := randIPv4Address(t) + book.AddAddress(addr, addr) + + assert.True(t, book.HasAddress(addr)) + + book.RemoveAddress(addr) + + assert.False(t, book.HasAddress(addr)) +} diff --git a/p2p/switch.go b/p2p/switch.go index 1abc6b14e..9f65a85e0 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -40,6 +40,8 @@ type AddrBook interface { AddOurAddress(*NetAddress) OurAddress(*NetAddress) bool MarkGood(*NetAddress) + RemoveAddress(*NetAddress) + HasAddress(*NetAddress) bool Save() } @@ -351,17 +353,20 @@ func (sw *Switch) DialPeersAsync(addrBook AddrBook, peers []string, persistent b for _, err := range errs { sw.Logger.Error("Error in peer's address", "err", err) } + if len(errs) > 0 { + return errors.New("Errors in peer addresses (see errors above)") + } + + ourAddr := sw.nodeInfo.NetAddress() + // TODO: move this out of here ? if addrBook != nil { // add peers to `addrBook` - ourAddr := sw.nodeInfo.NetAddress() for _, netAddr := range netAddrs { // do not add our address or ID - if netAddr.Same(ourAddr) { - continue + if !netAddr.Same(ourAddr) { + addrBook.AddAddress(netAddr, ourAddr) } - // TODO: move this out of here ? - addrBook.AddAddress(netAddr, ourAddr) } // Persist some peers to disk right away. // NOTE: integration tests depend on this @@ -372,8 +377,14 @@ func (sw *Switch) DialPeersAsync(addrBook AddrBook, peers []string, persistent b perm := sw.rng.Perm(len(netAddrs)) for i := 0; i < len(perm); i++ { go func(i int) { - sw.randomSleep(0) j := perm[i] + + // do not dial ourselves + if netAddrs[j].Same(ourAddr) { + return + } + + sw.randomSleep(0) err := sw.DialPeerWithAddress(netAddrs[j], persistent) if err != nil { sw.Logger.Error("Error dialing peer", "err", err) @@ -386,11 +397,6 @@ func (sw *Switch) DialPeersAsync(addrBook AddrBook, peers []string, persistent b // DialPeerWithAddress dials the given peer and runs sw.addPeer if it connects and authenticates successfully. // If `persistent == true`, the switch will always try to reconnect to this peer if the connection ever fails. func (sw *Switch) DialPeerWithAddress(addr *NetAddress, persistent bool) error { - // do not dial ourselves - if sw.addrBook.OurAddress(addr) { - return ErrSwitchConnectToSelf - } - sw.dialing.Set(string(addr.ID), addr) defer sw.dialing.Delete(string(addr.ID)) return sw.addOutboundPeerWithConfig(addr, sw.peerConfig, persistent) @@ -532,9 +538,14 @@ func (sw *Switch) addPeer(pc peerConn) error { // Avoid self if sw.nodeKey.ID() == peerID { - // add given address to the address book to avoid dialing ourselves again - // this is our public address - sw.addrBook.AddOurAddress(peerNodeInfo.NetAddress()) + addr := peerNodeInfo.NetAddress() + + // remove the given address from the address book if we're added it earlier + sw.addrBook.RemoveAddress(addr) + + // add the given address to the address book to avoid dialing ourselves + // again this is our public address + sw.addrBook.AddOurAddress(addr) return ErrSwitchConnectToSelf } diff --git a/p2p/switch_test.go b/p2p/switch_test.go index 4995ad3f2..8fec3e7ab 100644 --- a/p2p/switch_test.go +++ b/p2p/switch_test.go @@ -90,7 +90,9 @@ func MakeSwitchPair(t testing.TB, initSwitch func(int, *Switch) *Switch) (*Switc } func initSwitchFunc(i int, sw *Switch) *Switch { - sw.SetAddrBook(&addrBookMock{ourAddrs: make(map[string]struct{})}) + sw.SetAddrBook(&addrBookMock{ + addrs: make(map[string]struct{}, 0), + ourAddrs: make(map[string]struct{}, 0)}) // Make two reactors of two channels each sw.AddReactor("foo", NewTestReactor([]*conn.ChannelDescriptor{ @@ -180,32 +182,24 @@ func TestConnAddrFilter(t *testing.T) { func TestSwitchFiltersOutItself(t *testing.T) { s1 := MakeSwitch(config, 1, "127.0.0.2", "123.123.123", initSwitchFunc) - addr := s1.NodeInfo().NetAddress() + // addr := s1.NodeInfo().NetAddress() - // add ourselves like we do in node.go#427 - s1.addrBook.AddOurAddress(addr) - - // addr should be rejected immediately because of the same IP & port - err := s1.DialPeerWithAddress(addr, false) - if assert.Error(t, err) { - assert.Equal(t, ErrSwitchConnectToSelf, err) - } + // // add ourselves like we do in node.go#427 + // s1.addrBook.AddOurAddress(addr) // simulate s1 having a public IP by creating a remote peer with the same ID rp := &remotePeer{PrivKey: s1.nodeKey.PrivKey, Config: DefaultPeerConfig()} rp.Start() // addr should be rejected in addPeer based on the same ID - err = s1.DialPeerWithAddress(rp.Addr(), false) + err := s1.DialPeerWithAddress(rp.Addr(), false) if assert.Error(t, err) { assert.Equal(t, ErrSwitchConnectToSelf, err) } - // addr should be rejected immediately because during previous step we changed node's public IP - err = s1.DialPeerWithAddress(rp.Addr(), false) - if assert.Error(t, err) { - assert.Equal(t, ErrSwitchConnectToSelf, err) - } + assert.True(t, s1.addrBook.OurAddress(rp.Addr())) + + assert.False(t, s1.addrBook.HasAddress(rp.Addr())) rp.Stop() @@ -379,16 +373,27 @@ func BenchmarkSwitchBroadcast(b *testing.B) { } type addrBookMock struct { + addrs map[string]struct{} ourAddrs map[string]struct{} } var _ AddrBook = (*addrBookMock)(nil) -func (book *addrBookMock) AddAddress(addr *NetAddress, src *NetAddress) error { return nil } -func (book *addrBookMock) AddOurAddress(addr *NetAddress) { book.ourAddrs[addr.String()] = struct{}{} } +func (book *addrBookMock) AddAddress(addr *NetAddress, src *NetAddress) error { + book.addrs[addr.String()] = struct{}{} + return nil +} +func (book *addrBookMock) AddOurAddress(addr *NetAddress) { book.ourAddrs[addr.String()] = struct{}{} } func (book *addrBookMock) OurAddress(addr *NetAddress) bool { _, ok := book.ourAddrs[addr.String()] return ok } func (book *addrBookMock) MarkGood(*NetAddress) {} -func (book *addrBookMock) Save() {} +func (book *addrBookMock) HasAddress(addr *NetAddress) bool { + _, ok := book.addrs[addr.String()] + return ok +} +func (book *addrBookMock) RemoveAddress(addr *NetAddress) { + delete(book.addrs, addr.String()) +} +func (book *addrBookMock) Save() {} From 3233c318ea567d732c7905532ad77946690eb26e Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Fri, 6 Apr 2018 12:35:48 +0200 Subject: [PATCH 52/54] only log errors, dial correct addresses "this means if there are lookup errors or typos in the persistent_peers, tendermint will fail to start ? didn't some one ask for us not to do this previously ?" --- p2p/switch.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/p2p/switch.go b/p2p/switch.go index 9f65a85e0..e412d3cc9 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -350,12 +350,10 @@ func (sw *Switch) IsDialing(id ID) bool { // DialPeersAsync dials a list of peers asynchronously in random order (optionally, making them persistent). func (sw *Switch) DialPeersAsync(addrBook AddrBook, peers []string, persistent bool) error { netAddrs, errs := NewNetAddressStrings(peers) + // only log errors, dial correct addresses for _, err := range errs { sw.Logger.Error("Error in peer's address", "err", err) } - if len(errs) > 0 { - return errors.New("Errors in peer addresses (see errors above)") - } ourAddr := sw.nodeInfo.NetAddress() From 3d32474da8fd687e9408a5a692a89c9d99afd173 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Fri, 6 Apr 2018 13:26:05 +0200 Subject: [PATCH 53/54] make linter happy --- p2p/switch_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/p2p/switch_test.go b/p2p/switch_test.go index 8fec3e7ab..c02eb26d3 100644 --- a/p2p/switch_test.go +++ b/p2p/switch_test.go @@ -91,8 +91,8 @@ func MakeSwitchPair(t testing.TB, initSwitch func(int, *Switch) *Switch) (*Switc func initSwitchFunc(i int, sw *Switch) *Switch { sw.SetAddrBook(&addrBookMock{ - addrs: make(map[string]struct{}, 0), - ourAddrs: make(map[string]struct{}, 0)}) + addrs: make(map[string]struct{}), + ourAddrs: make(map[string]struct{})}) // Make two reactors of two channels each sw.AddReactor("foo", NewTestReactor([]*conn.ChannelDescriptor{ From 9cc2cf362fdbd15821bcc832beeb67425b9c06f0 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 6 Apr 2018 23:03:27 +0300 Subject: [PATCH 54/54] changelog and version --- CHANGELOG.md | 21 ++++++++++++++++----- version/version.go | 6 +++--- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 267100510..c5dc06753 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,6 @@ BREAKING CHANGES: - Upgrade consensus for more real-time use of evidence FEATURES: -- Peer reputation management - Use the chain as its own CA for nodes and validators - Tooling to run multiple blockchains/apps, possibly in a single process - State syncing (without transaction replay) @@ -25,20 +24,32 @@ BUG FIXES: - Graceful handling/recovery for apps that have non-determinism or fail to halt - Graceful handling/recovery for violations of safety, or liveness -## 0.18.0 (TBD) +## 0.18.0 (April 6th, 2018) BREAKING: + +- [types] Merkle tree uses different encoding for varints (see tmlibs v0.8.0) +- [types] ValidtorSet.GetByAddress returns -1 if no validator found - [p2p] require all addresses come with an ID no matter what +- [rpc] Listening address must contain tcp:// or unix:// prefix + +FEATURES: + +- [rpc] StartHTTPAndTLSServer (not used yet) +- [rpc] Include validator's voting power in `/status` +- [rpc] `/tx` and `/tx_search` responses now include the transaction hash +- [rpc] Include peer NodeIDs in `/net_info` IMPROVEMENTS: -- [p2p] seeds respond with a bias towards good peers - [config] trim whitespace from elements of lists (like `persistent_peers`) -- [rpc] `/tx` and `/tx_search` responses now include the transaction hash -- [rpc] include validator power in `/status` +- [rpc] `/tx_search` results are sorted by height - [p2p] do not try to connect to ourselves (ok, maybe only once) +- [p2p] seeds respond with a bias towards good peers BUG FIXES: - [rpc] fix subscribing using an abci.ResponseDeliverTx tag +- [rpc] fix tx_indexers matchRange +- [rpc] fix unsubscribing (see tmlibs v0.8.0) ## 0.17.1 (March 27th, 2018) diff --git a/version/version.go b/version/version.go index 74662b808..a8fbfa84f 100644 --- a/version/version.go +++ b/version/version.go @@ -1,13 +1,13 @@ package version const Maj = "0" -const Min = "17" -const Fix = "1" +const Min = "18" +const Fix = "0" var ( // Version is the current version of Tendermint // Must be a string because scripts like dist.sh read this file. - Version = "0.17.1" + Version = "0.18.0" // GitCommit is the current HEAD set using ldflags. GitCommit string