From 4722410e5e9a8c14135ff8abc335272d5fd34c97 Mon Sep 17 00:00:00 2001 From: Anton Kalyaev Date: Fri, 23 Dec 2016 20:08:12 +0400 Subject: [PATCH 1/3] test validator set changes more extensively --- consensus/reactor_test.go | 55 +++++++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index f90bacc69..55efd7166 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -105,13 +105,16 @@ func TestValidatorSetChanges(t *testing.T) { wg.Done() }) - newValidatorPubKey := css[nVals].privValidator.(*types.PrivValidator).PubKey - newValidatorTx := dummy.MakeValSetChangeTx(newValidatorPubKey.Bytes(), uint64(testMinPower)) + //--------------------------------------------------------------------------- + // Adding one validator + + newValidatorPubKey1 := css[nVals].privValidator.(*types.PrivValidator).PubKey + newValidatorTx1 := dummy.MakeValSetChangeTx(newValidatorPubKey1.Bytes(), uint64(testMinPower)) // wait till everyone makes block 2 // ensure the commit includes all validators // send newValTx to change vals in block 3 - waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, newValidatorTx) + waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, newValidatorTx1) // wait till everyone makes block 3. // it includes the commit for block 2, which is by the original validator set @@ -122,13 +125,55 @@ func TestValidatorSetChanges(t *testing.T) { waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) // the commits for block 4 should be with the updated validator set - activeVals[string(newValidatorPubKey.Address())] = struct{}{} + activeVals[string(newValidatorPubKey1.Address())] = struct{}{} // wait till everyone makes block 5 // it includes the commit for block 4, which should have the updated validator set waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) - // TODO: test more changes! + //--------------------------------------------------------------------------- + // Changing the voting power of one validator + + updateValidatorTx1 := dummy.MakeValSetChangeTx(newValidatorPubKey1.Bytes(), 25) + previousTotalVotingPower := css[nVals].LastValidators.TotalVotingPower() + + waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, updateValidatorTx1) + waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) + waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) + waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) + if css[nVals].LastValidators.TotalVotingPower() == previousTotalVotingPower { + t.Errorf("expected voting power to change (before: %d, after: %d)", previousTotalVotingPower, css[nVals].LastValidators.TotalVotingPower()) + } + + //--------------------------------------------------------------------------- + // Adding two validators at once + + newValidatorPubKey2 := css[nVals+1].privValidator.(*types.PrivValidator).PubKey + newValidatorTx2 := dummy.MakeValSetChangeTx(newValidatorPubKey2.Bytes(), uint64(testMinPower)) + + newValidatorPubKey3 := css[nVals+2].privValidator.(*types.PrivValidator).PubKey + newValidatorTx3 := dummy.MakeValSetChangeTx(newValidatorPubKey3.Bytes(), uint64(testMinPower)) + + waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, newValidatorTx2, newValidatorTx3) + waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) + waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) + activeVals[string(newValidatorPubKey2.Address())] = struct{}{} + activeVals[string(newValidatorPubKey3.Address())] = struct{}{} + waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) + + //--------------------------------------------------------------------------- + // Removing two validators at once + + removeValidatorTx2 := dummy.MakeValSetChangeTx(newValidatorPubKey2.Bytes(), 0) + removeValidatorTx3 := dummy.MakeValSetChangeTx(newValidatorPubKey3.Bytes(), 0) + + waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, removeValidatorTx2, removeValidatorTx3) + waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) + waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) + delete(activeVals, string(newValidatorPubKey2.Address())) + delete(activeVals, string(newValidatorPubKey3.Address())) + waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) + } func waitForAndValidateBlock(t *testing.T, n int, activeVals map[string]struct{}, eventChans []chan interface{}, css []*ConsensusState, txs ...[]byte) { From cb2f2b94eed04279c33b8720ddba523e3b2334e2 Mon Sep 17 00:00:00 2001 From: Anton Kalyaev Date: Mon, 26 Dec 2016 18:39:58 +0400 Subject: [PATCH 2/3] log stages to stdout --- consensus/reactor_test.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index 55efd7166..bc290c8d3 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -106,7 +106,7 @@ func TestValidatorSetChanges(t *testing.T) { }) //--------------------------------------------------------------------------- - // Adding one validator + log.Info("Testing adding one validator") newValidatorPubKey1 := css[nVals].privValidator.(*types.PrivValidator).PubKey newValidatorTx1 := dummy.MakeValSetChangeTx(newValidatorPubKey1.Bytes(), uint64(testMinPower)) @@ -132,7 +132,7 @@ func TestValidatorSetChanges(t *testing.T) { waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) //--------------------------------------------------------------------------- - // Changing the voting power of one validator + log.Info("Testing changing the voting power of one validator") updateValidatorTx1 := dummy.MakeValSetChangeTx(newValidatorPubKey1.Bytes(), 25) previousTotalVotingPower := css[nVals].LastValidators.TotalVotingPower() @@ -146,7 +146,7 @@ func TestValidatorSetChanges(t *testing.T) { } //--------------------------------------------------------------------------- - // Adding two validators at once + log.Info("Testing adding two validators at once") newValidatorPubKey2 := css[nVals+1].privValidator.(*types.PrivValidator).PubKey newValidatorTx2 := dummy.MakeValSetChangeTx(newValidatorPubKey2.Bytes(), uint64(testMinPower)) @@ -162,7 +162,7 @@ func TestValidatorSetChanges(t *testing.T) { waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) //--------------------------------------------------------------------------- - // Removing two validators at once + log.Info("Testing removing two validators at once") removeValidatorTx2 := dummy.MakeValSetChangeTx(newValidatorPubKey2.Bytes(), 0) removeValidatorTx3 := dummy.MakeValSetChangeTx(newValidatorPubKey3.Bytes(), 0) @@ -215,17 +215,15 @@ func timeoutWaitGroup(t *testing.T, n int, f func(*sync.WaitGroup, int)) { go f(wg, i) } - // Make wait into a channel done := make(chan struct{}) go func() { wg.Wait() close(done) }() - tick := time.NewTicker(time.Second * 3) select { case <-done: - case <-tick.C: + case <-time.After(time.Second * 3): t.Fatalf("Timed out waiting for all validators to commit a block") } } From d68cdce2d5207f57631704cde0fa7e29318e09e9 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 11 Jan 2017 15:32:03 -0500 Subject: [PATCH 3/3] consensus: check HasAll when TwoThirdsMajority --- consensus/byzantine_test.go | 2 +- consensus/common_test.go | 18 +++-- consensus/reactor_test.go | 128 +++++++++++++++++++++++++++++++----- consensus/state.go | 11 +++- types/validator.go | 2 +- 5 files changed, 134 insertions(+), 27 deletions(-) diff --git a/consensus/byzantine_test.go b/consensus/byzantine_test.go index 7cd6089ea..989103cce 100644 --- a/consensus/byzantine_test.go +++ b/consensus/byzantine_test.go @@ -29,7 +29,7 @@ func init() { // Heal partition and ensure A sees the commit func TestByzantine(t *testing.T) { N := 4 - css := randConsensusNet(N, "consensus_byzantine_test", newMockTickerFunc(false)) + css := randConsensusNet(N, "consensus_byzantine_test", newMockTickerFunc(false), newCounter) // give the byzantine validator a normal ticker css[0].SetTimeoutTicker(NewTimeoutTicker()) diff --git a/consensus/common_test.go b/consensus/common_test.go index 9f65a66c1..1be06785a 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -256,7 +256,7 @@ func randConsensusState(nValidators int) (*ConsensusState, []*validatorStub) { return cs, vss } -func randConsensusNet(nValidators int, testName string, tickerFunc func() TimeoutTicker) []*ConsensusState { +func randConsensusNet(nValidators int, testName string, tickerFunc func() TimeoutTicker, appFunc func() tmsp.Application) []*ConsensusState { genDoc, privVals := randGenesisDoc(nValidators, false, 10) css := make([]*ConsensusState, nValidators) for i := 0; i < nValidators; i++ { @@ -265,14 +265,14 @@ func randConsensusNet(nValidators int, testName string, tickerFunc func() Timeou state.Save() thisConfig := tendermint_test.ResetConfig(Fmt("%s_%d", testName, i)) EnsureDir(thisConfig.GetString("cs_wal_dir"), 0700) // dir for wal - css[i] = newConsensusStateWithConfig(thisConfig, state, privVals[i], counter.NewCounterApplication(true)) + css[i] = newConsensusStateWithConfig(thisConfig, state, privVals[i], appFunc()) css[i].SetTimeoutTicker(tickerFunc()) } return css } // nPeers = nValidators + nNotValidator -func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerFunc func() TimeoutTicker) []*ConsensusState { +func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerFunc func() TimeoutTicker, appFunc func() tmsp.Application) []*ConsensusState { genDoc, privVals := randGenesisDoc(nValidators, false, int64(testMinPower)) css := make([]*ConsensusState, nPeers) for i := 0; i < nPeers; i++ { @@ -290,8 +290,7 @@ func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerF privVal.SetFile(tempFilePath) } - dir, _ := ioutil.TempDir("/tmp", "persistent-dummy") - css[i] = newConsensusStateWithConfig(thisConfig, state, privVal, dummy.NewPersistentDummyApplication(dir)) + css[i] = newConsensusStateWithConfig(thisConfig, state, privVal, appFunc()) css[i].SetTimeoutTicker(tickerFunc()) } return css @@ -416,3 +415,12 @@ func (m *mockTicker) Chan() <-chan timeoutInfo { } //------------------------------------ + +func newCounter() tmsp.Application { + return counter.NewCounterApplication(true) +} + +func newPersistentDummy() tmsp.Application { + dir, _ := ioutil.TempDir("/tmp", "persistent-dummy") + return dummy.NewPersistentDummyApplication(dir) +} diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index bc290c8d3..995f4a387 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -24,7 +24,7 @@ func init() { // Ensure a testnet makes blocks func TestReactor(t *testing.T) { N := 4 - css := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true)) + css := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter) reactors := make([]*ConsensusReactor, N) eventChans := make([]chan interface{}, N) for i := 0; i < N; i++ { @@ -55,16 +55,100 @@ func TestReactor(t *testing.T) { timeoutWaitGroup(t, N, func(wg *sync.WaitGroup, j int) { <-eventChans[j] wg.Done() - }) + }, css) } //------------------------------------------------------------- // ensure we can make blocks despite cycling a validator set +func TestVotingPowerChange(t *testing.T) { + nVals := 4 + css := randConsensusNet(nVals, "consensus_voting_power_changes_test", newMockTickerFunc(true), newPersistentDummy) + reactors := make([]*ConsensusReactor, nVals) + eventChans := make([]chan interface{}, nVals) + for i := 0; i < nVals; i++ { + reactors[i] = NewConsensusReactor(css[i], true) // so we dont start the consensus states + + eventSwitch := events.NewEventSwitch() + _, err := eventSwitch.Start() + if err != nil { + t.Fatalf("Failed to start switch: %v", err) + } + + reactors[i].SetEventSwitch(eventSwitch) + eventChans[i] = subscribeToEventRespond(eventSwitch, "tester", types.EventStringNewBlock()) + } + p2p.MakeConnectedSwitches(nVals, func(i int, s *p2p.Switch) *p2p.Switch { + s.AddReactor("CONSENSUS", reactors[i]) + return s + }, p2p.Connect2Switches) + + // now that everyone is connected, start the state machines + // If we started the state machines before everyone was connected, + // we'd block when the cs fires NewBlockEvent and the peers are trying to start their reactors + for i := 0; i < nVals; i++ { + s := reactors[i].conS.GetState() + reactors[i].SwitchToConsensus(s) + } + + // map of active validators + activeVals := make(map[string]struct{}) + for i := 0; i < nVals; i++ { + activeVals[string(css[i].privValidator.GetAddress())] = struct{}{} + } + + // wait till everyone makes block 1 + timeoutWaitGroup(t, nVals, func(wg *sync.WaitGroup, j int) { + <-eventChans[j] + eventChans[j] <- struct{}{} + wg.Done() + }, css) + + //--------------------------------------------------------------------------- + log.Info("---------------------------- Testing changing the voting power of one validator a few times") + + val1PubKey := css[0].privValidator.(*types.PrivValidator).PubKey + updateValidatorTx := dummy.MakeValSetChangeTx(val1PubKey.Bytes(), 25) + previousTotalVotingPower := css[0].GetRoundState().LastValidators.TotalVotingPower() + + waitForAndValidateBlock(t, nVals, activeVals, eventChans, css, updateValidatorTx) + waitForAndValidateBlock(t, nVals, activeVals, eventChans, css) + waitForAndValidateBlock(t, nVals, activeVals, eventChans, css) + waitForAndValidateBlock(t, nVals, activeVals, eventChans, css) + + if css[0].GetRoundState().LastValidators.TotalVotingPower() == previousTotalVotingPower { + t.Fatalf("expected voting power to change (before: %d, after: %d)", previousTotalVotingPower, css[0].GetRoundState().LastValidators.TotalVotingPower()) + } + + updateValidatorTx = dummy.MakeValSetChangeTx(val1PubKey.Bytes(), 2) + previousTotalVotingPower = css[0].GetRoundState().LastValidators.TotalVotingPower() + + waitForAndValidateBlock(t, nVals, activeVals, eventChans, css, updateValidatorTx) + waitForAndValidateBlock(t, nVals, activeVals, eventChans, css) + waitForAndValidateBlock(t, nVals, activeVals, eventChans, css) + waitForAndValidateBlock(t, nVals, activeVals, eventChans, css) + + if css[0].GetRoundState().LastValidators.TotalVotingPower() == previousTotalVotingPower { + t.Fatalf("expected voting power to change (before: %d, after: %d)", previousTotalVotingPower, css[0].GetRoundState().LastValidators.TotalVotingPower()) + } + + updateValidatorTx = dummy.MakeValSetChangeTx(val1PubKey.Bytes(), 100) + previousTotalVotingPower = css[0].GetRoundState().LastValidators.TotalVotingPower() + + waitForAndValidateBlock(t, nVals, activeVals, eventChans, css, updateValidatorTx) + waitForAndValidateBlock(t, nVals, activeVals, eventChans, css) + waitForAndValidateBlock(t, nVals, activeVals, eventChans, css) + waitForAndValidateBlock(t, nVals, activeVals, eventChans, css) + + if css[0].GetRoundState().LastValidators.TotalVotingPower() == previousTotalVotingPower { + t.Fatalf("expected voting power to change (before: %d, after: %d)", previousTotalVotingPower, css[0].GetRoundState().LastValidators.TotalVotingPower()) + } +} + func TestValidatorSetChanges(t *testing.T) { nPeers := 8 nVals := 4 - css := randConsensusNetWithPeers(nVals, nPeers, "consensus_val_set_changes_test", newMockTickerFunc(true)) + css := randConsensusNetWithPeers(nVals, nPeers, "consensus_val_set_changes_test", newMockTickerFunc(true), newPersistentDummy) reactors := make([]*ConsensusReactor, nPeers) eventChans := make([]chan interface{}, nPeers) for i := 0; i < nPeers; i++ { @@ -103,10 +187,10 @@ func TestValidatorSetChanges(t *testing.T) { <-eventChans[j] eventChans[j] <- struct{}{} wg.Done() - }) + }, css) //--------------------------------------------------------------------------- - log.Info("Testing adding one validator") + log.Info("---------------------------- Testing adding one validator") newValidatorPubKey1 := css[nVals].privValidator.(*types.PrivValidator).PubKey newValidatorTx1 := dummy.MakeValSetChangeTx(newValidatorPubKey1.Bytes(), uint64(testMinPower)) @@ -132,21 +216,23 @@ func TestValidatorSetChanges(t *testing.T) { waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) //--------------------------------------------------------------------------- - log.Info("Testing changing the voting power of one validator") + log.Info("---------------------------- Testing changing the voting power of one validator") - updateValidatorTx1 := dummy.MakeValSetChangeTx(newValidatorPubKey1.Bytes(), 25) - previousTotalVotingPower := css[nVals].LastValidators.TotalVotingPower() + updateValidatorPubKey1 := css[nVals].privValidator.(*types.PrivValidator).PubKey + updateValidatorTx1 := dummy.MakeValSetChangeTx(updateValidatorPubKey1.Bytes(), 25) + previousTotalVotingPower := css[nVals].GetRoundState().LastValidators.TotalVotingPower() waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, updateValidatorTx1) waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) - if css[nVals].LastValidators.TotalVotingPower() == previousTotalVotingPower { - t.Errorf("expected voting power to change (before: %d, after: %d)", previousTotalVotingPower, css[nVals].LastValidators.TotalVotingPower()) + + if css[nVals].GetRoundState().LastValidators.TotalVotingPower() == previousTotalVotingPower { + t.Errorf("expected voting power to change (before: %d, after: %d)", previousTotalVotingPower, css[nVals].GetRoundState().LastValidators.TotalVotingPower()) } //--------------------------------------------------------------------------- - log.Info("Testing adding two validators at once") + log.Info("---------------------------- Testing adding two validators at once") newValidatorPubKey2 := css[nVals+1].privValidator.(*types.PrivValidator).PubKey newValidatorTx2 := dummy.MakeValSetChangeTx(newValidatorPubKey2.Bytes(), uint64(testMinPower)) @@ -162,7 +248,7 @@ func TestValidatorSetChanges(t *testing.T) { waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) //--------------------------------------------------------------------------- - log.Info("Testing removing two validators at once") + log.Info("---------------------------- Testing removing two validators at once") removeValidatorTx2 := dummy.MakeValSetChangeTx(newValidatorPubKey2.Bytes(), 0) removeValidatorTx3 := dummy.MakeValSetChangeTx(newValidatorPubKey3.Bytes(), 0) @@ -173,14 +259,13 @@ func TestValidatorSetChanges(t *testing.T) { delete(activeVals, string(newValidatorPubKey2.Address())) delete(activeVals, string(newValidatorPubKey3.Address())) waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) - } func waitForAndValidateBlock(t *testing.T, n int, activeVals map[string]struct{}, eventChans []chan interface{}, css []*ConsensusState, txs ...[]byte) { timeoutWaitGroup(t, n, func(wg *sync.WaitGroup, j int) { newBlockI := <-eventChans[j] newBlock := newBlockI.(types.EventDataNewBlock).Block - log.Info("Got block", "height", newBlock.Height, "validator", j) + log.Warn("Got block", "height", newBlock.Height, "validator", j) err := validateBlock(newBlock, activeVals) if err != nil { t.Fatal(err) @@ -191,7 +276,8 @@ func waitForAndValidateBlock(t *testing.T, n int, activeVals map[string]struct{} eventChans[j] <- struct{}{} wg.Done() - }) + log.Warn("Done wait group", "height", newBlock.Height, "validator", j) + }, css) } // expects high synchrony! @@ -208,7 +294,7 @@ func validateBlock(block *types.Block, activeVals map[string]struct{}) error { return nil } -func timeoutWaitGroup(t *testing.T, n int, f func(*sync.WaitGroup, int)) { +func timeoutWaitGroup(t *testing.T, n int, f func(*sync.WaitGroup, int), css []*ConsensusState) { wg := new(sync.WaitGroup) wg.Add(n) for i := 0; i < n; i++ { @@ -223,7 +309,13 @@ func timeoutWaitGroup(t *testing.T, n int, f func(*sync.WaitGroup, int)) { select { case <-done: - case <-time.After(time.Second * 3): - t.Fatalf("Timed out waiting for all validators to commit a block") + case <-time.After(time.Second * 10): + for i, cs := range css { + fmt.Println("#################") + fmt.Println("Validator", i) + fmt.Println(cs.GetRoundState()) + fmt.Println("") + } + panic("Timed out waiting for all validators to commit a block") } } diff --git a/consensus/state.go b/consensus/state.go index 594cebd67..3020a214a 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -1390,8 +1390,7 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool, if cs.LastCommit.HasAll() { // if we have all the votes now, - // schedule the timeoutCommit to happen right away - // NOTE: this won't apply if only one validator + // go straight to new round (skip timeout commit) // cs.scheduleTimeout(time.Duration(0), cs.Height, 0, RoundStepNewHeight) cs.enterNewRound(cs.Height, 0) } @@ -1452,6 +1451,14 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool, cs.enterNewRound(height, vote.Round) cs.enterPrecommit(height, vote.Round) cs.enterCommit(height, vote.Round) + + if precommits.HasAll() { + // if we have all the votes now, + // go straight to new round (skip timeout commit) + // cs.scheduleTimeout(time.Duration(0), cs.Height, 0, RoundStepNewHeight) + cs.enterNewRound(cs.Height, 0) + } + } } else if cs.Round <= vote.Round && precommits.HasTwoThirdsAny() { cs.enterNewRound(height, vote.Round) diff --git a/types/validator.go b/types/validator.go index 479824e62..c4ecef56e 100644 --- a/types/validator.go +++ b/types/validator.go @@ -61,7 +61,7 @@ func (v *Validator) String() string { if v == nil { return "nil-Validator" } - return fmt.Sprintf("Validator{%X %v %v VP:%v A:%v}", + return fmt.Sprintf("Validator{%X %v VP:%v A:%v}", v.Address, v.PubKey, v.VotingPower,