Merge pull request #359 from tendermint/val-set-changes

Val set changes

consensus/byzantine_test.go

@@ -29,7 +29,7 @@ func init() {
 // Heal partition and ensure A sees the commit
 func TestByzantine(t *testing.T) {
 	N := 4
-	css := randConsensusNet(N, "consensus_byzantine_test", newMockTickerFunc(false))
+	css := randConsensusNet(N, "consensus_byzantine_test", newMockTickerFunc(false), newCounter)
 
 	// give the byzantine validator a normal ticker
 	css[0].SetTimeoutTicker(NewTimeoutTicker())

consensus/common_test.go

@@ -256,7 +256,7 @@ func randConsensusState(nValidators int) (*ConsensusState, []*validatorStub) {
 	return cs, vss
 }
 
-func randConsensusNet(nValidators int, testName string, tickerFunc func() TimeoutTicker) []*ConsensusState {
+func randConsensusNet(nValidators int, testName string, tickerFunc func() TimeoutTicker, appFunc func() tmsp.Application) []*ConsensusState {
 	genDoc, privVals := randGenesisDoc(nValidators, false, 10)
 	css := make([]*ConsensusState, nValidators)
 	for i := 0; i < nValidators; i++ {
@@ -265,14 +265,14 @@ func randConsensusNet(nValidators int, testName string, tickerFunc func() Timeou
 		state.Save()
 		thisConfig := tendermint_test.ResetConfig(Fmt("%s_%d", testName, i))
 		EnsureDir(thisConfig.GetString("cs_wal_dir"), 0700) // dir for wal
-		css[i] = newConsensusStateWithConfig(thisConfig, state, privVals[i], counter.NewCounterApplication(true))
+		css[i] = newConsensusStateWithConfig(thisConfig, state, privVals[i], appFunc())
 		css[i].SetTimeoutTicker(tickerFunc())
 	}
 	return css
 }
 
 // nPeers = nValidators + nNotValidator
-func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerFunc func() TimeoutTicker) []*ConsensusState {
+func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerFunc func() TimeoutTicker, appFunc func() tmsp.Application) []*ConsensusState {
 	genDoc, privVals := randGenesisDoc(nValidators, false, int64(testMinPower))
 	css := make([]*ConsensusState, nPeers)
 	for i := 0; i < nPeers; i++ {
@@ -290,8 +290,7 @@ func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerF
 			privVal.SetFile(tempFilePath)
 		}
 
-		dir, _ := ioutil.TempDir("/tmp", "persistent-dummy")
-		css[i] = newConsensusStateWithConfig(thisConfig, state, privVal, dummy.NewPersistentDummyApplication(dir))
+		css[i] = newConsensusStateWithConfig(thisConfig, state, privVal, appFunc())
 		css[i].SetTimeoutTicker(tickerFunc())
 	}
 	return css
@@ -416,3 +415,12 @@ func (m *mockTicker) Chan() <-chan timeoutInfo {
 }
 
 //------------------------------------
+
+func newCounter() tmsp.Application {
+	return counter.NewCounterApplication(true)
+}
+
+func newPersistentDummy() tmsp.Application {
+	dir, _ := ioutil.TempDir("/tmp", "persistent-dummy")
+	return dummy.NewPersistentDummyApplication(dir)
+}

consensus/reactor_test.go

@@ -24,7 +24,7 @@ func init() {
 // Ensure a testnet makes blocks
 func TestReactor(t *testing.T) {
 	N := 4
-	css := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true))
+	css := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter)
 	reactors := make([]*ConsensusReactor, N)
 	eventChans := make([]chan interface{}, N)
 	for i := 0; i < N; i++ {
@@ -55,16 +55,100 @@ func TestReactor(t *testing.T) {
 	timeoutWaitGroup(t, N, func(wg *sync.WaitGroup, j int) {
 		<-eventChans[j]
 		wg.Done()
-	})
+	}, css)
 }
 
 //-------------------------------------------------------------
 // ensure we can make blocks despite cycling a validator set
 
+func TestVotingPowerChange(t *testing.T) {
+	nVals := 4
+	css := randConsensusNet(nVals, "consensus_voting_power_changes_test", newMockTickerFunc(true), newPersistentDummy)
+	reactors := make([]*ConsensusReactor, nVals)
+	eventChans := make([]chan interface{}, nVals)
+	for i := 0; i < nVals; i++ {
+		reactors[i] = NewConsensusReactor(css[i], true) // so we dont start the consensus states
+
+		eventSwitch := events.NewEventSwitch()
+		_, err := eventSwitch.Start()
+		if err != nil {
+			t.Fatalf("Failed to start switch: %v", err)
+		}
+
+		reactors[i].SetEventSwitch(eventSwitch)
+		eventChans[i] = subscribeToEventRespond(eventSwitch, "tester", types.EventStringNewBlock())
+	}
+	p2p.MakeConnectedSwitches(nVals, func(i int, s *p2p.Switch) *p2p.Switch {
+		s.AddReactor("CONSENSUS", reactors[i])
+		return s
+	}, p2p.Connect2Switches)
+
+	// now that everyone is connected,  start the state machines
+	// If we started the state machines before everyone was connected,
+	// we'd block when the cs fires NewBlockEvent and the peers are trying to start their reactors
+	for i := 0; i < nVals; i++ {
+		s := reactors[i].conS.GetState()
+		reactors[i].SwitchToConsensus(s)
+	}
+
+	// map of active validators
+	activeVals := make(map[string]struct{})
+	for i := 0; i < nVals; i++ {
+		activeVals[string(css[i].privValidator.GetAddress())] = struct{}{}
+	}
+
+	// wait till everyone makes block 1
+	timeoutWaitGroup(t, nVals, func(wg *sync.WaitGroup, j int) {
+		<-eventChans[j]
+		eventChans[j] <- struct{}{}
+		wg.Done()
+	}, css)
+
+	//---------------------------------------------------------------------------
+	log.Info("---------------------------- Testing changing the voting power of one validator a few times")
+
+	val1PubKey := css[0].privValidator.(*types.PrivValidator).PubKey
+	updateValidatorTx := dummy.MakeValSetChangeTx(val1PubKey.Bytes(), 25)
+	previousTotalVotingPower := css[0].GetRoundState().LastValidators.TotalVotingPower()
+
+	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css, updateValidatorTx)
+	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css)
+	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css)
+	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css)
+
+	if css[0].GetRoundState().LastValidators.TotalVotingPower() == previousTotalVotingPower {
+		t.Fatalf("expected voting power to change (before: %d, after: %d)", previousTotalVotingPower, css[0].GetRoundState().LastValidators.TotalVotingPower())
+	}
+
+	updateValidatorTx = dummy.MakeValSetChangeTx(val1PubKey.Bytes(), 2)
+	previousTotalVotingPower = css[0].GetRoundState().LastValidators.TotalVotingPower()
+
+	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css, updateValidatorTx)
+	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css)
+	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css)
+	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css)
+
+	if css[0].GetRoundState().LastValidators.TotalVotingPower() == previousTotalVotingPower {
+		t.Fatalf("expected voting power to change (before: %d, after: %d)", previousTotalVotingPower, css[0].GetRoundState().LastValidators.TotalVotingPower())
+	}
+
+	updateValidatorTx = dummy.MakeValSetChangeTx(val1PubKey.Bytes(), 100)
+	previousTotalVotingPower = css[0].GetRoundState().LastValidators.TotalVotingPower()
+
+	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css, updateValidatorTx)
+	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css)
+	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css)
+	waitForAndValidateBlock(t, nVals, activeVals, eventChans, css)
+
+	if css[0].GetRoundState().LastValidators.TotalVotingPower() == previousTotalVotingPower {
+		t.Fatalf("expected voting power to change (before: %d, after: %d)", previousTotalVotingPower, css[0].GetRoundState().LastValidators.TotalVotingPower())
+	}
+}
+
 func TestValidatorSetChanges(t *testing.T) {
 	nPeers := 8
 	nVals := 4
-	css := randConsensusNetWithPeers(nVals, nPeers, "consensus_val_set_changes_test", newMockTickerFunc(true))
+	css := randConsensusNetWithPeers(nVals, nPeers, "consensus_val_set_changes_test", newMockTickerFunc(true), newPersistentDummy)
 	reactors := make([]*ConsensusReactor, nPeers)
 	eventChans := make([]chan interface{}, nPeers)
 	for i := 0; i < nPeers; i++ {
@@ -103,15 +187,18 @@ func TestValidatorSetChanges(t *testing.T) {
 		<-eventChans[j]
 		eventChans[j] <- struct{}{}
 		wg.Done()
-	})
+	}, css)
 
-	newValidatorPubKey := css[nVals].privValidator.(*types.PrivValidator).PubKey
-	newValidatorTx := dummy.MakeValSetChangeTx(newValidatorPubKey.Bytes(), uint64(testMinPower))
+	//---------------------------------------------------------------------------
+	log.Info("---------------------------- Testing adding one validator")
+
+	newValidatorPubKey1 := css[nVals].privValidator.(*types.PrivValidator).PubKey
+	newValidatorTx1 := dummy.MakeValSetChangeTx(newValidatorPubKey1.Bytes(), uint64(testMinPower))
 
 	// wait till everyone makes block 2
 	// ensure the commit includes all validators
 	// send newValTx to change vals in block 3
-	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, newValidatorTx)
+	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, newValidatorTx1)
 
 	// wait till everyone makes block 3.
 	// it includes the commit for block 2, which is by the original validator set
@@ -122,20 +209,63 @@ func TestValidatorSetChanges(t *testing.T) {
 	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
 
 	// the commits for block 4 should be with the updated validator set
-	activeVals[string(newValidatorPubKey.Address())] = struct{}{}
+	activeVals[string(newValidatorPubKey1.Address())] = struct{}{}
 
 	// wait till everyone makes block 5
 	// it includes the commit for block 4, which should have the updated validator set
 	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
 
-	// TODO: test more changes!
+	//---------------------------------------------------------------------------
+	log.Info("---------------------------- Testing changing the voting power of one validator")
+
+	updateValidatorPubKey1 := css[nVals].privValidator.(*types.PrivValidator).PubKey
+	updateValidatorTx1 := dummy.MakeValSetChangeTx(updateValidatorPubKey1.Bytes(), 25)
+	previousTotalVotingPower := css[nVals].GetRoundState().LastValidators.TotalVotingPower()
+
+	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, updateValidatorTx1)
+	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
+	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
+	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
+
+	if css[nVals].GetRoundState().LastValidators.TotalVotingPower() == previousTotalVotingPower {
+		t.Errorf("expected voting power to change (before: %d, after: %d)", previousTotalVotingPower, css[nVals].GetRoundState().LastValidators.TotalVotingPower())
+	}
+
+	//---------------------------------------------------------------------------
+	log.Info("---------------------------- Testing adding two validators at once")
+
+	newValidatorPubKey2 := css[nVals+1].privValidator.(*types.PrivValidator).PubKey
+	newValidatorTx2 := dummy.MakeValSetChangeTx(newValidatorPubKey2.Bytes(), uint64(testMinPower))
+
+	newValidatorPubKey3 := css[nVals+2].privValidator.(*types.PrivValidator).PubKey
+	newValidatorTx3 := dummy.MakeValSetChangeTx(newValidatorPubKey3.Bytes(), uint64(testMinPower))
+
+	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, newValidatorTx2, newValidatorTx3)
+	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
+	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
+	activeVals[string(newValidatorPubKey2.Address())] = struct{}{}
+	activeVals[string(newValidatorPubKey3.Address())] = struct{}{}
+	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
+
+	//---------------------------------------------------------------------------
+	log.Info("---------------------------- Testing removing two validators at once")
+
+	removeValidatorTx2 := dummy.MakeValSetChangeTx(newValidatorPubKey2.Bytes(), 0)
+	removeValidatorTx3 := dummy.MakeValSetChangeTx(newValidatorPubKey3.Bytes(), 0)
+
+	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, removeValidatorTx2, removeValidatorTx3)
+	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
+	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
+	delete(activeVals, string(newValidatorPubKey2.Address()))
+	delete(activeVals, string(newValidatorPubKey3.Address()))
+	waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css)
 }
 
 func waitForAndValidateBlock(t *testing.T, n int, activeVals map[string]struct{}, eventChans []chan interface{}, css []*ConsensusState, txs ...[]byte) {
 	timeoutWaitGroup(t, n, func(wg *sync.WaitGroup, j int) {
 		newBlockI := <-eventChans[j]
 		newBlock := newBlockI.(types.EventDataNewBlock).Block
-		log.Info("Got block", "height", newBlock.Height, "validator", j)
+		log.Warn("Got block", "height", newBlock.Height, "validator", j)
 		err := validateBlock(newBlock, activeVals)
 		if err != nil {
 			t.Fatal(err)
@@ -146,7 +276,8 @@ func waitForAndValidateBlock(t *testing.T, n int, activeVals map[string]struct{}
 
 		eventChans[j] <- struct{}{}
 		wg.Done()
-	})
+		log.Warn("Done wait group", "height", newBlock.Height, "validator", j)
+	}, css)
 }
 
 // expects high synchrony!
@@ -163,24 +294,28 @@ func validateBlock(block *types.Block, activeVals map[string]struct{}) error {
 	return nil
 }
 
-func timeoutWaitGroup(t *testing.T, n int, f func(*sync.WaitGroup, int)) {
+func timeoutWaitGroup(t *testing.T, n int, f func(*sync.WaitGroup, int), css []*ConsensusState) {
 	wg := new(sync.WaitGroup)
 	wg.Add(n)
 	for i := 0; i < n; i++ {
 		go f(wg, i)
 	}
 
-	// Make wait into a channel
 	done := make(chan struct{})
 	go func() {
 		wg.Wait()
 		close(done)
 	}()
 
-	tick := time.NewTicker(time.Second * 3)
 	select {
 	case <-done:
-	case <-tick.C:
-		t.Fatalf("Timed out waiting for all validators to commit a block")
+	case <-time.After(time.Second * 10):
+		for i, cs := range css {
+			fmt.Println("#################")
+			fmt.Println("Validator", i)
+			fmt.Println(cs.GetRoundState())
+			fmt.Println("")
+		}
+		panic("Timed out waiting for all validators to commit a block")
 	}
 }

consensus/state.go

@@ -1390,8 +1390,7 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool,
 
 			if cs.LastCommit.HasAll() {
 				// if we have all the votes now,
-				// schedule the timeoutCommit to happen right away
-				// NOTE: this won't apply if only one validator
+				// go straight to new round (skip timeout commit)
 				// cs.scheduleTimeout(time.Duration(0), cs.Height, 0, RoundStepNewHeight)
 				cs.enterNewRound(cs.Height, 0)
 			}
@@ -1452,6 +1451,14 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool,
 						cs.enterNewRound(height, vote.Round)
 						cs.enterPrecommit(height, vote.Round)
 						cs.enterCommit(height, vote.Round)
+
+						if precommits.HasAll() {
+							// if we have all the votes now,
+							// go straight to new round (skip timeout commit)
+							// cs.scheduleTimeout(time.Duration(0), cs.Height, 0, RoundStepNewHeight)
+							cs.enterNewRound(cs.Height, 0)
+						}
+
 					}
 				} else if cs.Round <= vote.Round && precommits.HasTwoThirdsAny() {
 					cs.enterNewRound(height, vote.Round)

types/validator.go

@@ -61,7 +61,7 @@ func (v *Validator) String() string {
 	if v == nil {
 		return "nil-Validator"
 	}
-	return fmt.Sprintf("Validator{%X %v %v VP:%v A:%v}",
+	return fmt.Sprintf("Validator{%X %v VP:%v A:%v}",
 		v.Address,
 		v.PubKey,
 		v.VotingPower,