From c1729addce47798d3362e2c41be6a4065954c1a5 Mon Sep 17 00:00:00 2001
From: Jae Kwon <jae@tendermint.com>
Date: Thu, 15 Sep 2016 17:55:07 -0700
Subject: [PATCH] Fix BFT issue where VoteSetMaj23Message wasn't being sent
 where prs.Round == blockStore.Round()

---
 consensus/height_vote_set.go | 10 ++++++----
 consensus/reactor.go         | 10 +++++++---
 node/node.go                 |  2 +-
 types/vote_set.go            |  6 ++++--
 4 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/consensus/height_vote_set.go b/consensus/height_vote_set.go
index 7ba53b9cf..e7f4be3b9 100644
--- a/consensus/height_vote_set.go
+++ b/consensus/height_vote_set.go
@@ -24,6 +24,8 @@ but which round is not known in advance, so when a peer
 provides a precommit for a round greater than mtx.round,
 we create a new entry in roundVoteSets but also remember the
 peer to prevent abuse.
+We let each peer provide us with up to 2 unexpected "catchup" rounds.
+One for their LastCommit round, and another for the official commit round.
 */
 type HeightVoteSet struct {
 	chainID string
@@ -33,7 +35,7 @@ type HeightVoteSet struct {
 	mtx               sync.Mutex
 	round             int                  // max tracked round
 	roundVoteSets     map[int]RoundVoteSet // keys: [0...round]
-	peerCatchupRounds map[string]int       // keys: peer.Key; values: round
+	peerCatchupRounds map[string][]int     // keys: peer.Key; values: at most 2 rounds
 }
 
 func NewHeightVoteSet(chainID string, height int, valSet *types.ValidatorSet) *HeightVoteSet {
@@ -51,7 +53,7 @@ func (hvs *HeightVoteSet) Reset(height int, valSet *types.ValidatorSet) {
 	hvs.height = height
 	hvs.valSet = valSet
 	hvs.roundVoteSets = make(map[int]RoundVoteSet)
-	hvs.peerCatchupRounds = make(map[string]int)
+	hvs.peerCatchupRounds = make(map[string][]int)
 
 	hvs.addRound(0)
 	hvs.round = 0
@@ -108,10 +110,10 @@ func (hvs *HeightVoteSet) AddVote(vote *types.Vote, peerKey string) (added bool,
 	}
 	voteSet := hvs.getVoteSet(vote.Round, vote.Type)
 	if voteSet == nil {
-		if _, ok := hvs.peerCatchupRounds[peerKey]; !ok {
+		if rndz := hvs.peerCatchupRounds[peerKey]; len(rndz) < 2 {
 			hvs.addRound(vote.Round)
 			voteSet = hvs.getVoteSet(vote.Round, vote.Type)
-			hvs.peerCatchupRounds[peerKey] = vote.Round
+			hvs.peerCatchupRounds[peerKey] = append(rndz, vote.Round)
 		} else {
 			// Peer has sent a vote that does not match our round,
 			// for more than one round.  Bad peer!
diff --git a/consensus/reactor.go b/consensus/reactor.go
index 4171c7a84..c1c2aaf33 100644
--- a/consensus/reactor.go
+++ b/consensus/reactor.go
@@ -637,9 +637,13 @@ OUTER_LOOP:
 		// Maybe send Height/CatchupCommitRound/CatchupCommit.
 		{
 			prs := ps.GetRoundState()
-			if prs.CatchupCommitRound != -1 && 1 < prs.Height && prs.Height <= conR.conS.blockStore.Height() {
-				log.Warn("uh", "CatchupCommitRound", prs.CatchupCommitRound, "prs.Height", prs.Height, "blockstoreHeight", conR.conS.blockStore.Height())
-				commit := conR.conS.blockStore.LoadBlockCommit(prs.Height)
+			if prs.CatchupCommitRound != -1 && 0 < prs.Height && prs.Height <= conR.conS.blockStore.Height() {
+				var commit *types.Commit
+				if prs.Height == conR.conS.blockStore.Height() {
+					commit = conR.conS.blockStore.LoadSeenCommit(prs.Height)
+				} else {
+					commit = conR.conS.blockStore.LoadBlockCommit(prs.Height)
+				}
 				peer.TrySend(StateChannel, struct{ ConsensusMessage }{&VoteSetMaj23Message{
 					Height:  prs.Height,
 					Round:   commit.Round(),
diff --git a/node/node.go b/node/node.go
index 7c4a08e3c..404c368d5 100644
--- a/node/node.go
+++ b/node/node.go
@@ -102,7 +102,7 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato
 
 	// Make ConsensusReactor
 	consensusState := consensus.NewConsensusState(config, state.Copy(), proxyApp.Consensus(), blockStore, mempool)
-	consensusReactor := consensus.NewConsensusReactor(consensusState, blockStore, fastSync)
+	consensusReactor := consensus.NewConsensusReactor(consensusState, fastSync)
 	if privValidator != nil {
 		consensusReactor.SetPrivValidator(privValidator)
 	}
diff --git a/types/vote_set.go b/types/vote_set.go
index e659571b8..e7598532e 100644
--- a/types/vote_set.go
+++ b/types/vote_set.go
@@ -409,10 +409,12 @@ func (voteSet *VoteSet) StringIndented(indent string) string {
 %s  H:%v R:%v T:%v
 %s  %v
 %s  %v
+%s  %v
 %s}`,
 		indent, voteSet.height, voteSet.round, voteSet.type_,
 		indent, strings.Join(voteStrings, "\n"+indent+"  "),
 		indent, voteSet.votesBitArray,
+		indent, voteSet.peerMaj23s,
 		indent)
 }
 
@@ -422,8 +424,8 @@ func (voteSet *VoteSet) StringShort() string {
 	}
 	voteSet.mtx.Lock()
 	defer voteSet.mtx.Unlock()
-	return fmt.Sprintf(`VoteSet{H:%v R:%v T:%v +2/3:%v %v}`,
-		voteSet.height, voteSet.round, voteSet.type_, voteSet.maj23, voteSet.votesBitArray)
+	return fmt.Sprintf(`VoteSet{H:%v R:%v T:%v +2/3:%v %v %v}`,
+		voteSet.height, voteSet.round, voteSet.type_, voteSet.maj23, voteSet.votesBitArray, voteSet.peerMaj23s)
 }
 
 //--------------------------------------------------------------------------------