diff --git a/blockchain/store.go b/blockchain/store.go index f9d54cd23..f0f4e77f9 100644 --- a/blockchain/store.go +++ b/blockchain/store.go @@ -18,9 +18,9 @@ Simple low level store for blocks. There are three types of information stored: - BlockMeta: Meta information about each block - Block part: Parts of each block, aggregated w/ PartSet - - Validation: The Validation part of each block, for gossiping commit votes + - Validation: The Validation part of each block, for gossiping precommit votes -Currently the commit signatures are duplicated in the Block parts as +Currently the precommit signatures are duplicated in the Block parts as well as the Validation. In the future this may change, perhaps by moving the Validation data outside the Block. */ @@ -101,7 +101,7 @@ func (bs *BlockStore) LoadBlockMeta(height uint) *types.BlockMeta { return meta } -// NOTE: the Commit-vote heights are for the block at `height-1` +// NOTE: the Precommit-vote heights are for the block at `height-1` // Since these are included in the subsequent block, the height // is off by 1. func (bs *BlockStore) LoadBlockValidation(height uint) *types.Validation { @@ -118,7 +118,7 @@ func (bs *BlockStore) LoadBlockValidation(height uint) *types.Validation { return validation } -// NOTE: the Commit-vote heights are for the block at `height` +// NOTE: the Precommit-vote heights are for the block at `height` func (bs *BlockStore) LoadSeenValidation(height uint) *types.Validation { var n int64 var err error @@ -134,12 +134,10 @@ func (bs *BlockStore) LoadSeenValidation(height uint) *types.Validation { } // blockParts: Must be parts of the block -// seenValidation: The +2/3 commits that were seen which finalized the height. +// seenValidation: The +2/3 precommits that were seen which committed at height. // If all the nodes restart after committing a block, -// we need this to reload the commits to catch-up nodes to the +// we need this to reload the precommits to catch-up nodes to the // most recent height. Otherwise they'd stall at H-1. -// Also good to have to debug consensus issues & punish wrong-signers -// whose commits weren't included in the block. func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, seenValidation *types.Validation) { height := block.Height if height != bs.height+1 { @@ -163,7 +161,7 @@ func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s blockValidationBytes := binary.BinaryBytes(block.Validation) bs.db.Set(calcBlockValidationKey(height), blockValidationBytes) - // Save seen validation (seen +2/3 commits) + // Save seen validation (seen +2/3 precommits for block) seenValidationBytes := binary.BinaryBytes(seenValidation) bs.db.Set(calcSeenValidationKey(height), seenValidationBytes) diff --git a/consensus/height_vote_set.go b/consensus/height_vote_set.go index 5b5bbdb61..dbcc112f9 100644 --- a/consensus/height_vote_set.go +++ b/consensus/height_vote_set.go @@ -15,9 +15,6 @@ type RoundVoteSet struct { } // Keeps track of VoteSets for all the rounds of a height. -// We add the commit votes to all the affected rounds, -// and for new rounds carry over the commit set. Commits have -// an associated round, so the performance hit won't be O(rounds). type HeightVoteSet struct { height uint valSet *sm.ValidatorSet @@ -25,7 +22,6 @@ type HeightVoteSet struct { mtx sync.Mutex round uint // max tracked round roundVoteSets map[uint]RoundVoteSet // keys: [0...round] - commits *VoteSet // all commits for height } func NewHeightVoteSet(height uint, valSet *sm.ValidatorSet) *HeightVoteSet { @@ -33,7 +29,6 @@ func NewHeightVoteSet(height uint, valSet *sm.ValidatorSet) *HeightVoteSet { height: height, valSet: valSet, roundVoteSets: make(map[uint]RoundVoteSet), - commits: NewVoteSet(height, 0, types.VoteTypeCommit, valSet), } hvs.SetRound(0) return hvs @@ -49,7 +44,7 @@ func (hvs *HeightVoteSet) Round() uint { return hvs.round } -// Create more RoundVoteSets up to round with all commits carried over. +// Create more RoundVoteSets up to round. func (hvs *HeightVoteSet) SetRound(round uint) { hvs.mtx.Lock() defer hvs.mtx.Unlock() @@ -58,9 +53,7 @@ func (hvs *HeightVoteSet) SetRound(round uint) { } for r := hvs.round + 1; r <= round; r++ { prevotes := NewVoteSet(hvs.height, r, types.VoteTypePrevote, hvs.valSet) - prevotes.AddFromCommits(hvs.commits) precommits := NewVoteSet(hvs.height, r, types.VoteTypePrecommit, hvs.valSet) - precommits.AddFromCommits(hvs.commits) hvs.roundVoteSets[r] = RoundVoteSet{ Prevotes: prevotes, Precommits: precommits, @@ -78,39 +71,35 @@ func (hvs *HeightVoteSet) AddByAddress(address []byte, vote *types.Vote) (added return } added, index, err = voteSet.AddByAddress(address, vote) - if err != nil { - return - } - // If vote is commit, also add to all prevote/precommit for future rounds. - if vote.Type == types.VoteTypeCommit { - for round := vote.Round + 1; round <= hvs.round; round++ { - voteSet := hvs.getVoteSet(round, types.VoteTypePrevote) - _, _, err = voteSet.AddByAddress(address, vote) - if err != nil { - // TODO slash for prevote after commit - log.Warn("Prevote after commit", "address", address, "vote", vote) - } - voteSet = hvs.getVoteSet(round, types.VoteTypePrecommit) - _, _, err = voteSet.AddByAddress(address, vote) - if err != nil { - // TODO slash for prevote after commit - log.Warn("Prevote after commit", "address", address, "vote", vote) - } - } - } return } -func (hvs *HeightVoteSet) GetVoteSet(round uint, type_ byte) *VoteSet { +func (hvs *HeightVoteSet) Prevotes(round uint) *VoteSet { hvs.mtx.Lock() defer hvs.mtx.Unlock() - return hvs.getVoteSet(round, type_) + return hvs.getVoteSet(round, types.VoteTypePrevote) } -func (hvs *HeightVoteSet) getVoteSet(round uint, type_ byte) *VoteSet { - if type_ == types.VoteTypeCommit { - return hvs.commits +func (hvs *HeightVoteSet) Precommits(round uint) *VoteSet { + hvs.mtx.Lock() + defer hvs.mtx.Unlock() + return hvs.getVoteSet(round, types.VoteTypePrecommit) +} + +// Last round that has +2/3 prevotes for a particular block or nik. +// Returns -1 if no such round exists. +func (hvs *HeightVoteSet) POLRound() int { + hvs.mtx.Lock() + defer hvs.mtx.Unlock() + for r := hvs.round; r >= 0; r-- { + if hvs.getVoteSet(r, types.VoteTypePrevote).HasTwoThirdsMajority() { + return int(r) + } } + return -1 +} + +func (hvs *HeightVoteSet) getVoteSet(round uint, type_ byte) *VoteSet { rvs, ok := hvs.roundVoteSets[round] if !ok { return nil @@ -130,8 +119,7 @@ func (hvs *HeightVoteSet) String() string { } func (hvs *HeightVoteSet) StringIndented(indent string) string { - vsStrings := make([]string, 0, hvs.round*2+1) - vsStrings = append(vsStrings, hvs.commits.StringShort()) + vsStrings := make([]string, 0, hvs.round*2) for round := uint(0); round <= hvs.round; round++ { voteSetString := hvs.roundVoteSets[round].Prevotes.StringShort() vsStrings = append(vsStrings, voteSetString) diff --git a/consensus/pol.go b/consensus/pol.go deleted file mode 100644 index 7314eb48e..000000000 --- a/consensus/pol.go +++ /dev/null @@ -1,101 +0,0 @@ -package consensus - -import ( - "fmt" - - "github.com/tendermint/tendermint/account" - "github.com/tendermint/tendermint/binary" - . "github.com/tendermint/tendermint/common" - sm "github.com/tendermint/tendermint/state" - "github.com/tendermint/tendermint/types" -) - -// Each signature of a POL (proof-of-lock, see whitepaper) is -// either a prevote or a commit. -// Commits require an additional round which is strictly less than -// the POL round. Prevote rounds are equal to the POL round. -type POLVoteSignature struct { - Round uint `json:"round"` - Signature account.SignatureEd25519 `json:"signature"` -} - -// Proof of lock. -// +2/3 of validators' prevotes for a given blockhash (or nil) -type POL struct { - Height uint `json:"height"` - Round uint `json:"round"` - BlockHash []byte `json:"block_hash"` // Could be nil, which makes this a proof of unlock. - BlockParts types.PartSetHeader `json:"block_parts"` // When BlockHash is nil, this is zero. - Votes []POLVoteSignature `json:"votes"` // Prevote and commit signatures in ValidatorSet order. -} - -// Returns whether +2/3 have prevoted/committed for BlockHash. -func (pol *POL) Verify(valSet *sm.ValidatorSet) error { - - if uint(len(pol.Votes)) != valSet.Size() { - return fmt.Errorf("Invalid POL votes count: Expected %v, got %v", - valSet.Size(), len(pol.Votes)) - } - - talliedVotingPower := uint64(0) - prevoteDoc := account.SignBytes(config.GetString("chain_id"), &types.Vote{ - Height: pol.Height, Round: pol.Round, Type: types.VoteTypePrevote, - BlockHash: pol.BlockHash, - BlockParts: pol.BlockParts, - }) - seenValidators := map[string]struct{}{} - - for idx, vote := range pol.Votes { - // vote may be zero, in which case skip. - if vote.Signature.IsZero() { - continue - } - voteDoc := prevoteDoc - _, val := valSet.GetByIndex(uint(idx)) - - // Commit vote? - if vote.Round < pol.Round { - voteDoc = account.SignBytes(config.GetString("chain_id"), &types.Vote{ - Height: pol.Height, Round: vote.Round, Type: types.VoteTypeCommit, - BlockHash: pol.BlockHash, - BlockParts: pol.BlockParts, - }) - } else if vote.Round > pol.Round { - return fmt.Errorf("Invalid commit round %v for POL %v", vote.Round, pol) - } - - // Validate - if _, seen := seenValidators[string(val.Address)]; seen { - return fmt.Errorf("Duplicate validator for vote %v for POL %v", vote, pol) - } - - if !val.PubKey.VerifyBytes(voteDoc, vote.Signature) { - return fmt.Errorf("Invalid signature for vote %v for POL %v", vote, pol) - } - - // Tally - seenValidators[string(val.Address)] = struct{}{} - talliedVotingPower += val.VotingPower - } - - if talliedVotingPower > valSet.TotalVotingPower()*2/3 { - return nil - } else { - return fmt.Errorf("Invalid POL, insufficient voting power %v, needed %v", - talliedVotingPower, (valSet.TotalVotingPower()*2/3 + 1)) - } - -} - -func (pol *POL) StringShort() string { - if pol == nil { - return "nil-POL" - } else { - return fmt.Sprintf("POL{H:%v R:%v BH:%X}", pol.Height, pol.Round, - Fingerprint(pol.BlockHash), pol.BlockParts) - } -} - -func (pol *POL) MakePartSet() *types.PartSet { - return types.NewPartSetFromData(binary.BinaryBytes(pol)) -} diff --git a/consensus/pol_test.go b/consensus/pol_test.go deleted file mode 100644 index 5b56969d8..000000000 --- a/consensus/pol_test.go +++ /dev/null @@ -1,213 +0,0 @@ -package consensus - -import ( - "github.com/tendermint/tendermint/binary" - . "github.com/tendermint/tendermint/common" - _ "github.com/tendermint/tendermint/config/tendermint_test" - sm "github.com/tendermint/tendermint/state" - "github.com/tendermint/tendermint/types" - - "bytes" - "testing" -) - -// NOTE: see consensus/test.go for common test methods. - -// Convenience method. -// Signs the vote and sets the POL's vote at the desired index -// Returns the POLVoteSignature pointer, so you can modify it afterwards. -func signAddPOLVoteSignature(val *sm.PrivValidator, valSet *sm.ValidatorSet, vote *types.Vote, pol *POL) *POLVoteSignature { - vote = vote.Copy() - err := val.SignVote(config.GetString("chain_id"), vote) - if err != nil { - panic(err) - } - idx, _ := valSet.GetByAddress(val.Address) // now we have the index - pol.Votes[idx] = POLVoteSignature{vote.Round, vote.Signature} - return &pol.Votes[idx] -} - -func TestVerifyVotes(t *testing.T) { - height, round := uint(1), uint(0) - _, valSet, privValidators := randVoteSet(height, round, types.VoteTypePrevote, 10, 1) - - // Make a POL with -2/3 votes. - blockHash := RandBytes(32) - pol := &POL{ - Height: height, Round: round, BlockHash: blockHash, - Votes: make([]POLVoteSignature, valSet.Size()), - } - voteProto := &types.Vote{ - Height: height, Round: round, Type: types.VoteTypePrevote, BlockHash: blockHash, - } - for i := 0; i < 6; i++ { - signAddPOLVoteSignature(privValidators[i], valSet, voteProto, pol) - } - - // Check that validation fails. - if err := pol.Verify(valSet); err == nil { - t.Errorf("Expected POL.Verify() to fail, not enough votes.") - } - - // Insert another vote to make +2/3 - signAddPOLVoteSignature(privValidators[7], valSet, voteProto, pol) - - // Check that validation succeeds. - if err := pol.Verify(valSet); err != nil { - t.Errorf("POL.Verify() failed: %v", err) - } -} - -func TestVerifyInvalidVote(t *testing.T) { - height, round := uint(1), uint(0) - _, valSet, privValidators := randVoteSet(height, round, types.VoteTypePrevote, 10, 1) - - // Make a POL with +2/3 votes with the wrong signature. - blockHash := RandBytes(32) - pol := &POL{ - Height: height, Round: round, BlockHash: blockHash, - Votes: make([]POLVoteSignature, valSet.Size()), - } - voteProto := &types.Vote{ - Height: height, Round: round, Type: types.VoteTypePrevote, BlockHash: blockHash, - } - for i := 0; i < 7; i++ { - polVoteSig := signAddPOLVoteSignature(privValidators[i], valSet, voteProto, pol) - polVoteSig.Signature[0] += byte(0x01) // mutated! - } - - // Check that validation fails. - if err := pol.Verify(valSet); err == nil { - t.Errorf("Expected POL.Verify() to fail, wrong signatures.") - } -} - -func TestVerifyCommits(t *testing.T) { - height, round := uint(1), uint(2) - _, valSet, privValidators := randVoteSet(height, round, types.VoteTypePrevote, 10, 1) - - // Make a POL with +2/3 votes. - blockHash := RandBytes(32) - pol := &POL{ - Height: height, Round: round, BlockHash: blockHash, - Votes: make([]POLVoteSignature, valSet.Size()), - } - voteProto := &types.Vote{ - Height: height, Round: round - 1, Type: types.VoteTypeCommit, BlockHash: blockHash, - } - for i := 0; i < 7; i++ { - signAddPOLVoteSignature(privValidators[i], valSet, voteProto, pol) - } - - // Check that validation succeeds. - if err := pol.Verify(valSet); err != nil { - t.Errorf("POL.Verify() failed: %v", err) - } -} - -func TestVerifyInvalidCommits(t *testing.T) { - height, round := uint(1), uint(2) - _, valSet, privValidators := randVoteSet(height, round, types.VoteTypePrevote, 10, 1) - - // Make a POL with +2/3 votes with the wrong signature. - blockHash := RandBytes(32) - pol := &POL{ - Height: height, Round: round, BlockHash: blockHash, - Votes: make([]POLVoteSignature, valSet.Size()), - } - voteProto := &types.Vote{ - Height: height, Round: round - 1, Type: types.VoteTypeCommit, BlockHash: blockHash, - } - for i := 0; i < 7; i++ { - polVoteSig := signAddPOLVoteSignature(privValidators[i], valSet, voteProto, pol) - polVoteSig.Signature[0] += byte(0x01) - } - - // Check that validation fails. - if err := pol.Verify(valSet); err == nil { - t.Errorf("Expected POL.Verify() to fail, wrong signatures.") - } -} - -func TestVerifyInvalidCommitRounds(t *testing.T) { - height, round := uint(1), uint(2) - _, valSet, privValidators := randVoteSet(height, round, types.VoteTypePrevote, 10, 1) - - // Make a POL with +2/3 commits for the current round. - blockHash := RandBytes(32) - pol := &POL{ - Height: height, Round: round, BlockHash: blockHash, - Votes: make([]POLVoteSignature, valSet.Size()), - } - voteProto := &types.Vote{ - Height: height, Round: round, Type: types.VoteTypeCommit, BlockHash: blockHash, - } - for i := 0; i < 7; i++ { - signAddPOLVoteSignature(privValidators[i], valSet, voteProto, pol) - } - - // Check that validation fails. - if err := pol.Verify(valSet); err == nil { - t.Errorf("Expected POL.Verify() to fail, same round.") - } -} - -func TestVerifyInvalidCommitRounds2(t *testing.T) { - height, round := uint(1), uint(2) - _, valSet, privValidators := randVoteSet(height, round, types.VoteTypePrevote, 10, 1) - - // Make a POL with +2/3 commits for future round. - blockHash := RandBytes(32) - pol := &POL{ - Height: height, Round: round, BlockHash: blockHash, - Votes: make([]POLVoteSignature, valSet.Size()), - } - voteProto := &types.Vote{ - Height: height, Round: round + 1, Type: types.VoteTypeCommit, BlockHash: blockHash, - } - for i := 0; i < 7; i++ { - polVoteSig := signAddPOLVoteSignature(privValidators[i], valSet, voteProto, pol) - polVoteSig.Round += 1 // mutate round - } - - // Check that validation fails. - if err := pol.Verify(valSet); err == nil { - t.Errorf("Expected POL.Verify() to fail, future round.") - } -} - -func TestReadWrite(t *testing.T) { - height, round := uint(1), uint(2) - _, valSet, privValidators := randVoteSet(height, round, types.VoteTypePrevote, 10, 1) - - // Make a POL with +2/3 votes. - blockHash := RandBytes(32) - pol := &POL{ - Height: height, Round: round, BlockHash: blockHash, - Votes: make([]POLVoteSignature, valSet.Size()), - } - voteProto := &types.Vote{ - Height: height, Round: round, Type: types.VoteTypePrevote, BlockHash: blockHash, - } - for i := 0; i < 7; i++ { - signAddPOLVoteSignature(privValidators[i], valSet, voteProto, pol) - } - - // Write it to a buffer. - buf, n, err := new(bytes.Buffer), new(int64), new(error) - binary.WriteBinary(pol, buf, n, err) - if *err != nil { - t.Fatalf("Failed to write POL: %v", *err) - } - - // Read from buffer. - pol2 := binary.ReadBinary(&POL{}, buf, n, err).(*POL) - if *err != nil { - t.Fatalf("Failed to read POL: %v", *err) - } - - // Check that validation succeeds. - if err := pol2.Verify(valSet); err != nil { - t.Errorf("POL.Verify() failed: %v", err) - } -} diff --git a/consensus/state.go b/consensus/state.go index 78527f072..9dc4dd34f 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -2,55 +2,95 @@ Consensus State Machine Overview: -* Propose, Prevote, Precommit represent state machine stages. (aka RoundStep, or step). - Each take a predetermined amount of time depending on the round number. -* The Commit step can be entered by two means: - 1. After the Precommit step, +2/3 Precommits were found - 2. At any time, +2/3 Commits were found -* Once in the Commit stage, two conditions must both be satisfied - before proceeding to the next height NewHeight. -* The Propose step of the next height does not begin until - at least Delta duration *after* +2/3 Commits were found. - The step stays at NewHeight until this timeout occurs before - proceeding to Propose. -* The NewHeight is a transition period after the height is incremented, - where the node still receives late commits before potentially proposing. - The height should be incremented because a block had been - "committed by the network", and clients should see that - reflected as a new height. +* NewHeight, NewRound, Propose, Prevote, Precommit represent state machine steps. (aka RoundStep). +* To "prevote/precommit" something means to broadcast a prevote/precommit vote for something. +* During NewHeight/NewRound/Propose/Prevote/Precommit: + * Nodes gossip the locked proposal, if locked on a proposal. + * Nodes gossip the proposal proposed by the designated proposer for that round. + * Nodes gossip prevotes/precommits for rounds [0...currentRound+1] (currentRound+1 for catch-up) +* Upon each state transition, the height/round/step is broadcast to neighboring peers. +* The set of +2/3 of precommits from the same round for a block "commits the block". + +* NewRound: + * Set up new round. --> Then, goto Propose + +* Propose: + * Upon entering Propose: + * The designated proposer proposes a block. + * The Propose step ends: + * After `timeoutPropose` after entering Propose. --> Then, goto Prevote + * After receiving proposal block and POL prevotes are ready. --> Then, goto Prevote + * After any +2/3 prevotes received for the next round. --> Then, goto Prevote next round + * After any +2/3 precommits received for the next round. --> Then, goto Precommit next round + * After +2/3 precommits received for a particular block. --> Then, goto Commit + +* Prevote: + * Upon entering Prevote, each validator broadcasts its prevote vote. + * If the validator is locked on a block, it prevotes that. + * Else, if the proposed block from the previous step is good, it prevotes that. + * Else, if the proposal is invalid or wasn't received on time, it prevotes . + * The Prevote step ends: + * After +2/3 prevotes for a particular block or . --> Then, goto Precommit + * After `timeoutPrevote` after receiving any +2/3 prevotes. --> Then, goto Precommit + * After any +2/3 prevotes received for the next round. --> Then, goto Prevote next round + * After any +2/3 precommits received for the next round. --> Then, goto Precommit next round + * After +2/3 precommits received for a particular block. --> Then, goto Commit + +* Precommit: + * Upon entering Precommit, each validator broadcasts its precommit vote. + * If the validator had seen +2/3 of prevotes for a particular block, + it locks (changes lock to) that block and precommits that block. + * Else, if the validator had seen +2/3 of prevotes for nil, it unlocks and precommits . + * Else, if +2/3 of prevotes for a particular block or nil is not received on time, + it precommits what it's locked on, or . + * The Precommit step ends: + * After +2/3 precommits for a particular block. --> Then, goto Commit + * After +2/3 precommits for . --> Then, goto NewRound next round + * After `timeoutPrecommit` after receiving any +2/3 precommits. --> Then, goto NewRound next round + * After any +2/3 prevotes received for the next round. --> Then, goto Prevote next round + * After any +2/3 precommits received for the next round. --> Then, goto Precommit next round + +* Commit: + * Set CommitTime = now + * Wait until block is received, then goto NewHeight + +* NewHeight: + * Upon entering NewHeight, + * Move Precommits to LastPrecommits and increment height. + * Wait until `CommitTime+timeoutCommit` to receive straggler commits. --> Then, goto NewRound round 0 + +* Proof of Safety: + If a good validator commits at round R, it's because it saw +2/3 of precommits for round R. + This implies that (assuming tolerance bounds) +1/3 of honest nodes are still locked at round R+1. + These locked validators will remain locked until they see +2/3 prevote for something + else, but this won't happen because +1/3 are locked and honest. + +* Proof of Liveness: + Lemma 1: If +1/3 good nodes are locked on two different blocks, the proposers' POLRound will + eventually cause nodes locked from the earlier round to unlock. + -> `timeoutProposalR` increments with round R, while the block.size && POL prevote size + are fixed, so eventually we'll be able to "fully gossip" the block & POL. + TODO: cap the block.size at something reasonable. + Lemma 2: If a good node is at round R, neighboring good nodes will soon catch up to round R. +-------------------------------------+ - | | - v |(Wait til CommitTime + Delta) + v |(Wait til `CommmitTime+timeoutCommit`) +-----------+ +-----+-----+ +----------> | Propose +--------------+ | NewHeight | | +-----------+ | +-----------+ | | ^ - | | | - | | | - |(Else) v | + |(Else, after timeoutPrecommit) v | +-----+-----+ +-----------+ | | Precommit | <------------------------+ Prevote | | +-----+-----+ +-----------+ | - |(If +2/3 Precommits found) | - | | - | + (When +2/3 Commits found) | - | | | - v v | - +------------------------------------------------------------------------------+ - | Commit | | - | | | - | +----------------+ * Save Block | | - | |Get Block Parts |---> * Stage Block +--+ + | - | +----------------+ * Broadcast Commit | * Setup New Height | - | | * Move Commits set to | - | +--> LastCommits to continue | - | | collecting commits | - | +-----------------+ | * Broadcast New State | - | |Get +2/3 Commits |--> * Set CommitTime +--+ | - | +-----------------+ | - | | - +------------------------------------------------------------------------------+ + |(When +2/3 Precommits for block found) | + v | + +--------------------------------------------------------------------+ + | Commit | + | | + | * Set CommitTime = now; | + | * Wait for block, then stage/save/commit block; | + +--------------------------------------------------------------------+ */ @@ -60,7 +100,6 @@ import ( "bytes" "errors" "fmt" - "math" "sync" "sync/atomic" "time" @@ -76,15 +115,13 @@ import ( "github.com/tendermint/tendermint/types" ) -const ( - roundDeadlinePrevote = float64(1.0 / 3.0) // When the prevote is due. - roundDeadlinePrecommit = float64(2.0 / 3.0) // When the precommit vote is due. -) - var ( - RoundDuration0 = 10 * time.Second // The first round is 60 seconds long. - RoundDurationDelta = 3 * time.Second // Each successive round lasts 15 seconds longer. - newHeightDelta = RoundDuration0 / 3 // The time to wait between commitTime and startTime of next consensus rounds. + timeoutPropose = 3000 * time.Millisecond // Maximum duration of RoundStepPropose + timeoutPrevote0 = 1000 * time.Millisecond // After any +2/3 prevotes received, wait this long for stragglers. + timeoutPrevoteDelta = 0500 * time.Millisecond // timeoutPrevoteN is timeoutPrevote0 + timeoutPrevoteDelta*N + timeoutPrecommit0 = 1000 * time.Millisecond // After any +2/3 precommits received, wait this long for stragglers. + timeoutPrecommitDelta = 0500 * time.Millisecond // timeoutPrecommitN is timeoutPrecommit0 + timeoutPrecommitDelta*N + timeoutCommit = 2000 * time.Millisecond // After +2/3 commits received for committed block, wait this long for stragglers in the next height's RoundStepNewHeight. ) var ( @@ -97,12 +134,15 @@ var ( type RoundStepType uint8 // These must be numeric, ordered. const ( - RoundStepNewHeight = RoundStepType(0x01) // Round0 for new height started, wait til CommitTime + Delta - RoundStepNewRound = RoundStepType(0x02) // Pseudostep, immediately goes to RoundStepPropose - RoundStepPropose = RoundStepType(0x03) // Did propose, gossip proposal - RoundStepPrevote = RoundStepType(0x04) // Did prevote, gossip prevotes - RoundStepPrecommit = RoundStepType(0x05) // Did precommit, gossip precommits - RoundStepCommit = RoundStepType(0x06) // Entered commit state machine + RoundStepNewHeight = RoundStepType(0x01) // Wait til CommitTime + timeoutCommit + RoundStepNewRound = RoundStepType(0x02) // Setup new round and go to RoundStepPropose + RoundStepPropose = RoundStepType(0x03) // Did propose, gossip proposal + RoundStepPrevote = RoundStepType(0x04) // Did prevote, gossip prevotes + RoundStepPrevoteWait = RoundStepType(0x05) // Did receive any +2/3 prevotes, start timeout + RoundStepPrecommit = RoundStepType(0x06) // Did precommit, gossip precommits + RoundStepPrecommitWait = RoundStepType(0x07) // Did receive any +2/3 precommits, start timeout + RoundStepCommit = RoundStepType(0x08) // Entered commit state machine + // NOTE: RoundStepNewHeight acts as RoundStepCommitWait. ) func (rs RoundStepType) String() string { @@ -115,8 +155,12 @@ func (rs RoundStepType) String() string { return "RoundStepPropose" case RoundStepPrevote: return "RoundStepPrevote" + case RoundStepPrevoteWait: + return "RoundStepPrevoteWait" case RoundStepPrecommit: return "RoundStepPrecommit" + case RoundStepPrecommitWait: + return "RoundStepPrecommitWait" case RoundStepCommit: return "RoundStepCommit" default: @@ -124,51 +168,6 @@ func (rs RoundStepType) String() string { } } -//----------------------------------------------------------------------------- -// RoundAction enum type - -type RoundActionType string - -const ( - RoundActionPropose = RoundActionType("PR") // Propose and goto RoundStepPropose - RoundActionPrevote = RoundActionType("PV") // Prevote and goto RoundStepPrevote - RoundActionPrecommit = RoundActionType("PC") // Precommit and goto RoundStepPrecommit - RoundActionTryCommit = RoundActionType("TC") // Goto RoundStepCommit, or RoundStepPropose for next round. - RoundActionCommit = RoundActionType("CM") // Goto RoundStepCommit upon +2/3 commits - RoundActionTryFinalize = RoundActionType("TF") // Maybe goto RoundStepPropose for next round. -) - -func (rat RoundActionType) String() string { - switch rat { - case RoundActionPropose: - return "RoundActionPropose" - case RoundActionPrevote: - return "RoundActionPrevote" - case RoundActionPrecommit: - return "RoundActionPrecommit" - case RoundActionTryCommit: - return "RoundActionTryCommit" - case RoundActionCommit: - return "RoundActionCommit" - case RoundActionTryFinalize: - return "RoundActionTryFinalize" - default: - panic(Fmt("Unknown RoundAction %X", rat)) - } -} - -//----------------------------------------------------------------------------- - -type RoundAction struct { - Height uint // The block height for which consensus is reaching for. - Round uint // The round number at given height. - Action RoundActionType // Action to perform. -} - -func (ra RoundAction) String() string { - return Fmt("RoundAction{H:%v R:%v A:%v}", ra.Height, ra.Round, ra.Action) -} - //----------------------------------------------------------------------------- // Immutable when returned from ConsensusState.GetRoundState() @@ -177,20 +176,15 @@ type RoundState struct { Round uint Step RoundStepType StartTime time.Time - CommitTime time.Time // Time when +2/3 commits were found + CommitTime time.Time // Subjective time when +2/3 precommits for Block at Round were found Validators *sm.ValidatorSet Proposal *Proposal ProposalBlock *types.Block ProposalBlockParts *types.PartSet - ProposalPOL *POL - ProposalPOLParts *types.PartSet LockedBlock *types.Block LockedBlockParts *types.PartSet - LockedPOL *POL // Rarely needed, so no LockedPOLParts. - Prevotes *VoteSet - Precommits *VoteSet - Commits *VoteSet - LastCommits *VoteSet + Votes *HeightVoteSet + LastPrecommits *VoteSet // Last precommits for Height-1 } func (rs *RoundState) String() string { @@ -205,13 +199,9 @@ func (rs *RoundState) StringIndented(indent string) string { %s Validators: %v %s Proposal: %v %s ProposalBlock: %v %v -%s ProposalPOL: %v %v %s LockedBlock: %v %v -%s LockedPOL: %v -%s Prevotes: %v -%s Precommits: %v -%s Commits: %v -%s LastCommits: %v +%s Votes: %v +%s LastPrecommits: %v %s}`, indent, rs.Height, rs.Round, rs.Step, indent, rs.StartTime, @@ -219,13 +209,9 @@ func (rs *RoundState) StringIndented(indent string) string { indent, rs.Validators.StringIndented(indent+" "), indent, rs.Proposal, indent, rs.ProposalBlockParts.StringShort(), rs.ProposalBlock.StringShort(), - indent, rs.ProposalPOLParts.StringShort(), rs.ProposalPOL.StringShort(), indent, rs.LockedBlockParts.StringShort(), rs.LockedBlock.StringShort(), - indent, rs.LockedPOL.StringShort(), - indent, rs.Prevotes.StringIndented(indent+" "), - indent, rs.Precommits.StringIndented(indent+" "), - indent, rs.Commits.StringIndented(indent+" "), - indent, rs.LastCommits.StringShort(), + indent, rs.Votes.StringIndented(indent+" "), + indent, rs.LastPrecommits.StringShort(), indent) } @@ -245,15 +231,13 @@ type ConsensusState struct { blockStore *bc.BlockStore mempoolReactor *mempl.MempoolReactor privValidator *sm.PrivValidator - runActionCh chan RoundAction newStepCh chan *RoundState mtx sync.Mutex RoundState - state *sm.State // State until height-1. - stagedBlock *types.Block // Cache last staged block. - stagedState *sm.State // Cache result of staged block. - lastCommitVoteHeight uint // Last called commitVoteBlock() or saveCommitVoteBlock() on. + state *sm.State // State until height-1. + stagedBlock *types.Block // Cache last staged block. + stagedState *sm.State // Cache result of staged block. evsw events.Fireable evc *events.EventCache // set in stageBlock and passed into state @@ -264,40 +248,40 @@ func NewConsensusState(state *sm.State, blockStore *bc.BlockStore, mempoolReacto quit: make(chan struct{}), blockStore: blockStore, mempoolReactor: mempoolReactor, - runActionCh: make(chan RoundAction, 1), - newStepCh: make(chan *RoundState, 1), + newStepCh: make(chan *RoundState, 10), } cs.updateToState(state, true) - cs.reconstructLastCommits(state) + cs.maybeRebond() + cs.reconstructLastPrecommits(state) return cs } -// Reconstruct LastCommits from SeenValidation, which we saved along with the block, +// Reconstruct LastPrecommits from SeenValidation, which we saved along with the block, // (which happens even before saving the state) -func (cs *ConsensusState) reconstructLastCommits(state *sm.State) { +func (cs *ConsensusState) reconstructLastPrecommits(state *sm.State) { if state.LastBlockHeight == 0 { return } - lastCommits := NewVoteSet(state.LastBlockHeight, 0, types.VoteTypeCommit, state.LastBondedValidators) + lastPrecommits := NewVoteSet(state.LastBlockHeight, 0, types.VoteTypePrecommit, state.LastBondedValidators) seenValidation := cs.blockStore.LoadSeenValidation(state.LastBlockHeight) - for idx, commit := range seenValidation.Commits { - commitVote := &types.Vote{ + for idx, precommit := range seenValidation.Precommits { + precommitVote := &types.Vote{ Height: state.LastBlockHeight, - Round: commit.Round, - Type: types.VoteTypeCommit, + Round: seenValidation.Round, + Type: types.VoteTypePrecommit, BlockHash: state.LastBlockHash, BlockParts: state.LastBlockParts, - Signature: commit.Signature, + Signature: precommit.Signature, } - added, _, err := lastCommits.AddByIndex(uint(idx), commitVote) + added, _, err := lastPrecommits.AddByIndex(uint(idx), precommitVote) if !added || err != nil { - panic(Fmt("Failed to reconstruct LastCommits: %v", err)) + panic(Fmt("Failed to reconstruct LastPrecommits: %v", err)) } } - if !lastCommits.HasTwoThirdsMajority() { - panic("Failed to reconstruct LastCommits: Does not have +2/3 maj") + if !lastPrecommits.HasTwoThirdsMajority() { + panic("Failed to reconstruct LastPrecommits: Does not have +2/3 maj") } - cs.LastCommits = lastCommits + cs.LastPrecommits = lastPrecommits } func (cs *ConsensusState) GetState() *sm.State { @@ -324,10 +308,20 @@ func (cs *ConsensusState) NewStepCh() chan *RoundState { func (cs *ConsensusState) Start() { if atomic.CompareAndSwapUint32(&cs.started, 0, 1) { log.Info("Starting ConsensusState") - go cs.stepTransitionRoutine() + cs.scheduleRound0() } } +func (cs *ConsensusState) scheduleRound0(height uint) { + sleepDuration := cs.StartTime.Sub(time.Now()) + go func() { + if sleepDuration > 0 { + time.Sleep(sleepDuration) + } + cs.EnterNewRound(height, 0) + }() +} + func (cs *ConsensusState) Stop() { if atomic.CompareAndSwapUint32(&cs.stopped, 0, 1) { log.Info("Stopping ConsensusState") @@ -339,186 +333,23 @@ func (cs *ConsensusState) IsStopped() bool { return atomic.LoadUint32(&cs.stopped) == 1 } -func (cs *ConsensusState) queueAction(ra RoundAction) { - go func() { - cs.runActionCh <- ra - }() -} - -// Source of all round state transitions (and votes). -func (cs *ConsensusState) stepTransitionRoutine() { - - // For clarity, all state transitions that happen after some timeout are here. - // Schedule the next action by pushing a RoundAction{} to cs.runActionCh. - scheduleNextAction := func() { - // NOTE: rs must be fetched in the same callstack as the caller. - rs := cs.getRoundState() - go func() { - // NOTE: We can push directly to runActionCh because - // we're running in a separate goroutine, which avoids deadlocks. - round, roundStartTime, RoundDuration, _, elapsedRatio := calcRoundInfo(rs.StartTime) - log.Debug("Scheduling next action", "height", rs.Height, "round", round, "step", rs.Step, "roundStartTime", roundStartTime, "elapsedRatio", elapsedRatio) - switch rs.Step { - case RoundStepNewHeight: - // We should run RoundActionPropose when rs.StartTime passes. - if elapsedRatio < 0 { - // startTime is in the future. - time.Sleep(time.Duration((-1.0 * elapsedRatio) * float64(RoundDuration))) - } - cs.runActionCh <- RoundAction{rs.Height, rs.Round, RoundActionPropose} - case RoundStepNewRound: - // Pseudostep: Immediately goto propose. - cs.runActionCh <- RoundAction{rs.Height, rs.Round, RoundActionPropose} - case RoundStepPropose: - // Wake up when it's time to vote. - time.Sleep(time.Duration((roundDeadlinePrevote - elapsedRatio) * float64(RoundDuration))) - cs.runActionCh <- RoundAction{rs.Height, rs.Round, RoundActionPrevote} - case RoundStepPrevote: - // Wake up when it's time to precommit. - time.Sleep(time.Duration((roundDeadlinePrecommit - elapsedRatio) * float64(RoundDuration))) - cs.runActionCh <- RoundAction{rs.Height, rs.Round, RoundActionPrecommit} - case RoundStepPrecommit: - // Wake up when the round is over. - time.Sleep(time.Duration((1.0 - elapsedRatio) * float64(RoundDuration))) - cs.runActionCh <- RoundAction{rs.Height, rs.Round, RoundActionTryCommit} - case RoundStepCommit: - // There's nothing to scheudle, we're waiting for - // ProposalBlockParts.IsComplete() && - // Commits.HasTwoThirdsMajority() - panic("The next action from RoundStepCommit is not scheduled by time") - default: - panic("Should not happen") - } - }() - } - - if cs.getRoundState().Step < RoundStepCommit { - scheduleNextAction() - } else { - // Race condition with receipt of commits, maybe. - // We shouldn't have to schedule anything. - } - - // NOTE: All ConsensusState.RunAction*() calls come from here. - // Since only one routine calls them, it is safe to assume that - // the RoundState Height/Round/Step won't change concurrently. - // However, other fields like Proposal could change concurrent - // due to gossip routines. -ACTION_LOOP: - for { - var roundAction RoundAction - select { - case roundAction = <-cs.runActionCh: - case <-cs.quit: - return - } - - height, round, action := roundAction.Height, roundAction.Round, roundAction.Action - rs := cs.GetRoundState() - - // Continue if action is not relevant - if height != rs.Height { - log.Debug("Discarding round action: Height mismatch", "height", rs.Height, "roundAction", roundAction) - continue - } - // If action <= RoundActionPrecommit, the round must match too. - if action <= RoundActionPrecommit && round != rs.Round { - log.Debug("Discarding round action: Round mismatch", "round", rs.Round, "roundAction", roundAction) - continue - } - - log.Info("Running round action", "height", rs.Height, "round", rs.Round, "step", rs.Step, "roundAction", roundAction, "startTime", rs.StartTime) - - // Run action - switch action { - case RoundActionPropose: - if rs.Step != RoundStepNewHeight && rs.Step != RoundStepNewRound { - continue ACTION_LOOP - } - cs.RunActionPropose(rs.Height, rs.Round) - scheduleNextAction() - continue ACTION_LOOP - - case RoundActionPrevote: - if rs.Step >= RoundStepPrevote { - continue ACTION_LOOP - } - cs.RunActionPrevote(rs.Height, rs.Round) - scheduleNextAction() - continue ACTION_LOOP - - case RoundActionPrecommit: - if rs.Step >= RoundStepPrecommit { - continue ACTION_LOOP - } - cs.RunActionPrecommit(rs.Height, rs.Round) - scheduleNextAction() - continue ACTION_LOOP - - case RoundActionTryCommit: - if rs.Step >= RoundStepCommit { - continue ACTION_LOOP - } - if rs.Precommits.HasTwoThirdsMajority() { - // Enter RoundStepCommit and commit. - cs.RunActionCommit(rs.Height) - continue ACTION_LOOP - } else { - // Could not commit, move onto next round. - cs.SetupNewRound(rs.Height, rs.Round+1) - // cs.Step is now at RoundStepNewRound - scheduleNextAction() - continue ACTION_LOOP - } - - case RoundActionCommit: - if rs.Step >= RoundStepCommit { - continue ACTION_LOOP - } - // Enter RoundStepCommit and commit. - cs.RunActionCommit(rs.Height) - continue ACTION_LOOP - - case RoundActionTryFinalize: - if cs.TryFinalizeCommit(rs.Height) { - // Now at new height - // cs.Step is at RoundStepNewHeight or RoundStepNewRound. - // fire some events! - go func() { - newBlock := cs.blockStore.LoadBlock(cs.state.LastBlockHeight) - cs.evsw.FireEvent(types.EventStringNewBlock(), newBlock) - cs.evc.Flush() - }() - scheduleNextAction() - continue ACTION_LOOP - } else { - // do not schedule next action. - continue ACTION_LOOP - } - - default: - panic("Unknown action") - } - - // For clarity, ensure that all switch cases call "continue" - panic("Should not happen.") - } -} - // Updates ConsensusState and increments height to match that of state. -// If calculated round is greater than 0 (based on BlockTime or calculated StartTime) -// then also sets up the appropriate round, and cs.Step becomes RoundStepNewRound. -// Otherwise the round is 0 and cs.Step becomes RoundStepNewHeight. +// The round becomes 0 and cs.Step becomes RoundStepNewHeight. func (cs *ConsensusState) updateToState(state *sm.State, contiguous bool) { - // Sanity check state. + // SANITY CHECK if contiguous && cs.Height > 0 && cs.Height != state.LastBlockHeight { panic(Fmt("updateToState() expected state height of %v but found %v", cs.Height, state.LastBlockHeight)) } + // END SANITY CHECK // Reset fields based on state. validators := state.BondedValidators height := state.LastBlockHeight + 1 // next desired block height + lastPrecommits := (*VoteSet)(nil) + if contiguous && cs.Votes != nil { + lastPrecommits = cs.Votes.Precommits(cs.Round) + } // RoundState fields cs.Height = height @@ -526,86 +357,53 @@ func (cs *ConsensusState) updateToState(state *sm.State, contiguous bool) { cs.Step = RoundStepNewHeight if cs.CommitTime.IsZero() { // "Now" makes it easier to sync up dev nodes. - // We add newHeightDelta to allow transactions + // We add timeoutCommit to allow transactions // to be gathered for the first block. // And alternative solution that relies on clocks: - // cs.StartTime = state.LastBlockTime.Add(newHeightDelta) - cs.StartTime = time.Now().Add(newHeightDelta) + // cs.StartTime = state.LastBlockTime.Add(timeoutCommit) + cs.StartTime = time.Now().Add(timeoutCommit) } else { - cs.StartTime = cs.CommitTime.Add(newHeightDelta) + cs.StartTime = cs.CommitTime.Add(timeoutCommit) } cs.CommitTime = time.Time{} cs.Validators = validators cs.Proposal = nil cs.ProposalBlock = nil cs.ProposalBlockParts = nil - cs.ProposalPOL = nil - cs.ProposalPOLParts = nil cs.LockedBlock = nil cs.LockedBlockParts = nil - cs.LockedPOL = nil - cs.Prevotes = NewVoteSet(height, 0, types.VoteTypePrevote, validators) - cs.Precommits = NewVoteSet(height, 0, types.VoteTypePrecommit, validators) - cs.LastCommits = cs.Commits - cs.Commits = NewVoteSet(height, 0, types.VoteTypeCommit, validators) + cs.Votes = NewHeightVoteSet(height, validators) + cs.LastPrecommits = lastPrecommits cs.state = state cs.stagedBlock = nil cs.stagedState = nil +} - // Update the round if we need to. - round := calcRound(cs.StartTime) - if round > 0 { - cs.setupNewRound(round) +// If we're unbonded, broadcast RebondTx. +func (cs *ConsensusState) maybeRebond() { + if cs.privValidator == nil || !cs.state.UnbondingValidators.HasAddress(cs.privValidator.Address) { + return } - - // If we've timed out, then send rebond tx. - if cs.privValidator != nil && cs.state.UnbondingValidators.HasAddress(cs.privValidator.Address) { - rebondTx := &types.RebondTx{ - Address: cs.privValidator.Address, - Height: cs.Height, - } - err := cs.privValidator.SignRebondTx(cs.state.ChainID, rebondTx) - if err == nil { - err := cs.mempoolReactor.BroadcastTx(rebondTx) - if err != nil { - log.Error("Failed to broadcast RebondTx", - "height", cs.Height, "round", cs.Round, "tx", rebondTx, "error", err) - } else { - log.Info("Signed and broadcast RebondTx", - "height", cs.Height, "round", cs.Round, "tx", rebondTx) - } + rebondTx := &types.RebondTx{ + Address: cs.privValidator.Address, + Height: cs.Height, + } + err := cs.privValidator.SignRebondTx(cs.state.ChainID, rebondTx) + if err == nil { + err := cs.mempoolReactor.BroadcastTx(rebondTx) + if err != nil { + log.Error("Failed to broadcast RebondTx", + "height", cs.Height, "round", cs.Round, "tx", rebondTx, "error", err) } else { - log.Warn("Error signing RebondTx", "height", cs.Height, "round", cs.Round, "tx", rebondTx, "error", err) + log.Info("Signed and broadcast RebondTx", + "height", cs.Height, "round", cs.Round, "tx", rebondTx) } + } else { + log.Warn("Error signing RebondTx", "height", cs.Height, "round", cs.Round, "tx", rebondTx, "error", err) } } -// After the call cs.Step becomes RoundStepNewRound. -func (cs *ConsensusState) setupNewRound(round uint) { - // Sanity check - if round == 0 { - panic("setupNewRound() should never be called for round 0") - } - - // Increment all the way to round. - validators := cs.Validators.Copy() - validators.IncrementAccum(round - cs.Round) - - cs.Round = round - cs.Step = RoundStepNewRound - cs.Validators = validators - cs.Proposal = nil - cs.ProposalBlock = nil - cs.ProposalBlockParts = nil - cs.ProposalPOL = nil - cs.ProposalPOLParts = nil - cs.Prevotes = NewVoteSet(cs.Height, round, types.VoteTypePrevote, validators) - cs.Prevotes.AddFromCommits(cs.Commits) - cs.Precommits = NewVoteSet(cs.Height, round, types.VoteTypePrecommit, validators) - cs.Precommits.AddFromCommits(cs.Commits) -} - func (cs *ConsensusState) SetPrivValidator(priv *sm.PrivValidator) { cs.mtx.Lock() defer cs.mtx.Unlock() @@ -614,105 +412,96 @@ func (cs *ConsensusState) SetPrivValidator(priv *sm.PrivValidator) { //----------------------------------------------------------------------------- -// Set up the round to desired round and set step to RoundStepNewRound -func (cs *ConsensusState) SetupNewRound(height uint, desiredRound uint) bool { +// Enter: +2/3 precommits for nil from previous round +// Enter: `timeoutPrecommits` after any +2/3 precommits +// Enter: `commitTime+timeoutCommit` from NewHeight +// NOTE: cs.StartTime was already set for height. +func (cs *ConsensusState) EnterNewRound(height uint, round uint) { cs.mtx.Lock() defer cs.mtx.Unlock() - if cs.Height != height { - return false + if cs.Height != height || cs.Round >= round { + log.Debug(Fmt("EnterNewRound(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step)) + return } - if desiredRound <= cs.Round { - return false + if now := time.Now(); cs.StartTime.After(now) { + log.Warn("Need to set a buffer and log.Warn() here for sanity.", "startTime", cs.StartTime, "now", now) } - cs.setupNewRound(desiredRound) - // c.Step is now RoundStepNewRound - cs.newStepCh <- cs.getRoundState() - return true + + // Increment validators if necessary + if cs.Round < round { + validators := cs.Validators.Copy() + validators.IncrementAccum(round - cs.Round) + } + + // Setup new round + cs.Round = round + cs.Step = RoundStepNewRound + cs.Validators = validators + cs.Proposal = nil + cs.ProposalBlock = nil + cs.ProposalBlockParts = nil + cs.Votes.SetRound(round + 1) // track next round. + + // Immediately go to EnterPropose. + go cs.EnterPropose(height, round) } -func (cs *ConsensusState) RunActionPropose(height uint, round uint) { +// Enter: from NewRound. +func (cs *ConsensusState) EnterPropose(height uint, round uint) { cs.mtx.Lock() defer cs.mtx.Unlock() - if cs.Height != height || cs.Round != round { + if cs.Height != height || cs.Round > round || (cs.Round == round && cs.Step >= RoundStepPropose) { + log.Debug(Fmt("EnterPropose(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step)) return } + defer func() { + // Done EnterPropose: + cs.Round = round cs.Step = RoundStepPropose cs.newStepCh <- cs.getRoundState() + + // If we already have the proposal + POL, then goto Prevote + if cs.isProposalComplete() { + go cs.EnterPrevote(height, round) + } + }() + + // This step times out after `timeoutPropose` + go func() { + time.Sleep(timeoutPropose) + cs.EnterPrevote(height, round) }() // Nothing to do if it's not our turn. if cs.privValidator == nil { return } + + // See if it is our turn to propose if !bytes.Equal(cs.Validators.Proposer().Address, cs.privValidator.Address) { - log.Debug("Not our turn to propose", "proposer", cs.Validators.Proposer().Address, "privValidator", cs.privValidator) + log.Debug("EnterPropose: Not our turn to propose", "proposer", cs.Validators.Proposer().Address, "privValidator", cs.privValidator) return } else { - log.Debug("Our turn to propose", "proposer", cs.Validators.Proposer().Address, "privValidator", cs.privValidator) + log.Debug("EnterPropose: Our turn to propose", "proposer", cs.Validators.Proposer().Address, "privValidator", cs.privValidator) } + // We are going to propose a block. + var block *types.Block var blockParts *types.PartSet - var pol *POL - var polParts *types.PartSet - // Decide on block and POL + // Decide on block if cs.LockedBlock != nil { // If we're locked onto a block, just choose that. - block = cs.LockedBlock - blockParts = cs.LockedBlockParts - pol = cs.LockedPOL + block, blockParts = cs.LockedBlock, cs.LockedBlockParts } else { - // Otherwise we should create a new proposal. - var validation *types.Validation - if cs.Height == 1 { - // We're creating a proposal for the first block. - // The validation is empty. - validation = &types.Validation{} - } else if cs.LastCommits.HasTwoThirdsMajority() { - // Make the validation from LastCommits - validation = cs.LastCommits.MakeValidation() - } else { - // We just don't have any validation for the previous block - log.Debug("Cannot propose anything: No validation for the previous block.") - return - } - txs := cs.mempoolReactor.Mempool.GetProposalTxs() - block = &types.Block{ - Header: &types.Header{ - ChainID: cs.state.ChainID, - Height: cs.Height, - Time: time.Now(), - Fees: 0, // TODO fees - NumTxs: uint(len(txs)), - LastBlockHash: cs.state.LastBlockHash, - LastBlockParts: cs.state.LastBlockParts, - StateHash: nil, // Will set afterwards. - }, - Validation: validation, - Data: &types.Data{ - Txs: txs, - }, - } - - // Set the types.Header.StateHash. - err := cs.state.SetBlockStateHash(block) - if err != nil { - log.Error("Error setting state hash", "error", err) - return - } - - blockParts = block.MakePartSet() - pol = cs.LockedPOL // If exists, is a PoUnlock. - } - - if pol != nil { - polParts = pol.MakePartSet() + // Create a new proposal block from state/txs from the mempool. + block, blockParts = cs.createProposalBlock() } // Make proposal - proposal := NewProposal(cs.Height, cs.Round, blockParts.Header(), polParts.Header()) + proposal := NewProposal(cs.Height, cs.Round, blockParts.Header(), cs.Votes.POLRound()) err := cs.privValidator.SignProposal(cs.state.ChainID, proposal) if err == nil { log.Info("Signed and set proposal", "height", cs.Height, "round", cs.Round, "proposal", proposal) @@ -721,36 +510,102 @@ func (cs *ConsensusState) RunActionPropose(height uint, round uint) { cs.Proposal = proposal cs.ProposalBlock = block cs.ProposalBlockParts = blockParts - cs.ProposalPOL = pol - cs.ProposalPOLParts = polParts } else { - log.Warn("Error signing proposal", "height", cs.Height, "round", cs.Round, "error", err) + log.Warn("EnterPropose: Error signing proposal", "height", cs.Height, "round", cs.Round, "error", err) + } +} + +func (cs *ConsensusState) isProposalComplete() bool { + if cs.Proposal == nil || cs.ProposalBlock == nil { + return false + } + return cs.Votes.Prevote(cs.Proposal.POLRound).HasTwoThirdsMajority() +} + +// Create the next block to propose and return it. +// NOTE: make it side-effect free for clarity. +func (cs *ConsensusState) createProposalBlock() (*types.Block, *types.PartSet) { + var validation *types.Validation + if cs.Height == 1 { + // We're creating a proposal for the first block. + // The validation is empty. + validation = &types.Validation{} + } else if cs.LastPrecommits.HasTwoThirdsMajority() { + // Make the validation from LastPrecommits + validation = cs.LastPrecommits.MakeValidation() + } else { + // This shouldn't happen. + log.Error("EnterPropose: Cannot propose anything: No validation for the previous block.") + return + } + txs := cs.mempoolReactor.Mempool.GetProposalTxs() + block = &types.Block{ + Header: &types.Header{ + ChainID: cs.state.ChainID, + Height: cs.Height, + Time: time.Now(), + Fees: 0, // TODO fees + NumTxs: uint(len(txs)), + LastBlockHash: cs.state.LastBlockHash, + LastBlockParts: cs.state.LastBlockParts, + StateHash: nil, // Will set afterwards. + }, + Validation: validation, + Data: &types.Data{ + Txs: txs, + }, + } + + // Set the block.Header.StateHash. + err := cs.state.ComputeBlockStateHash(block) + if err != nil { + log.Error("EnterPropose: Error setting state hash", "error", err) + return } + + blockParts = block.MakePartSet() + return block, blockParts } +// Enter: `timeoutPropose` after start of Propose. +// Enter: proposal block and POL is ready. +// Enter: any +2/3 prevotes for next round. // Prevote for LockedBlock if we're locked, or ProposealBlock if valid. // Otherwise vote nil. -func (cs *ConsensusState) RunActionPrevote(height uint, round uint) { +func (cs *ConsensusState) EnterPrevote(height uint, round uint) { cs.mtx.Lock() defer cs.mtx.Unlock() - if cs.Height != height || cs.Round != round { - panic(Fmt("RunActionPrevote(%v/%v), expected %v/%v", height, round, cs.Height, cs.Round)) + if cs.Height != height || cs.Round > round || (cs.Round == round && cs.Step >= RoundStepPrevote) { + log.Debug(Fmt("EnterPrevote(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step)) + return } + defer func() { + // Done EnterPrevote: + cs.Round = round cs.Step = RoundStepPrevote cs.newStepCh <- cs.getRoundState() + // Maybe immediately go to EnterPrevoteWait. + if cs.Votes.Prevotes(round).HasTwoThirdsAny() { + go cs.EnterPrevoteWait(height, round) + } }() + // Sign and broadcast vote as necessary + cs.doPrevote(height, round) +} + +func (cs *ConsensusState) doPrevote(height uint, round uint) { // If a block is locked, prevote that. if cs.LockedBlock != nil { - log.Debug("Block was locked") + log.Debug("EnterPrevote: Block was locked") cs.signAddVote(types.VoteTypePrevote, cs.LockedBlock.Hash(), cs.LockedBlockParts.Header()) return } // If ProposalBlock is nil, prevote nil. if cs.ProposalBlock == nil { - log.Warn("ProposalBlock is nil") + log.Warn("EnterPrevote: ProposalBlock is nil") cs.signAddVote(types.VoteTypePrevote, nil, types.PartSetHeader{}) return } @@ -759,7 +614,7 @@ func (cs *ConsensusState) RunActionPrevote(height uint, round uint) { err := cs.stageBlock(cs.ProposalBlock, cs.ProposalBlockParts) if err != nil { // ProposalBlock is invalid, prevote nil. - log.Warn("ProposalBlock is invalid", "error", err) + log.Warn("EnterPrevote: ProposalBlock is invalid", "error", err) cs.signAddVote(types.VoteTypePrevote, nil, types.PartSetHeader{}) return } @@ -769,107 +624,181 @@ func (cs *ConsensusState) RunActionPrevote(height uint, round uint) { return } -// Lock & Precommit the ProposalBlock if we have enough prevotes for it, -// or unlock an existing lock if +2/3 of prevotes were nil. -func (cs *ConsensusState) RunActionPrecommit(height uint, round uint) { +// Enter: any +2/3 prevotes for next round. +func (cs *ConsensusState) EnterPrevoteWait(height uint, round uint) { + cs.mtx.Lock() + defer cs.mtx.Unlock() + if cs.Height != height || cs.Round > round || (cs.Round == round && cs.Step >= RoundStepPrevoteWait) { + log.Debug(Fmt("EnterPrevoteWait(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step)) + return + } + if !cs.Votes.Prevotes(round).HasTwoThirdsAny() { + panic(Fmt("EnterPrevoteWait(%v/%v), but Prevotes does not have any +2/3 votes", height, round)) + } + + // Done EnterPrevoteWait: + cs.Round = round + cs.Step = RoundStepPrevoteWait + cs.newStepCh <- cs.getRoundState() + + // After `timeoutPrevote`, EnterPrecommit() + go func() { + time.Sleep(timeoutPrevote) + cs.EnterPrecommit(height, round) + }() +} + +// Enter: +2/3 precomits for block or nil. +// Enter: `timeoutPrevote` after any +2/3 prevotes. +// Enter: any +2/3 precommits for next round. +// Lock & precommit the ProposalBlock if we have enough prevotes for it, +// else, unlock an existing lock and precommit nil if +2/3 of prevotes were nil, +// else, precommit locked block or nil otherwise. +func (cs *ConsensusState) EnterPrecommit(height uint, round uint) { cs.mtx.Lock() defer cs.mtx.Unlock() - if cs.Height != height || cs.Round != round { - panic(Fmt("RunActionPrecommit(%v/%v), expected %v/%v", height, round, cs.Height, cs.Round)) + if cs.Height != height || cs.Round > round || (cs.Round == round && cs.Step >= RoundStepPrecommit) { + log.Debug(Fmt("EnterPrecommit(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step)) + return } + defer func() { + // Done EnterPrecommit: + cs.Round = round cs.Step = RoundStepPrecommit cs.newStepCh <- cs.getRoundState() + // Maybe immediately go to EnterPrecommitWait. + if cs.Votes.Precommits(round).HasTwoThirdsAny() { + go cs.EnterPrecommitWait(height, round) + } }() - hash, partsHeader, ok := cs.Prevotes.TwoThirdsMajority() + hash, partsHeader, ok := cs.Votes.Prevotes(round).TwoThirdsMajority() + + // If we don't have two thirds of prevotes, just precommit locked block or nil if !ok { - // If we don't have two thirds of prevotes, - // don't do anything at all. - log.Info("Insufficient prevotes for precommit") + if cs.LockedBlock != nil { + log.Info("EnterPrecommit: No +2/3 prevotes during EnterPrecommit. Precommitting lock.") + cs.signAddVote(types.VoteTypePrecommit, cs.LockedBlock.Hash(), cs.LockedBlockParts.Header()) + } else { + log.Info("EnterPrecommit: No +2/3 prevotes during EnterPrecommit. Precommitting nil.") + cs.signAddVote(types.VoteTypePrecommit, nil, types.PartSetHeader{}) + } return } - // Remember this POL. (hash may be nil) - cs.LockedPOL = cs.Prevotes.MakePOL() - - // If +2/3 prevoted nil. Just unlock. + // +2/3 prevoted nil. Unlock and precommit nil. if len(hash) == 0 { if cs.LockedBlock == nil { - log.Info("+2/3 prevoted for nil.") + log.Info("EnterPrecommit: +2/3 prevoted for nil.") } else { - log.Info("+2/3 prevoted for nil. Unlocking") + log.Info("EnterPrecommit: +2/3 prevoted for nil. Unlocking") cs.LockedBlock = nil cs.LockedBlockParts = nil } + cs.signAddVote(types.VoteTypePrecommit, nil, types.PartSetHeader{}) return } + // At this point, +2/3 prevoted for a particular block. + // If +2/3 prevoted for already locked block, precommit it. if cs.LockedBlock.HashesTo(hash) { - log.Info("+2/3 prevoted locked block.") + log.Info("EnterPrecommit: +2/3 prevoted locked block.") cs.signAddVote(types.VoteTypePrecommit, hash, partsHeader) return } - // If +2/3 prevoted for cs.ProposalBlock, lock it and precommit it. - if !cs.ProposalBlock.HashesTo(hash) { - log.Warn("Proposal does not hash to +2/3 prevotes") - // We don't have the block that validators prevoted. - // Unlock if we're locked. - cs.LockedBlock = nil - cs.LockedBlockParts = nil + // If +2/3 prevoted for proposal block, stage and precommit it + if cs.ProposalBlock.HashesTo(hash) { + log.Info("EnterPrecommit: +2/3 prevoted proposal block.") + // Validate the block. + if err := cs.stageBlock(cs.ProposalBlock, cs.ProposalBlockParts); err != nil { + panic(Fmt("EnterPrecommit: +2/3 prevoted for an invalid block: %v", err)) + } + cs.LockedBlock = cs.ProposalBlock + cs.LockedBlockParts = cs.ProposalBlockParts + cs.signAddVote(types.VoteTypePrecommit, hash, partsHeader) return } - // Validate the block. - if err := cs.stageBlock(cs.ProposalBlock, cs.ProposalBlockParts); err != nil { - // Prevent zombies. - log.Warn("+2/3 prevoted for an invalid block", "error", err) + // Otherwise, we need to fetch the +2/3 prevoted block. + // We don't have the block yet so we can't lock/precommit it. + cs.LockedBlock = nil + cs.LockedBlockParts = nil + if !cs.ProposalBlockParts.HasHeader(partsHeader) { + cs.ProposalBlock = nil + cs.ProposalBlockParts = types.NewPartSetFromHeader(partsHeader) + } + cs.signAddVote(types.VoteTypePrecommit, nil, PartSetHeader{}) + return +} + +// Enter: any +2/3 precommits for next round. +func (cs *ConsensusState) EnterPrecommitWait(height uint, round uint) { + cs.mtx.Lock() + defer cs.mtx.Unlock() + if cs.Height != height || cs.Round > round || (cs.Round == round && cs.Step >= RoundStepPrecommitWait) { + log.Debug(Fmt("EnterPrecommitWait(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step)) return } + if !cs.Votes.Precommits(round).HasTwoThirdsAny() { + panic(Fmt("EnterPrecommitWait(%v/%v), but Precommits does not have any +2/3 votes", height, round)) + } - cs.LockedBlock = cs.ProposalBlock - cs.LockedBlockParts = cs.ProposalBlockParts - cs.signAddVote(types.VoteTypePrecommit, hash, partsHeader) - return + // Done EnterPrecommitWait: + cs.Round = round + cs.Step = RoundStepPrecommitWait + cs.newStepCh <- cs.getRoundState() + + // After `timeoutPrecommit`, EnterNewRound() + go func() { + time.Sleep(timeoutPrecommit) + // If we have +2/3 of precommits for a particular block (or nil), + // we already entered commit (or the next round). + // So just try to transition to the next round, + // which is what we'd do otherwise. + cs.EnterNewRound(height, round+1) + }() } -// Enter commit step. See the diagram for details. -// There are two ways to enter this step: -// * After the Precommit step with +2/3 precommits, or, -// * Upon +2/3 commits regardless of current step -// Either way this action is run at most once per round. -func (cs *ConsensusState) RunActionCommit(height uint) { +// Enter: +2/3 precommits for block +func (cs *ConsensusState) EnterCommit(height uint) { cs.mtx.Lock() defer cs.mtx.Unlock() - if cs.Height != height { - panic(Fmt("RunActionCommit(%v), expected %v", height, cs.Height)) + if cs.Height != height || cs.Step >= RoundStepCommit { + log.Debug(Fmt("EnterCommit(%v): Invalid args. Current step: %v/%v/%v", height, cs.Height, cs.Round, cs.Step)) + return } + defer func() { + // Done Entercommit: + // keep ca.Round the same, it points to the right Precommits set. cs.Step = RoundStepCommit cs.newStepCh <- cs.getRoundState() + + // Maybe finalize immediately. + cs.TryFinalizeCommit(height) }() - // Sanity check. - // There are two ways to enter: - // 1. +2/3 precommits at the end of RoundStepPrecommit - // 2. +2/3 commits at any time + // SANITY CHECK hash, partsHeader, ok := cs.Precommits.TwoThirdsMajority() if !ok { - hash, partsHeader, ok = cs.Commits.TwoThirdsMajority() - if !ok { - panic("RunActionCommit() expects +2/3 precommits or commits") - } + panic("RunActionCommit() expects +2/3 precommits") } + // END SANITY CHECK - // Clear the Locked* fields and use cs.Proposed* + // The Locked* fields no longer matter. + // Move them over to ProposalBlock if they match the commit hash, + // otherwise they can now be cleared. if cs.LockedBlock.HashesTo(hash) { cs.ProposalBlock = cs.LockedBlock cs.ProposalBlockParts = cs.LockedBlockParts cs.LockedBlock = nil cs.LockedBlockParts = nil - cs.LockedPOL = nil + } else { + cs.LockedBlock = nil + cs.LockedBlockParts = nil } // If we don't have the block being committed, set up to get it. @@ -879,7 +808,6 @@ func (cs *ConsensusState) RunActionCommit(height uint) { // Set up ProposalBlockParts and keep waiting. cs.ProposalBlock = nil cs.ProposalBlockParts = types.NewPartSetFromHeader(partsHeader) - } else { // We just need to keep waiting. } @@ -887,59 +815,66 @@ func (cs *ConsensusState) RunActionCommit(height uint) { // We have the block, so sign a Commit-vote. cs.commitVoteBlock(cs.ProposalBlock, cs.ProposalBlockParts) } +} - // If we have the block AND +2/3 commits, queue RoundActionTryFinalize. - // Round will immediately become finalized. +// If we have the block AND +2/3 commits for it, finalize. +func (cs *ConsensusState) TryFinalizeCommit(height uint) { if cs.ProposalBlock.HashesTo(hash) && cs.Commits.HasTwoThirdsMajority() { - cs.queueAction(RoundAction{cs.Height, cs.Round, RoundActionTryFinalize}) + go cs.FinalizeCommit(height) } - } -// Returns true if Finalize happened, which increments height && sets -// the step to RoundStepNewHeight (or RoundStepNewRound, but probably not). -func (cs *ConsensusState) TryFinalizeCommit(height uint) bool { +// Increment height and goto RoundStepNewHeight +func (cs *ConsensusState) FinalizeCommit(height uint) { cs.mtx.Lock() defer cs.mtx.Unlock() - if cs.Height != height { - panic(Fmt("TryFinalizeCommit(%v), expected %v", height, cs.Height)) + if cs.Height != height || cs.Step != RoundStepCommit { + log.Debug(Fmt("FinalizeCommit(%v): Invalid args. Current step: %v/%v/%v", height, cs.Height, cs.Round, cs.Step)) + return } - if cs.Step == RoundStepCommit && - cs.Commits.HasTwoThirdsMajority() && - cs.ProposalBlockParts.IsComplete() { - - // Sanity check - if cs.ProposalBlock == nil { - panic(Fmt("Expected ProposalBlock to exist")) - } - hash, header, _ := cs.Commits.TwoThirdsMajority() - if !cs.ProposalBlock.HashesTo(hash) { - // XXX See: https://github.com/tendermint/tendermint/issues/44 - panic(Fmt("Expected ProposalBlock to hash to commit hash. Expected %X, got %X", hash, cs.ProposalBlock.Hash())) - } - if !cs.ProposalBlockParts.HasHeader(header) { - panic(Fmt("Expected ProposalBlockParts header to be commit header")) - } + hash, header, ok := cs.Commits.TwoThirdsMajority() - err := cs.stageBlock(cs.ProposalBlock, cs.ProposalBlockParts) - if err == nil { - log.Debug(Fmt("Finalizing commit of block: %v", cs.ProposalBlock)) - // We have the block, so save/stage/sign-commit-vote. - cs.saveCommitVoteBlock(cs.ProposalBlock, cs.ProposalBlockParts, cs.Commits) - // Increment height. - cs.updateToState(cs.stagedState, true) - // cs.Step is now RoundStepNewHeight or RoundStepNewRound - cs.newStepCh <- cs.getRoundState() - return true - } else { - // Prevent zombies. - // TODO: Does this ever happen? - panic(Fmt("+2/3 committed an invalid block: %v", err)) - } + // SANITY CHECK + if !ok { + panic(Fmt("Cannot FinalizeCommit, commit does not have two thirds majority")) + } + if !cs.ProposalBlockParts.HasHeader(header) { + panic(Fmt("Expected ProposalBlockParts header to be commit header")) + } + if !cs.ProposalBlock.HashesTo(hash) { + panic(Fmt("Cannot FinalizeCommit, ProposalBlock does not hash to commit hash")) + } + if err := cs.stageBlock(cs.ProposalBlock, cs.ProposalBlockParts); err != nil { + panic(Fmt("+2/3 committed an invalid block: %v", err)) } - return false + // END SANITY CHECK + + log.Debug(Fmt("Finalizing commit of block: %v", cs.ProposalBlock)) + // We have the block, so stage/save/commit-vote. + cs.saveBlock(cs.ProposalBlock, cs.ProposalBlockParts, cs.Commits) + cs.commitVoteBlock(cs.ProposalBlock, cs.ProposalBlockParts, cs.Commits) + // Fire off event + go func() { + cs.evsw.FireEvent(types.EventStringNewBlock(), cs.ProposalBlock) + cs.evc.Flush() + }(cs.ProposalBlock) + + // Increment height. + cs.updateToState(cs.stagedState, true) + + // If we're unbonded, broadcast RebondTx. + cs.maybeRebond() + + // By here, + // * cs.Height has been increment to height+1 + // * cs.Step is now RoundStepNewHeight + // * cs.StartTime is set to when we should start round0. + cs.newStepCh <- cs.getRoundState() + // Start round 0 when cs.StartTime. + go cs.scheduleRound0(height) + return } //----------------------------------------------------------------------------- @@ -959,7 +894,7 @@ func (cs *ConsensusState) SetProposal(proposal *Proposal) error { } // We don't care about the proposal if we're already in RoundStepCommit. - if cs.Step == RoundStepCommit { + if cs.Step >= RoundStepCommit { return nil } @@ -970,12 +905,10 @@ func (cs *ConsensusState) SetProposal(proposal *Proposal) error { cs.Proposal = proposal cs.ProposalBlockParts = types.NewPartSetFromHeader(proposal.BlockParts) - cs.ProposalPOLParts = types.NewPartSetFromHeader(proposal.POLParts) return nil } // NOTE: block is not necessarily valid. -// NOTE: This function may increment the height. func (cs *ConsensusState) AddProposalBlockPart(height uint, round uint, part *types.Part) (added bool, err error) { cs.mtx.Lock() defer cs.mtx.Unlock() @@ -999,38 +932,13 @@ func (cs *ConsensusState) AddProposalBlockPart(height uint, round uint, part *ty var err error cs.ProposalBlock = binary.ReadBinary(&types.Block{}, cs.ProposalBlockParts.GetReader(), &n, &err).(*types.Block) log.Debug("Received complete proposal", "hash", cs.ProposalBlock.Hash()) - // If we're already in the commit step, try to finalize round. - if cs.Step == RoundStepCommit { - cs.queueAction(RoundAction{cs.Height, cs.Round, RoundActionTryFinalize}) + if cs.Step == RoundStepPropose { + if cs.isProposalComplete() { + go cs.EnterPrevote(height, round) + } + } else if cs.Step == RoundStepCommit { + cs.TryFinalizeCommit(height) } - // XXX If POL is valid, consider unlocking. - return true, err - } - return true, nil -} - -// NOTE: POL is not necessarily valid. -func (cs *ConsensusState) AddProposalPOLPart(height uint, round uint, part *types.Part) (added bool, err error) { - cs.mtx.Lock() - defer cs.mtx.Unlock() - - if cs.Height != height || cs.Round != round { - return false, nil - } - - // We're not expecting a POL part. - if cs.ProposalPOLParts == nil { - return false, nil // TODO: bad peer? Return error? - } - - added, err = cs.ProposalPOLParts.AddPart(part) - if err != nil { - return added, err - } - if added && cs.ProposalPOLParts.IsComplete() { - var n int64 - var err error - cs.ProposalPOL = binary.ReadBinary(&POL{}, cs.ProposalPOLParts.GetReader(), &n, &err).(*POL) return true, err } return true, nil @@ -1046,54 +954,78 @@ func (cs *ConsensusState) AddVote(address []byte, vote *types.Vote) (added bool, //----------------------------------------------------------------------------- func (cs *ConsensusState) addVote(address []byte, vote *types.Vote) (added bool, index uint, err error) { - switch vote.Type { - case types.VoteTypePrevote: - // Prevotes checks for height+round match. - added, index, err = cs.Prevotes.AddByAddress(address, vote) + // A precommit for the previous height? + if vote.Height+1 == cs.Height && vote.Type == types.VoteTypePrecommit { + added, index, err = cs.LastPrecommits.AddByAddress(address, vote) if added { - log.Debug(Fmt("Added prevote: %v", cs.Prevotes.StringShort())) + log.Debug(Fmt("Added to lastPrecommits: %v", cs.LastPrecommits.StringShort())) } return - case types.VoteTypePrecommit: - // Precommits checks for height+round match. - added, index, err = cs.Precommits.AddByAddress(address, vote) + } + + // A prevote/precommit for this height? + if vote.Height == cs.Height { + added, index, err = cs.Votes.AddByAddress(address, vote) if added { - log.Debug(Fmt("Added precommit: %v", cs.Precommits.StringShort())) - } - return - case types.VoteTypeCommit: - if vote.Height == cs.Height { - // No need to check if vote.Round < cs.Round ... - // Prevotes && Precommits already checks that. - cs.Prevotes.AddByAddress(address, vote) - cs.Precommits.AddByAddress(address, vote) - added, index, err = cs.Commits.AddByAddress(address, vote) - if added && cs.Commits.HasTwoThirdsMajority() && cs.CommitTime.IsZero() { - cs.CommitTime = time.Now() - log.Debug(Fmt("Set CommitTime to %v", cs.CommitTime)) - if cs.Step < RoundStepCommit { - cs.queueAction(RoundAction{cs.Height, cs.Round, RoundActionCommit}) - } else { - cs.queueAction(RoundAction{cs.Height, cs.Round, RoundActionTryFinalize}) + switch vote.Type { + case types.VoteTypePrevote: + log.Debug(Fmt("Added to prevotes: %v", cs.Votes.Prevotes(vote.Round).StringShort())) + if cs.Round < vote.Round && cs.Votes.Prevotes(vote.Round).HasTwoThirdsAny() { + // Goto to Prevote next round. + go func() { + cs.EnterNewRound(height, vote.Round) + cs.EnterPrevote(height, vote.Round) + }() } + if cs.Round == vote.Round { + if cs.Votes.Prevotes(cs.Round).HasTwoThirdsMajority() { + // Goto Precommit, whether for block or nil. + go func() { + cs.EnterPrecommit(height, cs.Round) + }() + } + if cs.Votes.Prevotes(cs.Round).HasTwoThirdsAny() { + // Goto PrevoteWait + go func() { + cs.EnterPrevoteWait(height, cs.Round) + }() + } + } + case types.VoteTypePrecommit: + log.Debug(Fmt("Added to precommit: %v", cs.Votes.Precommits(vote.Round).StringShort())) + if cs.Round < vote.Round && cs.Votes.Precommits(vote.Round).HasTwoThirdsAny() { + // Skip to Precommit next round. + go func() { + cs.EnterNewRound(height, vote.Round) + cs.EnterPrecommit(height, vote.Round) + }() + } + if cs.Round == vote.Round { + if hash, _, ok := cs.Votes.Precommits(cs.Round).TwoThirdsMajority(); ok { + if len(hash) == 0 { + // If hash is nil, goto NewRound + cs.EnterNewRound(height, cs.Round+1) + } else { + // If hash is block, goto Commit + cs.EnterCommit(height, cs.Round) + } + } + if cs.Votes.Precommits(cs.Round).HasTwoThirdsAny() { + // Goto PrecommitWait + go func() { + cs.EnterPrecommitWait(height, cs.Round) + }() + } + } + default: + panic(Fmt("Unexpected vote type %X", vote.Type)) // Should not happen. } - if added { - log.Debug(Fmt("Added commit: %v\nprecommits: %v\nprevotes: %v", - cs.Commits.StringShort(), - cs.Precommits.StringShort(), - cs.Prevotes.StringShort())) - } - return } - if vote.Height+1 == cs.Height { - added, index, err = cs.LastCommits.AddByAddress(address, vote) - log.Debug(Fmt("Added lastCommits: %v", cs.LastCommits.StringShort())) - return - } - return false, 0, nil - default: - panic("Unknown vote type") + return } + + // Height mismatch, bad peer? TODO + return } func (cs *ConsensusState) stageBlock(block *types.Block, blockParts *types.PartSet) error { @@ -1147,34 +1079,12 @@ func (cs *ConsensusState) signAddVote(type_ byte, hash []byte, header types.Part } } -// sign a Commit-Vote -func (cs *ConsensusState) commitVoteBlock(block *types.Block, blockParts *types.PartSet) { - - // The proposal must be valid. - if err := cs.stageBlock(block, blockParts); err != nil { - // Prevent zombies. - log.Warn("commitVoteBlock() an invalid block", "error", err) - return - } - - // Commit-vote. - if cs.lastCommitVoteHeight < block.Height { - cs.signAddVote(types.VoteTypeCommit, block.Hash(), blockParts.Header()) - cs.lastCommitVoteHeight = block.Height - } else { - log.Error("Duplicate commitVoteBlock() attempt", "lastCommitVoteHeight", cs.lastCommitVoteHeight, "types.Height", block.Height) - } -} - -// Save Block, save the +2/3 Commits we've seen, -// and sign a Commit-Vote if we haven't already -func (cs *ConsensusState) saveCommitVoteBlock(block *types.Block, blockParts *types.PartSet, commits *VoteSet) { +// Save Block, save the +2/3 Commits we've seen +func (cs *ConsensusState) saveBlock(block *types.Block, blockParts *types.PartSet, commits *VoteSet) { // The proposal must be valid. if err := cs.stageBlock(block, blockParts); err != nil { - // Prevent zombies. - log.Warn("saveCommitVoteBlock() an invalid block", "error", err) - return + panic(Fmt("saveBlock() an invalid block: %v", err)) } // Save to blockStore. @@ -1189,66 +1099,9 @@ func (cs *ConsensusState) saveCommitVoteBlock(block *types.Block, blockParts *ty // Update mempool. cs.mempoolReactor.Mempool.ResetForBlockAndState(block, cs.stagedState) - // Commit-vote if we haven't already. - if cs.lastCommitVoteHeight < block.Height { - cs.signAddVote(types.VoteTypeCommit, block.Hash(), blockParts.Header()) - cs.lastCommitVoteHeight = block.Height - } } // implements events.Eventable func (cs *ConsensusState) SetFireable(evsw events.Fireable) { cs.evsw = evsw } - -//----------------------------------------------------------------------------- - -// total duration of given round -func calcRoundDuration(round uint) time.Duration { - return RoundDuration0 + RoundDurationDelta*time.Duration(round) -} - -// startTime is when round zero started. -func calcRoundStartTime(round uint, startTime time.Time) time.Time { - return startTime.Add(RoundDuration0*time.Duration(round) + - RoundDurationDelta*(time.Duration((int64(round)*int64(round)-int64(round))/2))) -} - -// calculates the current round given startTime of round zero. -// NOTE: round is zero if startTime is in the future. -func calcRound(startTime time.Time) uint { - now := time.Now() - if now.Before(startTime) { - return 0 - } - // Start + D_0 * R + D_delta * (R^2 - R)/2 <= Now; find largest integer R. - // D_delta * R^2 + (2D_0 - D_delta) * R + 2(Start - Now) <= 0. - // AR^2 + BR + C <= 0; A = D_delta, B = (2_D0 - D_delta), C = 2(Start - Now). - // R = Floor((-B + Sqrt(B^2 - 4AC))/2A) - A := float64(RoundDurationDelta) - B := 2.0*float64(RoundDuration0) - float64(RoundDurationDelta) - C := 2.0 * float64(startTime.Sub(now)) - R := math.Floor((-B + math.Sqrt(B*B-4.0*A*C)) / (2 * A)) - if math.IsNaN(R) { - panic("Could not calc round, should not happen") - } - if R > math.MaxInt32 { - panic(Fmt("Could not calc round, round overflow: %v", R)) - } - if R < 0 { - return 0 - } - return uint(R) -} - -// convenience -// NOTE: elapsedRatio can be negative if startTime is in the future. -func calcRoundInfo(startTime time.Time) (round uint, roundStartTime time.Time, RoundDuration time.Duration, - roundElapsed time.Duration, elapsedRatio float64) { - round = calcRound(startTime) - roundStartTime = calcRoundStartTime(round, startTime) - RoundDuration = calcRoundDuration(round) - roundElapsed = time.Now().Sub(roundStartTime) - elapsedRatio = float64(roundElapsed) / float64(RoundDuration) - return -} diff --git a/consensus/state_test.go b/consensus/state_test.go index af25ae016..48e8bb503 100644 --- a/consensus/state_test.go +++ b/consensus/state_test.go @@ -8,51 +8,6 @@ import ( "github.com/tendermint/tendermint/types" ) -func TestSetupRound(t *testing.T) { - cs, privValidators := randConsensusState() - val0 := privValidators[0] - - // Add a vote, precommit, and commit by val0. - voteTypes := []byte{types.VoteTypePrevote, types.VoteTypePrecommit, types.VoteTypeCommit} - for _, voteType := range voteTypes { - vote := &types.Vote{Height: 1, Round: 0, Type: voteType} // nil vote - err := val0.SignVote(cs.state.ChainID, vote) - if err != nil { - t.Error("Error signing vote: %v", err) - } - cs.AddVote(val0.Address, vote) - } - - // Ensure that vote appears in RoundState. - rs0 := cs.GetRoundState() - if vote := rs0.Prevotes.GetByAddress(val0.Address); vote == nil || vote.Type != types.VoteTypePrevote { - t.Errorf("Expected to find prevote but got %v", vote) - } - if vote := rs0.Precommits.GetByAddress(val0.Address); vote == nil || vote.Type != types.VoteTypePrecommit { - t.Errorf("Expected to find precommit but got %v", vote) - } - if vote := rs0.Commits.GetByAddress(val0.Address); vote == nil || vote.Type != types.VoteTypeCommit { - t.Errorf("Expected to find commit but got %v", vote) - } - - // Setup round 1 (next round) - cs.SetupNewRound(1, 1) - <-cs.NewStepCh() - - // Now the commit should be copied over to prevotes and precommits. - rs1 := cs.GetRoundState() - if vote := rs1.Prevotes.GetByAddress(val0.Address); vote == nil || vote.Type != types.VoteTypeCommit { - t.Errorf("Expected to find commit but got %v", vote) - } - if vote := rs1.Precommits.GetByAddress(val0.Address); vote == nil || vote.Type != types.VoteTypeCommit { - t.Errorf("Expected to find commit but got %v", vote) - } - if vote := rs1.Commits.GetByAddress(val0.Address); vote == nil || vote.Type != types.VoteTypeCommit { - t.Errorf("Expected to find commit but got %v", vote) - } - -} - func TestRunActionProposeNoPrivValidator(t *testing.T) { cs, _ := randConsensusState() cs.RunActionPropose(1, 0) @@ -82,128 +37,4 @@ func TestRunActionPropose(t *testing.T) { } } -func checkRoundState(t *testing.T, rs *RoundState, - height uint, round uint, step RoundStepType) { - if rs.Height != height { - t.Errorf("rs.Height should be %v, got %v", height, rs.Height) - } - if rs.Round != round { - t.Errorf("rs.Round should be %v, got %v", round, rs.Round) - } - if rs.Step != step { - t.Errorf("rs.Step should be %v, got %v", step, rs.Step) - } -} - -func TestRunActionPrecommitCommitFinalize(t *testing.T) { - cs, privValidators := randConsensusState() - val0 := privValidators[0] - cs.SetPrivValidator(val0) - - cs.RunActionPrecommit(1, 0) - <-cs.NewStepCh() // TODO: test this value too. - if cs.Precommits.GetByAddress(val0.Address) != nil { - t.Errorf("RunActionPrecommit should return nil without a proposal") - } - - cs.RunActionPropose(1, 0) - <-cs.NewStepCh() // TODO: test this value too. - - cs.RunActionPrecommit(1, 0) - <-cs.NewStepCh() // TODO: test this value too. - if cs.Precommits.GetByAddress(val0.Address) != nil { - t.Errorf("RunActionPrecommit should return nil, not enough prevotes") - } - - // Add at least +2/3 prevotes. - for i := 0; i < 7; i++ { - vote := &types.Vote{ - Height: 1, - Round: 0, - Type: types.VoteTypePrevote, - BlockHash: cs.ProposalBlock.Hash(), - BlockParts: cs.ProposalBlockParts.Header(), - } - err := privValidators[i].SignVote(cs.state.ChainID, vote) - if err != nil { - t.Error("Error signing vote: %v", err) - } - cs.AddVote(privValidators[i].Address, vote) - } - - // Test RunActionPrecommit success: - cs.RunActionPrecommit(1, 0) - <-cs.NewStepCh() // TODO: test this value too. - if cs.Precommits.GetByAddress(val0.Address) == nil { - t.Errorf("RunActionPrecommit should have succeeded") - } - checkRoundState(t, cs.GetRoundState(), 1, 0, RoundStepPrecommit) - - // Add at least +2/3 precommits. - for i := 0; i < 7; i++ { - if bytes.Equal(privValidators[i].Address, val0.Address) { - if cs.Precommits.GetByAddress(val0.Address) == nil { - t.Errorf("Proposer should already have signed a precommit vote") - } - continue - } - vote := &types.Vote{ - Height: 1, - Round: 0, - Type: types.VoteTypePrecommit, - BlockHash: cs.ProposalBlock.Hash(), - BlockParts: cs.ProposalBlockParts.Header(), - } - err := privValidators[i].SignVote(cs.state.ChainID, vote) - if err != nil { - t.Error("Error signing vote: %v", err) - } - added, _, err := cs.AddVote(privValidators[i].Address, vote) - if !added || err != nil { - t.Errorf("Error adding precommit: %v", err) - } - } - - // Test RunActionCommit success: - cs.RunActionCommit(1) - <-cs.NewStepCh() // TODO: test this value too. - if cs.Commits.GetByAddress(val0.Address) == nil { - t.Errorf("RunActionCommit should have succeeded") - } - checkRoundState(t, cs.GetRoundState(), 1, 0, RoundStepCommit) - - // cs.CommitTime should still be zero - if !cs.CommitTime.IsZero() { - t.Errorf("Expected CommitTime to yet be zero") - } - - // Add at least +2/3 commits. - for i := 0; i < 7; i++ { - if bytes.Equal(privValidators[i].Address, val0.Address) { - if cs.Commits.GetByAddress(val0.Address) == nil { - t.Errorf("Proposer should already have signed a commit vote") - } - continue - } - vote := &types.Vote{ - Height: 1, - Round: uint(i), // Doesn't matter what round - Type: types.VoteTypeCommit, - BlockHash: cs.ProposalBlock.Hash(), - BlockParts: cs.ProposalBlockParts.Header(), - } - err := privValidators[i].SignVote(cs.state.ChainID, vote) - if err != nil { - t.Error("Error signing vote: %v", err) - } - added, _, err := cs.AddVote(privValidators[i].Address, vote) - if !added || err != nil { - t.Errorf("Error adding commit: %v", err) - } - } - - // Test TryFinalizeCommit: - cs.TryFinalizeCommit(1) - <-cs.NewStepCh() // TODO: test this value too. - checkRoundState(t, cs.GetRoundState(), 2, 0, RoundStepNewHeight) -} +// TODO write better consensus state tests diff --git a/consensus/types/proposal.go b/consensus/types/proposal.go index d2fb4b15f..5009e6819 100644 --- a/consensus/types/proposal.go +++ b/consensus/types/proposal.go @@ -20,29 +20,28 @@ type Proposal struct { Height uint `json:"height"` Round uint `json:"round"` BlockParts types.PartSetHeader `json:"block_parts"` - POLParts types.PartSetHeader `json:"pol_parts"` + POLRound int `json:"pol_round"` // -1 if null. Signature account.SignatureEd25519 `json:"signature"` } -func NewProposal(height uint, round uint, blockParts, polParts types.PartSetHeader) *Proposal { +func NewProposal(height uint, round uint, blockParts types.PartSetHeader, polRound int) *Proposal { return &Proposal{ Height: height, Round: round, BlockParts: blockParts, - POLParts: polParts, + POLRound: polRound, } } func (p *Proposal) String() string { return fmt.Sprintf("Proposal{%v/%v %v %v %v}", p.Height, p.Round, - p.BlockParts, p.POLParts, p.Signature) + p.BlockParts, p.POLRound, p.Signature) } func (p *Proposal) WriteSignBytes(chainID string, w io.Writer, n *int64, err *error) { binary.WriteTo([]byte(Fmt(`{"chain_id":"%s"`, chainID)), w, n, err) binary.WriteTo([]byte(`,"proposal":{"block_parts":`), w, n, err) p.BlockParts.WriteSignBytes(w, n, err) - binary.WriteTo([]byte(Fmt(`,"height":%v,"pol_parts":`, p.Height)), w, n, err) - p.POLParts.WriteSignBytes(w, n, err) + binary.WriteTo([]byte(Fmt(`,"height":%v,"pol_round":%v`, p.Height, p.POLRound)), w, n, err) binary.WriteTo([]byte(Fmt(`,"round":%v}}`, p.Round)), w, n, err) } diff --git a/consensus/vote_set.go b/consensus/vote_set.go index 39f7747b5..136d1463e 100644 --- a/consensus/vote_set.go +++ b/consensus/vote_set.go @@ -39,9 +39,6 @@ func NewVoteSet(height uint, round uint, type_ byte, valSet *sm.ValidatorSet) *V if height == 0 { panic("Cannot make VoteSet for height == 0, doesn't make sense.") } - if type_ == types.VoteTypeCommit && round != 0 { - panic("Expected round 0 for commit vote set") - } return &VoteSet{ height: height, round: round, @@ -111,10 +108,9 @@ func (voteSet *VoteSet) addByIndex(valIndex uint, vote *types.Vote) (bool, uint, func (voteSet *VoteSet) addVote(val *sm.Validator, valIndex uint, vote *types.Vote) (bool, uint, error) { // Make sure the step matches. (or that vote is commit && round < voteSet.round) - if vote.Height != voteSet.height || - (vote.Type != types.VoteTypeCommit && vote.Round != voteSet.round) || - (vote.Type != types.VoteTypeCommit && vote.Type != voteSet.type_) || - (vote.Type == types.VoteTypeCommit && voteSet.type_ != types.VoteTypeCommit && vote.Round >= voteSet.round) { + if (vote.Height != voteSet.height) || + (vote.Round != voteSet.round) || + (vote.Type != voteSet.type_) { return false, 0, types.ErrVoteUnexpectedStep } @@ -155,18 +151,6 @@ func (voteSet *VoteSet) addVote(val *sm.Validator, valIndex uint, vote *types.Vo return true, valIndex, nil } -// Assumes that commits VoteSet is valid. -func (voteSet *VoteSet) AddFromCommits(commits *VoteSet) { - for valIndex, commit := range commits.votes { - if commit == nil { - continue - } - if commit.Round < voteSet.round { - voteSet.addByIndex(uint(valIndex), commit) - } - } -} - func (voteSet *VoteSet) BitArray() *BitArray { if voteSet == nil { return nil @@ -201,6 +185,15 @@ func (voteSet *VoteSet) HasTwoThirdsMajority() bool { return voteSet.maj23Exists } +func (voteSet *VoteSet) HasTwoThirdsAny() bool { + if voteSet == nil { + return false + } + voteSet.mtx.Lock() + defer voteSet.mtx.Unlock() + return voteSet.totalBlockHashVotes > voteSet.valSet.TotalVotingPower()*2/3 +} + // Returns either a blockhash (or nil) that received +2/3 majority. // If there exists no such majority, returns (nil, false). func (voteSet *VoteSet) TwoThirdsMajority() (hash []byte, parts types.PartSetHeader, ok bool) { @@ -213,50 +206,16 @@ func (voteSet *VoteSet) TwoThirdsMajority() (hash []byte, parts types.PartSetHea } } -func (voteSet *VoteSet) MakePOL() *POL { - if voteSet.type_ != types.VoteTypePrevote { - panic("Cannot MakePOL() unless VoteSet.Type is types.VoteTypePrevote") - } - voteSet.mtx.Lock() - defer voteSet.mtx.Unlock() - if !voteSet.maj23Exists { - return nil - } - pol := &POL{ - Height: voteSet.height, - Round: voteSet.round, - BlockHash: voteSet.maj23Hash, - BlockParts: voteSet.maj23Parts, - Votes: make([]POLVoteSignature, voteSet.valSet.Size()), - } - for valIndex, vote := range voteSet.votes { - if vote == nil { - continue - } - if !bytes.Equal(vote.BlockHash, voteSet.maj23Hash) { - continue - } - if !vote.BlockParts.Equals(voteSet.maj23Parts) { - continue - } - pol.Votes[valIndex] = POLVoteSignature{ - Round: vote.Round, - Signature: vote.Signature, - } - } - return pol -} - func (voteSet *VoteSet) MakeValidation() *types.Validation { - if voteSet.type_ != types.VoteTypeCommit { - panic("Cannot MakeValidation() unless VoteSet.Type is types.VoteTypeCommit") + if voteSet.type_ != types.VoteTypePrecommit { + panic("Cannot MakeValidation() unless VoteSet.Type is types.VoteTypePrecommit") } voteSet.mtx.Lock() defer voteSet.mtx.Unlock() if len(voteSet.maj23Hash) == 0 { panic("Cannot MakeValidation() unless a blockhash has +2/3") } - commits := make([]types.Commit, voteSet.valSet.Size()) + precommits := make([]types.Precommit, voteSet.valSet.Size()) voteSet.valSet.Iterate(func(valIndex uint, val *sm.Validator) bool { vote := voteSet.votes[valIndex] if vote == nil { @@ -268,11 +227,12 @@ func (voteSet *VoteSet) MakeValidation() *types.Validation { if !vote.BlockParts.Equals(voteSet.maj23Parts) { return false } - commits[valIndex] = types.Commit{val.Address, vote.Round, vote.Signature} + precommits[valIndex] = types.Precommit{val.Address, vote.Signature} return false }) return &types.Validation{ - Commits: commits, + Round: voteSet.round, + Precommits: precommits, } } diff --git a/consensus/vote_set_test.go b/consensus/vote_set_test.go index 8cc919f31..46bafed61 100644 --- a/consensus/vote_set_test.go +++ b/consensus/vote_set_test.go @@ -225,69 +225,12 @@ func TestBadVotes(t *testing.T) { } } -func TestAddCommitsToPrevoteVotes(t *testing.T) { - height, round := uint(2), uint(5) - voteSet, _, privValidators := randVoteSet(height, round, types.VoteTypePrevote, 10, 1) - - // val0, val1, val2, val3, val4, val5 vote for nil. - vote := &types.Vote{Height: height, Round: round, Type: types.VoteTypePrevote, BlockHash: nil} - for i := 0; i < 6; i++ { - signAddVote(privValidators[i], vote, voteSet) - } - hash, header, ok := voteSet.TwoThirdsMajority() - if hash != nil || !header.IsZero() || ok { - t.Errorf("There should be no 2/3 majority") - } - - // Attempt to add a commit from val6 at a previous height - vote = &types.Vote{Height: height - 1, Round: round, Type: types.VoteTypeCommit, BlockHash: nil} - added, _ := signAddVote(privValidators[6], vote, voteSet) - if added { - t.Errorf("Expected VoteSet.Add to fail, wrong height.") - } - - // Attempt to add a commit from val6 at a later round - vote = &types.Vote{Height: height, Round: round + 1, Type: types.VoteTypeCommit, BlockHash: nil} - added, _ = signAddVote(privValidators[6], vote, voteSet) - if added { - t.Errorf("Expected VoteSet.Add to fail, cannot add future round vote.") - } - - // Attempt to add a commit from val6 for currrent height/round. - vote = &types.Vote{Height: height, Round: round, Type: types.VoteTypeCommit, BlockHash: nil} - added, err := signAddVote(privValidators[6], vote, voteSet) - if added || err == nil { - t.Errorf("Expected VoteSet.Add to fail, only prior round commits can be added.") - } - - // Add commit from val6 at a previous round - vote = &types.Vote{Height: height, Round: round - 1, Type: types.VoteTypeCommit, BlockHash: nil} - added, err = signAddVote(privValidators[6], vote, voteSet) - if !added || err != nil { - t.Errorf("Expected VoteSet.Add to succeed, commit for prior rounds are relevant.") - } - - // Also add commit from val7 for previous round. - vote = &types.Vote{Height: height, Round: round - 2, Type: types.VoteTypeCommit, BlockHash: nil} - added, err = signAddVote(privValidators[7], vote, voteSet) - if !added || err != nil { - t.Errorf("Expected VoteSet.Add to succeed. err: %v", err) - } - - // We should have 2/3 majority - hash, header, ok = voteSet.TwoThirdsMajority() - if hash != nil || !header.IsZero() || !ok { - t.Errorf("There should be 2/3 majority for nil") - } - -} - func TestMakeValidation(t *testing.T) { height, round := uint(1), uint(0) - voteSet, _, privValidators := randVoteSet(height, round, types.VoteTypeCommit, 10, 1) + voteSet, _, privValidators := randVoteSet(height, round, types.VoteTypePrecommit, 10, 1) blockHash, blockParts := CRandBytes(32), types.PartSetHeader{123, CRandBytes(32)} - vote := &types.Vote{Height: height, Round: round, Type: types.VoteTypeCommit, + vote := &types.Vote{Height: height, Round: round, Type: types.VoteTypePrecommit, BlockHash: blockHash, BlockParts: blockParts} // 6 out of 10 voted for some block. @@ -313,11 +256,11 @@ func TestMakeValidation(t *testing.T) { validation := voteSet.MakeValidation() // Validation should have 10 elements - if len(validation.Commits) != 10 { - t.Errorf("Validation Commits should have the same number of commits as validators") + if len(validation.Precommits) != 10 { + t.Errorf("Validation Precommits should have the same number of precommits as validators") } - // Ensure that Validation commits are ordered. + // Ensure that Validation precommits are ordered. if err := validation.ValidateBasic(); err != nil { t.Errorf("Error in Validation.ValidateBasic(): %v", err) } diff --git a/state/execution.go b/state/execution.go index c2edef8b3..1d85c9498 100644 --- a/state/execution.go +++ b/state/execution.go @@ -40,28 +40,28 @@ func execBlock(s *State, block *types.Block, blockPartsHeader types.PartSetHeade // Validate block Validation. if block.Height == 1 { - if len(block.Validation.Commits) != 0 { - return errors.New("Block at height 1 (first block) should have no Validation commits") + if len(block.Validation.Precommits) != 0 { + return errors.New("Block at height 1 (first block) should have no Validation precommits") } } else { - if uint(len(block.Validation.Commits)) != s.LastBondedValidators.Size() { + if uint(len(block.Validation.Precommits)) != s.LastBondedValidators.Size() { return errors.New(Fmt("Invalid block validation size. Expected %v, got %v", - s.LastBondedValidators.Size(), len(block.Validation.Commits))) + s.LastBondedValidators.Size(), len(block.Validation.Precommits))) } var sumVotingPower uint64 s.LastBondedValidators.Iterate(func(index uint, val *Validator) bool { - commit := block.Validation.Commits[index] - if commit.IsZero() { + precommit := block.Validation.Precommits[index] + if precommit.IsZero() { return false } else { vote := &types.Vote{ Height: block.Height - 1, - Round: commit.Round, - Type: types.VoteTypeCommit, + Round: block.Validation.Round, + Type: types.VoteTypePrecommit, BlockHash: block.LastBlockHash, BlockParts: block.LastBlockParts, } - if val.PubKey.VerifyBytes(account.SignBytes(s.ChainID, vote), commit.Signature) { + if val.PubKey.VerifyBytes(account.SignBytes(s.ChainID, vote), precommit.Signature) { sumVotingPower += val.VotingPower return false } else { @@ -80,8 +80,8 @@ func execBlock(s *State, block *types.Block, blockPartsHeader types.PartSetHeade } // Update Validator.LastCommitHeight as necessary. - for i, commit := range block.Validation.Commits { - if commit.IsZero() { + for i, precommit := range block.Validation.Precommits { + if precommit.IsZero() { continue } _, val := s.LastBondedValidators.GetByIndex(uint(i)) @@ -111,7 +111,7 @@ func execBlock(s *State, block *types.Block, blockPartsHeader types.PartSetHeade // Create BlockCache to cache changes to state. blockCache := NewBlockCache(s) - // Commit each tx + // Execute each tx for _, tx := range block.Data.Txs { err := ExecTx(blockCache, tx, true, s.evc) if err != nil { @@ -726,21 +726,14 @@ func ExecTx(blockCache *BlockCache, tx_ types.Tx, runCall bool, evc events.Firea if tx.VoteA.Height != tx.VoteB.Height { return errors.New("DupeoutTx heights don't match") } - if tx.VoteA.Type == types.VoteTypeCommit && tx.VoteA.Round < tx.VoteB.Round { - // Check special case (not an error, validator must be slashed!) - // Validators should not sign another vote after committing. - } else if tx.VoteB.Type == types.VoteTypeCommit && tx.VoteB.Round < tx.VoteA.Round { - // We need to check both orderings of the votes - } else { - if tx.VoteA.Round != tx.VoteB.Round { - return errors.New("DupeoutTx rounds don't match") - } - if tx.VoteA.Type != tx.VoteB.Type { - return errors.New("DupeoutTx types don't match") - } - if bytes.Equal(tx.VoteA.BlockHash, tx.VoteB.BlockHash) { - return errors.New("DupeoutTx blockhashes shouldn't match") - } + if tx.VoteA.Round != tx.VoteB.Round { + return errors.New("DupeoutTx rounds don't match") + } + if tx.VoteA.Type != tx.VoteB.Type { + return errors.New("DupeoutTx types don't match") + } + if bytes.Equal(tx.VoteA.BlockHash, tx.VoteB.BlockHash) { + return errors.New("DupeoutTx blockhashes shouldn't match") } // Good! (Bad validator!) diff --git a/state/priv_validator.go b/state/priv_validator.go index e9ce7be5c..41c8b913a 100644 --- a/state/priv_validator.go +++ b/state/priv_validator.go @@ -1,7 +1,5 @@ package state -// TODO: This logic is crude. Should be more transactional. - import ( "errors" "fmt" @@ -23,7 +21,6 @@ const ( stepPropose = 1 stepPrevote = 2 stepPrecommit = 3 - stepCommit = 4 ) func voteToStep(vote *types.Vote) uint8 { @@ -32,8 +29,6 @@ func voteToStep(vote *types.Vote) uint8 { return stepPrevote case types.VoteTypePrecommit: return stepPrecommit - case types.VoteTypeCommit: - return stepCommit default: panic("Unknown vote type") } @@ -108,7 +103,6 @@ func (privVal *PrivValidator) save() { } } -// TODO: test func (privVal *PrivValidator) SignVote(chainID string, vote *types.Vote) error { privVal.mtx.Lock() defer privVal.mtx.Unlock() @@ -119,10 +113,6 @@ func (privVal *PrivValidator) SignVote(chainID string, vote *types.Vote) error { } // More cases for when the height matches if privVal.LastHeight == vote.Height { - // If attempting any sign after commit, panic - if privVal.LastStep == stepCommit { - return errors.New("SignVote on matching height after a commit") - } // If round regression, panic if privVal.LastRound > vote.Round { return errors.New("Round regression in SignVote") diff --git a/state/state.go b/state/state.go index fcb408502..e7486b277 100644 --- a/state/state.go +++ b/state/state.go @@ -130,7 +130,7 @@ func (s *State) Hash() []byte { } // Mutates the block in place and updates it with new state hash. -func (s *State) SetBlockStateHash(block *types.Block) error { +func (s *State) ComputeBlockStateHash(block *types.Block) error { sCopy := s.Copy() // sCopy has no event cache in it, so this won't fire events err := execBlock(sCopy, block, types.PartSetHeader{}) diff --git a/state/state_test.go b/state/state_test.go index acb809fbb..9e5219ddc 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -71,7 +71,7 @@ func TestCopyState(t *testing.T) { } } -func makeBlock(t *testing.T, state *State, commits []types.Commit, txs []types.Tx) *types.Block { +func makeBlock(t *testing.T, state *State, validation *types.Validation, txs []types.Tx) *types.Block { block := &types.Block{ Header: &types.Header{ ChainID: state.ChainID, @@ -83,16 +83,14 @@ func makeBlock(t *testing.T, state *State, commits []types.Commit, txs []types.T LastBlockParts: state.LastBlockParts, StateHash: nil, }, - Validation: &types.Validation{ - Commits: commits, - }, + Validation: validation, Data: &types.Data{ Txs: txs, }, } // Fill in block StateHash - err := state.SetBlockStateHash(block) + err := state.ComputeBlockStateHash(block) if err != nil { t.Error("Error appending initial block:", err) } @@ -620,21 +618,23 @@ func TestAddValidator(t *testing.T) { // The validation for the next block should only require 1 signature // (the new validator wasn't active for block0) - commit0 := &types.Vote{ + precommit0 := &types.Vote{ Height: 1, Round: 0, - Type: types.VoteTypeCommit, + Type: types.VoteTypePrecommit, BlockHash: block0.Hash(), BlockParts: block0Parts.Header(), } - privValidators[0].SignVote(s0.ChainID, commit0) + privValidators[0].SignVote(s0.ChainID, precommit0) block1 := makeBlock(t, s0, - []types.Commit{ - types.Commit{ - Address: privValidators[0].Address, - Round: 0, - Signature: commit0.Signature, + types.Validation{ + Round: 0, + Precommits: []types.Precommit{ + types.Precommit{ + Address: privValidators[0].Address, + Signature: precommit0.Signature, + }, }, }, nil, ) diff --git a/state/validator_set.go b/state/validator_set.go index 5bfc76f96..da5585c06 100644 --- a/state/validator_set.go +++ b/state/validator_set.go @@ -202,33 +202,33 @@ func (valSet *ValidatorSet) Iterate(fn func(index uint, val *Validator) bool) { // Verify that +2/3 of the set had signed the given signBytes func (valSet *ValidatorSet) VerifyValidation(chainID string, hash []byte, parts types.PartSetHeader, height uint, v *types.Validation) error { - if valSet.Size() != uint(len(v.Commits)) { + if valSet.Size() != uint(len(v.Precommits)) { return errors.New(Fmt("Invalid validation -- wrong set size: %v vs %v", - valSet.Size(), len(v.Commits))) + valSet.Size(), len(v.Precommits))) } talliedVotingPower := uint64(0) seenValidators := map[string]struct{}{} - for idx, commit := range v.Commits { + for idx, precommit := range v.Precommits { // may be zero, in which case skip. - if commit.Signature.IsZero() { + if precommit.Signature.IsZero() { continue } _, val := valSet.GetByIndex(uint(idx)) - commitSignBytes := account.SignBytes(chainID, &types.Vote{ - Height: height, Round: commit.Round, Type: types.VoteTypeCommit, + precommitSignBytes := account.SignBytes(chainID, &types.Vote{ + Height: height, Round: v.Round, Type: types.VoteTypePrecommit, BlockHash: hash, BlockParts: parts, }) // Validate if _, seen := seenValidators[string(val.Address)]; seen { - return fmt.Errorf("Duplicate validator for commit %v for Validation %v", commit, v) + return fmt.Errorf("Duplicate validator for precommit %v for Validation %v", precommit, v) } - if !val.PubKey.VerifyBytes(commitSignBytes, commit.Signature) { - return fmt.Errorf("Invalid signature for commit %v for Validation %v", commit, v) + if !val.PubKey.VerifyBytes(precommitSignBytes, precommit.Signature) { + return fmt.Errorf("Invalid signature for precommit %v for Validation %v", precommit, v) } // Tally diff --git a/types/block.go b/types/block.go index 8411edad1..c42dbeb9e 100644 --- a/types/block.go +++ b/types/block.go @@ -176,27 +176,27 @@ func (h *Header) StringIndented(indent string) string { //----------------------------------------------------------------------------- -type Commit struct { +type Precommit struct { Address []byte `json:"address"` - Round uint `json:"round"` Signature account.SignatureEd25519 `json:"signature"` } -func (commit Commit) IsZero() bool { - return commit.Round == 0 && commit.Signature.IsZero() +func (pc Precommit) IsZero() bool { + return pc.Signature.IsZero() } -func (commit Commit) String() string { - return fmt.Sprintf("Commit{A:%X R:%v %X}", commit.Address, commit.Round, Fingerprint(commit.Signature)) +func (pc Precommit) String() string { + return fmt.Sprintf("Precommit{A:%X %X}", pc.Address, Fingerprint(pc.Signature)) } //------------------------------------- -// NOTE: The Commits are in order of address to preserve the bonded ValidatorSet order. -// Any peer with a block can gossip commits by index with a peer without recalculating the +// NOTE: The Precommits are in order of address to preserve the bonded ValidatorSet order. +// Any peer with a block can gossip precommits by index with a peer without recalculating the // active ValidatorSet. type Validation struct { - Commits []Commit `json:"commits"` // Commits (or nil) of all active validators in address order. + Round uint `json:"round"` // Round for all precommits + Precommits []Precommit `json:"precommits"` // Precommits (or nil) of all active validators in address order. // Volatile hash []byte @@ -204,24 +204,24 @@ type Validation struct { } func (v *Validation) ValidateBasic() error { - if len(v.Commits) == 0 { - return errors.New("No commits in validation") + if len(v.Precommits) == 0 { + return errors.New("No precommits in validation") } lastAddress := []byte{} - for i := 0; i < len(v.Commits); i++ { - commit := v.Commits[i] - if commit.IsZero() { - if len(commit.Address) > 0 { - return errors.New("Zero commits should not have an address") + for i := 0; i < len(v.Precommits); i++ { + precommit := v.Precommits[i] + if precommit.IsZero() { + if len(precommit.Address) > 0 { + return errors.New("Zero precommits should not have an address") } } else { - if len(commit.Address) == 0 { - return errors.New("Nonzero commits should have an address") + if len(precommit.Address) == 0 { + return errors.New("Nonzero precommits should have an address") } - if len(lastAddress) > 0 && bytes.Compare(lastAddress, commit.Address) != -1 { - return errors.New("Invalid commit order") + if len(lastAddress) > 0 && bytes.Compare(lastAddress, precommit.Address) != -1 { + return errors.New("Invalid precommit order") } - lastAddress = commit.Address + lastAddress = precommit.Address } } return nil @@ -229,9 +229,10 @@ func (v *Validation) ValidateBasic() error { func (v *Validation) Hash() []byte { if v.hash == nil { - bs := make([]interface{}, len(v.Commits)) - for i, commit := range v.Commits { - bs[i] = commit + bs := make([]interface{}, 1+len(v.Precommits)) + bs[0] = v.Round + for i, precommit := range v.Precommits { + bs[1+i] = precommit } v.hash = merkle.SimpleHashFromBinaries(bs) } @@ -242,22 +243,24 @@ func (v *Validation) StringIndented(indent string) string { if v == nil { return "nil-Validation" } - commitStrings := make([]string, len(v.Commits)) - for i, commit := range v.Commits { - commitStrings[i] = commit.String() + precommitStrings := make([]string, len(v.Precommits)) + for i, precommit := range v.Precommits { + precommitStrings[i] = precommit.String() } return fmt.Sprintf(`Validation{ -%s %v +%s Round: %v +%s Precommits: %v %s}#%X`, - indent, strings.Join(commitStrings, "\n"+indent+" "), + indent, v.Round, + indent, strings.Join(precommitStrings, "\n"+indent+" "), indent, v.hash) } func (v *Validation) BitArray() *BitArray { if v.bitArray == nil { - v.bitArray = NewBitArray(uint(len(v.Commits))) - for i, commit := range v.Commits { - v.bitArray.SetIndex(uint(i), !commit.IsZero()) + v.bitArray = NewBitArray(uint(len(v.Precommits))) + for i, precommit := range v.Precommits { + v.bitArray.SetIndex(uint(i), !precommit.IsZero()) } } return v.bitArray diff --git a/types/vote.go b/types/vote.go index 365b5a9af..2e931c232 100644 --- a/types/vote.go +++ b/types/vote.go @@ -27,8 +27,6 @@ func (err *ErrVoteConflictingSignature) Error() string { } // Represents a prevote, precommit, or commit vote from validators for consensus. -// Commit votes get aggregated into the next block's Validaiton. -// See the whitepaper for details. type Vote struct { Height uint `json:"height"` Round uint `json:"round"` @@ -42,7 +40,6 @@ type Vote struct { const ( VoteTypePrevote = byte(0x01) VoteTypePrecommit = byte(0x02) - VoteTypeCommit = byte(0x03) ) func (vote *Vote) WriteSignBytes(chainID string, w io.Writer, n *int64, err *error) { @@ -63,8 +60,6 @@ func (vote *Vote) String() string { typeString = "Prevote" case VoteTypePrecommit: typeString = "Precommit" - case VoteTypeCommit: - typeString = "Commit" default: panic("Unknown vote type") }