You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1544 lines
51 KiB

10 years ago
10 years ago
10 years ago
10 years ago
9 years ago
10 years ago
10 years ago
10 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
9 years ago
9 years ago
10 years ago
10 years ago
10 years ago
10 years ago
9 years ago
9 years ago
9 years ago
8 years ago
9 years ago
10 years ago
10 years ago
9 years ago
9 years ago
9 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
9 years ago
9 years ago
9 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
9 years ago
9 years ago
10 years ago
9 years ago
9 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
10 years ago
10 years ago
9 years ago
9 years ago
10 years ago
10 years ago
10 years ago
9 years ago
10 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
10 years ago
10 years ago
  1. package consensus
  2. import (
  3. "bytes"
  4. "errors"
  5. "fmt"
  6. "reflect"
  7. "sync"
  8. "time"
  9. "github.com/ebuchman/fail-test"
  10. . "github.com/tendermint/go-common"
  11. cfg "github.com/tendermint/go-config"
  12. "github.com/tendermint/go-wire"
  13. bc "github.com/tendermint/tendermint/blockchain"
  14. mempl "github.com/tendermint/tendermint/mempool"
  15. "github.com/tendermint/tendermint/proxy"
  16. sm "github.com/tendermint/tendermint/state"
  17. "github.com/tendermint/tendermint/types"
  18. )
  19. //-----------------------------------------------------------------------------
  20. // Timeout Parameters
  21. // All in milliseconds
  22. type TimeoutParams struct {
  23. Propose0 int
  24. ProposeDelta int
  25. Prevote0 int
  26. PrevoteDelta int
  27. Precommit0 int
  28. PrecommitDelta int
  29. Commit0 int
  30. }
  31. // Wait this long for a proposal
  32. func (tp *TimeoutParams) Propose(round int) time.Duration {
  33. return time.Duration(tp.Propose0+tp.ProposeDelta*round) * time.Millisecond
  34. }
  35. // After receiving any +2/3 prevote, wait this long for stragglers
  36. func (tp *TimeoutParams) Prevote(round int) time.Duration {
  37. return time.Duration(tp.Prevote0+tp.PrevoteDelta*round) * time.Millisecond
  38. }
  39. // After receiving any +2/3 precommits, wait this long for stragglers
  40. func (tp *TimeoutParams) Precommit(round int) time.Duration {
  41. return time.Duration(tp.Precommit0+tp.PrecommitDelta*round) * time.Millisecond
  42. }
  43. // After receiving +2/3 precommits for a single block (a commit), wait this long for stragglers in the next height's RoundStepNewHeight
  44. func (tp *TimeoutParams) Commit(t time.Time) time.Time {
  45. return t.Add(time.Duration(tp.Commit0) * time.Millisecond)
  46. }
  47. // Initialize parameters from config
  48. func InitTimeoutParamsFromConfig(config cfg.Config) *TimeoutParams {
  49. return &TimeoutParams{
  50. Propose0: config.GetInt("timeout_propose"),
  51. ProposeDelta: config.GetInt("timeout_propose_delta"),
  52. Prevote0: config.GetInt("timeout_prevote"),
  53. PrevoteDelta: config.GetInt("timeout_prevote_delta"),
  54. Precommit0: config.GetInt("timeout_precommit"),
  55. PrecommitDelta: config.GetInt("timeout_precommit_delta"),
  56. Commit0: config.GetInt("timeout_commit"),
  57. }
  58. }
  59. //-----------------------------------------------------------------------------
  60. // Errors
  61. var (
  62. ErrInvalidProposalSignature = errors.New("Error invalid proposal signature")
  63. ErrInvalidProposalPOLRound = errors.New("Error invalid proposal POL round")
  64. ErrAddingVote = errors.New("Error adding vote")
  65. ErrVoteHeightMismatch = errors.New("Error vote height mismatch")
  66. )
  67. //-----------------------------------------------------------------------------
  68. // RoundStepType enum type
  69. type RoundStepType uint8 // These must be numeric, ordered.
  70. const (
  71. RoundStepNewHeight = RoundStepType(0x01) // Wait til CommitTime + timeoutCommit
  72. RoundStepNewRound = RoundStepType(0x02) // Setup new round and go to RoundStepPropose
  73. RoundStepPropose = RoundStepType(0x03) // Did propose, gossip proposal
  74. RoundStepPrevote = RoundStepType(0x04) // Did prevote, gossip prevotes
  75. RoundStepPrevoteWait = RoundStepType(0x05) // Did receive any +2/3 prevotes, start timeout
  76. RoundStepPrecommit = RoundStepType(0x06) // Did precommit, gossip precommits
  77. RoundStepPrecommitWait = RoundStepType(0x07) // Did receive any +2/3 precommits, start timeout
  78. RoundStepCommit = RoundStepType(0x08) // Entered commit state machine
  79. // NOTE: RoundStepNewHeight acts as RoundStepCommitWait.
  80. )
  81. func (rs RoundStepType) String() string {
  82. switch rs {
  83. case RoundStepNewHeight:
  84. return "RoundStepNewHeight"
  85. case RoundStepNewRound:
  86. return "RoundStepNewRound"
  87. case RoundStepPropose:
  88. return "RoundStepPropose"
  89. case RoundStepPrevote:
  90. return "RoundStepPrevote"
  91. case RoundStepPrevoteWait:
  92. return "RoundStepPrevoteWait"
  93. case RoundStepPrecommit:
  94. return "RoundStepPrecommit"
  95. case RoundStepPrecommitWait:
  96. return "RoundStepPrecommitWait"
  97. case RoundStepCommit:
  98. return "RoundStepCommit"
  99. default:
  100. return "RoundStepUnknown" // Cannot panic.
  101. }
  102. }
  103. //-----------------------------------------------------------------------------
  104. // Immutable when returned from ConsensusState.GetRoundState()
  105. type RoundState struct {
  106. Height int // Height we are working on
  107. Round int
  108. Step RoundStepType
  109. StartTime time.Time
  110. CommitTime time.Time // Subjective time when +2/3 precommits for Block at Round were found
  111. Validators *types.ValidatorSet
  112. Proposal *types.Proposal
  113. ProposalBlock *types.Block
  114. ProposalBlockParts *types.PartSet
  115. LockedRound int
  116. LockedBlock *types.Block
  117. LockedBlockParts *types.PartSet
  118. Votes *HeightVoteSet
  119. CommitRound int //
  120. LastCommit *types.VoteSet // Last precommits at Height-1
  121. LastValidators *types.ValidatorSet
  122. }
  123. func (rs *RoundState) RoundStateEvent() types.EventDataRoundState {
  124. edrs := types.EventDataRoundState{
  125. Height: rs.Height,
  126. Round: rs.Round,
  127. Step: rs.Step.String(),
  128. RoundState: rs,
  129. }
  130. return edrs
  131. }
  132. func (rs *RoundState) String() string {
  133. return rs.StringIndented("")
  134. }
  135. func (rs *RoundState) StringIndented(indent string) string {
  136. return fmt.Sprintf(`RoundState{
  137. %s H:%v R:%v S:%v
  138. %s StartTime: %v
  139. %s CommitTime: %v
  140. %s Validators: %v
  141. %s Proposal: %v
  142. %s ProposalBlock: %v %v
  143. %s LockedRound: %v
  144. %s LockedBlock: %v %v
  145. %s Votes: %v
  146. %s LastCommit: %v
  147. %s LastValidators: %v
  148. %s}`,
  149. indent, rs.Height, rs.Round, rs.Step,
  150. indent, rs.StartTime,
  151. indent, rs.CommitTime,
  152. indent, rs.Validators.StringIndented(indent+" "),
  153. indent, rs.Proposal,
  154. indent, rs.ProposalBlockParts.StringShort(), rs.ProposalBlock.StringShort(),
  155. indent, rs.LockedRound,
  156. indent, rs.LockedBlockParts.StringShort(), rs.LockedBlock.StringShort(),
  157. indent, rs.Votes.StringIndented(indent+" "),
  158. indent, rs.LastCommit.StringShort(),
  159. indent, rs.LastValidators.StringIndented(indent+" "),
  160. indent)
  161. }
  162. func (rs *RoundState) StringShort() string {
  163. return fmt.Sprintf(`RoundState{H:%v R:%v S:%v ST:%v}`,
  164. rs.Height, rs.Round, rs.Step, rs.StartTime)
  165. }
  166. //-----------------------------------------------------------------------------
  167. var (
  168. msgQueueSize = 1000
  169. tickTockBufferSize = 10
  170. )
  171. // msgs from the reactor which may update the state
  172. type msgInfo struct {
  173. Msg ConsensusMessage `json:"msg"`
  174. PeerKey string `json:"peer_key"`
  175. }
  176. // internally generated messages which may update the state
  177. type timeoutInfo struct {
  178. Duration time.Duration `json:"duration"`
  179. Height int `json:"height"`
  180. Round int `json:"round"`
  181. Step RoundStepType `json:"step"`
  182. }
  183. func (ti *timeoutInfo) String() string {
  184. return fmt.Sprintf("%v ; %d/%d %v", ti.Duration, ti.Height, ti.Round, ti.Step)
  185. }
  186. type PrivValidator interface {
  187. GetAddress() []byte
  188. SignVote(chainID string, vote *types.Vote) error
  189. SignProposal(chainID string, proposal *types.Proposal) error
  190. }
  191. // Tracks consensus state across block heights and rounds.
  192. type ConsensusState struct {
  193. BaseService
  194. config cfg.Config
  195. proxyAppConn proxy.AppConnConsensus
  196. blockStore *bc.BlockStore
  197. mempool *mempl.Mempool
  198. privValidator PrivValidator
  199. mtx sync.Mutex
  200. RoundState
  201. state *sm.State // State until height-1.
  202. peerMsgQueue chan msgInfo // serializes msgs affecting state (proposals, block parts, votes)
  203. internalMsgQueue chan msgInfo // like peerMsgQueue but for our own proposals, parts, votes
  204. timeoutTicker *time.Ticker // ticker for timeouts
  205. tickChan chan timeoutInfo // start the timeoutTicker in the timeoutRoutine
  206. tockChan chan timeoutInfo // timeouts are relayed on tockChan to the receiveRoutine
  207. timeoutParams *TimeoutParams // parameters and functions for timeout intervals
  208. evsw types.EventSwitch
  209. wal *WAL
  210. replayMode bool // so we don't log signing errors during replay
  211. nSteps int // used for testing to limit the number of transitions the state makes
  212. // allow certain function to be overwritten for testing
  213. decideProposal func(height, round int)
  214. doPrevote func(height, round int)
  215. setProposal func(proposal *types.Proposal) error
  216. }
  217. func NewConsensusState(config cfg.Config, state *sm.State, proxyAppConn proxy.AppConnConsensus, blockStore *bc.BlockStore, mempool *mempl.Mempool) *ConsensusState {
  218. cs := &ConsensusState{
  219. config: config,
  220. proxyAppConn: proxyAppConn,
  221. blockStore: blockStore,
  222. mempool: mempool,
  223. peerMsgQueue: make(chan msgInfo, msgQueueSize),
  224. internalMsgQueue: make(chan msgInfo, msgQueueSize),
  225. timeoutTicker: new(time.Ticker),
  226. tickChan: make(chan timeoutInfo, tickTockBufferSize),
  227. tockChan: make(chan timeoutInfo, tickTockBufferSize),
  228. timeoutParams: InitTimeoutParamsFromConfig(config),
  229. }
  230. // set function defaults (may be overwritten before calling Start)
  231. cs.decideProposal = cs.defaultDecideProposal
  232. cs.doPrevote = cs.defaultDoPrevote
  233. cs.setProposal = cs.defaultSetProposal
  234. cs.updateToState(state)
  235. // Don't call scheduleRound0 yet.
  236. // We do that upon Start().
  237. cs.reconstructLastCommit(state)
  238. cs.BaseService = *NewBaseService(log, "ConsensusState", cs)
  239. return cs
  240. }
  241. //----------------------------------------
  242. // Public interface
  243. // implements events.Eventable
  244. func (cs *ConsensusState) SetEventSwitch(evsw types.EventSwitch) {
  245. cs.evsw = evsw
  246. }
  247. func (cs *ConsensusState) String() string {
  248. // better not to access shared variables
  249. return Fmt("ConsensusState") //(H:%v R:%v S:%v", cs.Height, cs.Round, cs.Step)
  250. }
  251. func (cs *ConsensusState) GetState() *sm.State {
  252. cs.mtx.Lock()
  253. defer cs.mtx.Unlock()
  254. return cs.state.Copy()
  255. }
  256. func (cs *ConsensusState) GetRoundState() *RoundState {
  257. cs.mtx.Lock()
  258. defer cs.mtx.Unlock()
  259. return cs.getRoundState()
  260. }
  261. func (cs *ConsensusState) getRoundState() *RoundState {
  262. rs := cs.RoundState // copy
  263. return &rs
  264. }
  265. func (cs *ConsensusState) GetValidators() (int, []*types.Validator) {
  266. cs.mtx.Lock()
  267. defer cs.mtx.Unlock()
  268. return cs.state.LastBlockHeight, cs.state.Validators.Copy().Validators
  269. }
  270. func (cs *ConsensusState) SetPrivValidator(priv PrivValidator) {
  271. cs.mtx.Lock()
  272. defer cs.mtx.Unlock()
  273. cs.privValidator = priv
  274. }
  275. func (cs *ConsensusState) OnStart() error {
  276. cs.BaseService.OnStart()
  277. walDir := cs.config.GetString("cs_wal_dir")
  278. err := EnsureDir(walDir, 0700)
  279. if err != nil {
  280. log.Error("Error ensuring ConsensusState wal dir", "error", err.Error())
  281. return err
  282. }
  283. err = cs.OpenWAL(walDir)
  284. if err != nil {
  285. log.Error("Error loading ConsensusState wal", "error", err.Error())
  286. return err
  287. }
  288. // we need the timeoutRoutine for replay so
  289. // we don't block on the tick chan.
  290. // NOTE: we will get a build up of garbage go routines
  291. // firing on the tockChan until the receiveRoutine is started
  292. // to deal with them (by that point, at most one will be valid)
  293. go cs.timeoutRoutine()
  294. // we may have lost some votes if the process crashed
  295. // reload from consensus log to catchup
  296. if err := cs.catchupReplay(cs.Height); err != nil {
  297. log.Error("Error on catchup replay", "error", err.Error())
  298. // let's go for it anyways, maybe we're fine
  299. }
  300. // now start the receiveRoutine
  301. go cs.receiveRoutine(0)
  302. // schedule the first round!
  303. // use GetRoundState so we don't race the receiveRoutine for access
  304. cs.scheduleRound0(cs.GetRoundState())
  305. return nil
  306. }
  307. // timeoutRoutine: receive requests for timeouts on tickChan and fire timeouts on tockChan
  308. // receiveRoutine: serializes processing of proposoals, block parts, votes; coordinates state transitions
  309. func (cs *ConsensusState) startRoutines(maxSteps int) {
  310. go cs.timeoutRoutine()
  311. go cs.receiveRoutine(maxSteps)
  312. }
  313. func (cs *ConsensusState) OnStop() {
  314. cs.BaseService.OnStop()
  315. // Make BaseService.Wait() wait until cs.wal.Wait()
  316. if cs.wal != nil && cs.IsRunning() {
  317. cs.wal.Wait()
  318. }
  319. }
  320. // Open file to log all consensus messages and timeouts for deterministic accountability
  321. func (cs *ConsensusState) OpenWAL(walDir string) (err error) {
  322. cs.mtx.Lock()
  323. defer cs.mtx.Unlock()
  324. wal, err := NewWAL(walDir, cs.config.GetBool("cs_wal_light"))
  325. if err != nil {
  326. return err
  327. }
  328. cs.wal = wal
  329. return nil
  330. }
  331. //------------------------------------------------------------
  332. // Public interface for passing messages into the consensus state,
  333. // possibly causing a state transition
  334. // TODO: should these return anything or let callers just use events?
  335. // May block on send if queue is full.
  336. func (cs *ConsensusState) AddVote(vote *types.Vote, peerKey string) (added bool, err error) {
  337. if peerKey == "" {
  338. cs.internalMsgQueue <- msgInfo{&VoteMessage{vote}, ""}
  339. } else {
  340. cs.peerMsgQueue <- msgInfo{&VoteMessage{vote}, peerKey}
  341. }
  342. // TODO: wait for event?!
  343. return false, nil
  344. }
  345. // May block on send if queue is full.
  346. func (cs *ConsensusState) SetProposal(proposal *types.Proposal, peerKey string) error {
  347. if peerKey == "" {
  348. cs.internalMsgQueue <- msgInfo{&ProposalMessage{proposal}, ""}
  349. } else {
  350. cs.peerMsgQueue <- msgInfo{&ProposalMessage{proposal}, peerKey}
  351. }
  352. // TODO: wait for event?!
  353. return nil
  354. }
  355. // May block on send if queue is full.
  356. func (cs *ConsensusState) AddProposalBlockPart(height, round int, part *types.Part, peerKey string) error {
  357. if peerKey == "" {
  358. cs.internalMsgQueue <- msgInfo{&BlockPartMessage{height, round, part}, ""}
  359. } else {
  360. cs.peerMsgQueue <- msgInfo{&BlockPartMessage{height, round, part}, peerKey}
  361. }
  362. // TODO: wait for event?!
  363. return nil
  364. }
  365. // May block on send if queue is full.
  366. func (cs *ConsensusState) SetProposalAndBlock(proposal *types.Proposal, block *types.Block, parts *types.PartSet, peerKey string) error {
  367. cs.SetProposal(proposal, peerKey)
  368. for i := 0; i < parts.Total(); i++ {
  369. part := parts.GetPart(i)
  370. cs.AddProposalBlockPart(proposal.Height, proposal.Round, part, peerKey)
  371. }
  372. return nil // TODO errors
  373. }
  374. //------------------------------------------------------------
  375. // internal functions for managing the state
  376. func (cs *ConsensusState) updateHeight(height int) {
  377. cs.Height = height
  378. }
  379. func (cs *ConsensusState) updateRoundStep(round int, step RoundStepType) {
  380. cs.Round = round
  381. cs.Step = step
  382. }
  383. // enterNewRound(height, 0) at cs.StartTime.
  384. func (cs *ConsensusState) scheduleRound0(rs *RoundState) {
  385. //log.Info("scheduleRound0", "now", time.Now(), "startTime", cs.StartTime)
  386. sleepDuration := rs.StartTime.Sub(time.Now())
  387. if sleepDuration < time.Duration(0) {
  388. sleepDuration = time.Duration(0)
  389. }
  390. cs.scheduleTimeout(sleepDuration, rs.Height, 0, RoundStepNewHeight)
  391. }
  392. // Attempt to schedule a timeout by sending timeoutInfo on the tickChan.
  393. // The timeoutRoutine is alwaya available to read from tickChan (it won't block).
  394. // The scheduling may fail if the timeoutRoutine has already scheduled a timeout for a later height/round/step.
  395. func (cs *ConsensusState) scheduleTimeout(duration time.Duration, height, round int, step RoundStepType) {
  396. cs.tickChan <- timeoutInfo{duration, height, round, step}
  397. }
  398. // send a msg into the receiveRoutine regarding our own proposal, block part, or vote
  399. func (cs *ConsensusState) sendInternalMessage(mi msgInfo) {
  400. select {
  401. case cs.internalMsgQueue <- mi:
  402. default:
  403. // NOTE: using the go-routine means our votes can
  404. // be processed out of order.
  405. // TODO: use CList here for strict determinism and
  406. // attempt push to internalMsgQueue in receiveRoutine
  407. log.Warn("Internal msg queue is full. Using a go-routine")
  408. go func() { cs.internalMsgQueue <- mi }()
  409. }
  410. }
  411. // Reconstruct LastCommit from SeenCommit, which we saved along with the block,
  412. // (which happens even before saving the state)
  413. func (cs *ConsensusState) reconstructLastCommit(state *sm.State) {
  414. if state.LastBlockHeight == 0 {
  415. return
  416. }
  417. seenCommit := cs.blockStore.LoadSeenCommit(state.LastBlockHeight)
  418. lastPrecommits := types.NewVoteSet(cs.config.GetString("chain_id"), state.LastBlockHeight, seenCommit.Round(), types.VoteTypePrecommit, state.LastValidators)
  419. for _, precommit := range seenCommit.Precommits {
  420. if precommit == nil {
  421. continue
  422. }
  423. added, err := lastPrecommits.AddVote(precommit)
  424. if !added || err != nil {
  425. PanicCrisis(Fmt("Failed to reconstruct LastCommit: %v", err))
  426. }
  427. }
  428. if !lastPrecommits.HasTwoThirdsMajority() {
  429. PanicSanity("Failed to reconstruct LastCommit: Does not have +2/3 maj")
  430. }
  431. cs.LastCommit = lastPrecommits
  432. }
  433. // Updates ConsensusState and increments height to match that of state.
  434. // The round becomes 0 and cs.Step becomes RoundStepNewHeight.
  435. func (cs *ConsensusState) updateToState(state *sm.State) {
  436. if cs.CommitRound > -1 && 0 < cs.Height && cs.Height != state.LastBlockHeight {
  437. PanicSanity(Fmt("updateToState() expected state height of %v but found %v",
  438. cs.Height, state.LastBlockHeight))
  439. }
  440. if cs.state != nil && cs.state.LastBlockHeight+1 != cs.Height {
  441. // This might happen when someone else is mutating cs.state.
  442. // Someone forgot to pass in state.Copy() somewhere?!
  443. PanicSanity(Fmt("Inconsistent cs.state.LastBlockHeight+1 %v vs cs.Height %v",
  444. cs.state.LastBlockHeight+1, cs.Height))
  445. }
  446. // If state isn't further out than cs.state, just ignore.
  447. // This happens when SwitchToConsensus() is called in the reactor.
  448. // We don't want to reset e.g. the Votes.
  449. if cs.state != nil && (state.LastBlockHeight <= cs.state.LastBlockHeight) {
  450. log.Notice("Ignoring updateToState()", "newHeight", state.LastBlockHeight+1, "oldHeight", cs.state.LastBlockHeight+1)
  451. return
  452. }
  453. // Reset fields based on state.
  454. validators := state.Validators
  455. height := state.LastBlockHeight + 1 // Next desired block height
  456. lastPrecommits := (*types.VoteSet)(nil)
  457. if cs.CommitRound > -1 && cs.Votes != nil {
  458. if !cs.Votes.Precommits(cs.CommitRound).HasTwoThirdsMajority() {
  459. PanicSanity("updateToState(state) called but last Precommit round didn't have +2/3")
  460. }
  461. lastPrecommits = cs.Votes.Precommits(cs.CommitRound)
  462. }
  463. // RoundState fields
  464. cs.updateHeight(height)
  465. cs.updateRoundStep(0, RoundStepNewHeight)
  466. if cs.CommitTime.IsZero() {
  467. // "Now" makes it easier to sync up dev nodes.
  468. // We add timeoutCommit to allow transactions
  469. // to be gathered for the first block.
  470. // And alternative solution that relies on clocks:
  471. // cs.StartTime = state.LastBlockTime.Add(timeoutCommit)
  472. cs.StartTime = cs.timeoutParams.Commit(time.Now())
  473. } else {
  474. cs.StartTime = cs.timeoutParams.Commit(cs.CommitTime)
  475. }
  476. cs.CommitTime = time.Time{}
  477. cs.Validators = validators
  478. cs.Proposal = nil
  479. cs.ProposalBlock = nil
  480. cs.ProposalBlockParts = nil
  481. cs.LockedRound = 0
  482. cs.LockedBlock = nil
  483. cs.LockedBlockParts = nil
  484. cs.Votes = NewHeightVoteSet(cs.config.GetString("chain_id"), height, validators)
  485. cs.CommitRound = -1
  486. cs.LastCommit = lastPrecommits
  487. cs.LastValidators = state.LastValidators
  488. cs.state = state
  489. // Finally, broadcast RoundState
  490. cs.newStep()
  491. }
  492. func (cs *ConsensusState) newStep() {
  493. rs := cs.RoundStateEvent()
  494. cs.wal.Save(rs)
  495. cs.nSteps += 1
  496. // newStep is called by updateToStep in NewConsensusState before the evsw is set!
  497. if cs.evsw != nil {
  498. types.FireEventNewRoundStep(cs.evsw, rs)
  499. }
  500. }
  501. //-----------------------------------------
  502. // the main go routines
  503. // the state machine sends on tickChan to start a new timer.
  504. // timers are interupted and replaced by new ticks from later steps
  505. // timeouts of 0 on the tickChan will be immediately relayed to the tockChan
  506. func (cs *ConsensusState) timeoutRoutine() {
  507. log.Debug("Starting timeout routine")
  508. var ti timeoutInfo
  509. for {
  510. select {
  511. case newti := <-cs.tickChan:
  512. log.Debug("Received tick", "old_ti", ti, "new_ti", newti)
  513. // ignore tickers for old height/round/step
  514. if newti.Height < ti.Height {
  515. continue
  516. } else if newti.Height == ti.Height {
  517. if newti.Round < ti.Round {
  518. continue
  519. } else if newti.Round == ti.Round {
  520. if ti.Step > 0 && newti.Step <= ti.Step {
  521. continue
  522. }
  523. }
  524. }
  525. ti = newti
  526. // if the newti has duration == 0, we relay to the tockChan immediately (no timeout)
  527. if ti.Duration == time.Duration(0) {
  528. go func(t timeoutInfo) { cs.tockChan <- t }(ti)
  529. continue
  530. }
  531. log.Debug("Scheduling timeout", "dur", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step)
  532. cs.timeoutTicker.Stop()
  533. cs.timeoutTicker = time.NewTicker(ti.Duration)
  534. case <-cs.timeoutTicker.C:
  535. log.Info("Timed out", "dur", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step)
  536. cs.timeoutTicker.Stop()
  537. // go routine here gaurantees timeoutRoutine doesn't block.
  538. // Determinism comes from playback in the receiveRoutine.
  539. // We can eliminate it by merging the timeoutRoutine into receiveRoutine
  540. // and managing the timeouts ourselves with a millisecond ticker
  541. go func(t timeoutInfo) { cs.tockChan <- t }(ti)
  542. case <-cs.Quit:
  543. return
  544. }
  545. }
  546. }
  547. // a nice idea but probably more trouble than its worth
  548. func (cs *ConsensusState) stopTimer() {
  549. cs.timeoutTicker.Stop()
  550. }
  551. // receiveRoutine handles messages which may cause state transitions.
  552. // it's argument (n) is the number of messages to process before exiting - use 0 to run forever
  553. // It keeps the RoundState and is the only thing that updates it.
  554. // Updates (state transitions) happen on timeouts, complete proposals, and 2/3 majorities
  555. func (cs *ConsensusState) receiveRoutine(maxSteps int) {
  556. for {
  557. if maxSteps > 0 {
  558. if cs.nSteps >= maxSteps {
  559. log.Warn("reached max steps. exiting receive routine")
  560. cs.nSteps = 0
  561. return
  562. }
  563. }
  564. rs := cs.RoundState
  565. var mi msgInfo
  566. select {
  567. case mi = <-cs.peerMsgQueue:
  568. cs.wal.Save(mi)
  569. // handles proposals, block parts, votes
  570. // may generate internal events (votes, complete proposals, 2/3 majorities)
  571. cs.handleMsg(mi, rs)
  572. case mi = <-cs.internalMsgQueue:
  573. cs.wal.Save(mi)
  574. // handles proposals, block parts, votes
  575. cs.handleMsg(mi, rs)
  576. case ti := <-cs.tockChan:
  577. cs.wal.Save(ti)
  578. // if the timeout is relevant to the rs
  579. // go to the next step
  580. cs.handleTimeout(ti, rs)
  581. case <-cs.Quit:
  582. // drain the internalMsgQueue in case we eg. signed a proposal but it didn't hit the wal
  583. FLUSH:
  584. for {
  585. select {
  586. case mi = <-cs.internalMsgQueue:
  587. cs.wal.Save(mi)
  588. cs.handleMsg(mi, rs)
  589. default:
  590. break FLUSH
  591. }
  592. }
  593. // close wal now that we're done writing to it
  594. if cs.wal != nil {
  595. cs.wal.Stop()
  596. }
  597. return
  598. }
  599. }
  600. }
  601. // state transitions on complete-proposal, 2/3-any, 2/3-one
  602. func (cs *ConsensusState) handleMsg(mi msgInfo, rs RoundState) {
  603. cs.mtx.Lock()
  604. defer cs.mtx.Unlock()
  605. var err error
  606. msg, peerKey := mi.Msg, mi.PeerKey
  607. switch msg := msg.(type) {
  608. case *ProposalMessage:
  609. // will not cause transition.
  610. // once proposal is set, we can receive block parts
  611. err = cs.setProposal(msg.Proposal)
  612. case *BlockPartMessage:
  613. // if the proposal is complete, we'll enterPrevote or tryFinalizeCommit
  614. _, err = cs.addProposalBlockPart(msg.Height, msg.Part, peerKey != "")
  615. if err != nil && msg.Round != cs.Round {
  616. err = nil
  617. }
  618. case *VoteMessage:
  619. // attempt to add the vote and dupeout the validator if its a duplicate signature
  620. // if the vote gives us a 2/3-any or 2/3-one, we transition
  621. err := cs.tryAddVote(msg.Vote, peerKey)
  622. if err == ErrAddingVote {
  623. // TODO: punish peer
  624. }
  625. // NOTE: the vote is broadcast to peers by the reactor listening
  626. // for vote events
  627. // TODO: If rs.Height == vote.Height && rs.Round < vote.Round,
  628. // the peer is sending us CatchupCommit precommits.
  629. // We could make note of this and help filter in broadcastHasVoteMessage().
  630. default:
  631. log.Warn("Unknown msg type", reflect.TypeOf(msg))
  632. }
  633. if err != nil {
  634. log.Error("Error with msg", "type", reflect.TypeOf(msg), "peer", peerKey, "error", err, "msg", msg)
  635. }
  636. }
  637. func (cs *ConsensusState) handleTimeout(ti timeoutInfo, rs RoundState) {
  638. log.Debug("Received tock", "timeout", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step)
  639. // timeouts must be for current height, round, step
  640. if ti.Height != rs.Height || ti.Round < rs.Round || (ti.Round == rs.Round && ti.Step < rs.Step) {
  641. log.Debug("Ignoring tock because we're ahead", "height", rs.Height, "round", rs.Round, "step", rs.Step)
  642. return
  643. }
  644. // the timeout will now cause a state transition
  645. cs.mtx.Lock()
  646. defer cs.mtx.Unlock()
  647. switch ti.Step {
  648. case RoundStepNewHeight:
  649. // NewRound event fired from enterNewRound.
  650. // XXX: should we fire timeout here?
  651. cs.enterNewRound(ti.Height, 0)
  652. case RoundStepPropose:
  653. types.FireEventTimeoutPropose(cs.evsw, cs.RoundStateEvent())
  654. cs.enterPrevote(ti.Height, ti.Round)
  655. case RoundStepPrevoteWait:
  656. types.FireEventTimeoutWait(cs.evsw, cs.RoundStateEvent())
  657. cs.enterPrecommit(ti.Height, ti.Round)
  658. case RoundStepPrecommitWait:
  659. types.FireEventTimeoutWait(cs.evsw, cs.RoundStateEvent())
  660. cs.enterNewRound(ti.Height, ti.Round+1)
  661. default:
  662. panic(Fmt("Invalid timeout step: %v", ti.Step))
  663. }
  664. }
  665. //-----------------------------------------------------------------------------
  666. // State functions
  667. // Used internally by handleTimeout and handleMsg to make state transitions
  668. // Enter: +2/3 precommits for nil at (height,round-1)
  669. // Enter: `timeoutPrecommits` after any +2/3 precommits from (height,round-1)
  670. // Enter: `startTime = commitTime+timeoutCommit` from NewHeight(height)
  671. // NOTE: cs.StartTime was already set for height.
  672. func (cs *ConsensusState) enterNewRound(height int, round int) {
  673. if cs.Height != height || round < cs.Round || (cs.Round == round && cs.Step != RoundStepNewHeight) {
  674. log.Debug(Fmt("enterNewRound(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  675. return
  676. }
  677. if now := time.Now(); cs.StartTime.After(now) {
  678. log.Warn("Need to set a buffer and log.Warn() here for sanity.", "startTime", cs.StartTime, "now", now)
  679. }
  680. // cs.stopTimer()
  681. log.Notice(Fmt("enterNewRound(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  682. // Increment validators if necessary
  683. validators := cs.Validators
  684. if cs.Round < round {
  685. validators = validators.Copy()
  686. validators.IncrementAccum(round - cs.Round)
  687. }
  688. // Setup new round
  689. // we don't fire newStep for this step,
  690. // but we fire an event, so update the round step first
  691. cs.updateRoundStep(round, RoundStepNewRound)
  692. cs.Validators = validators
  693. if round == 0 {
  694. // We've already reset these upon new height,
  695. // and meanwhile we might have received a proposal
  696. // for round 0.
  697. } else {
  698. cs.Proposal = nil
  699. cs.ProposalBlock = nil
  700. cs.ProposalBlockParts = nil
  701. }
  702. cs.Votes.SetRound(round + 1) // also track next round (round+1) to allow round-skipping
  703. types.FireEventNewRound(cs.evsw, cs.RoundStateEvent())
  704. // Immediately go to enterPropose.
  705. cs.enterPropose(height, round)
  706. }
  707. // Enter: from NewRound(height,round).
  708. func (cs *ConsensusState) enterPropose(height int, round int) {
  709. if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPropose <= cs.Step) {
  710. log.Debug(Fmt("enterPropose(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  711. return
  712. }
  713. log.Info(Fmt("enterPropose(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  714. defer func() {
  715. // Done enterPropose:
  716. cs.updateRoundStep(round, RoundStepPropose)
  717. cs.newStep()
  718. // If we have the whole proposal + POL, then goto Prevote now.
  719. // else, we'll enterPrevote when the rest of the proposal is received (in AddProposalBlockPart),
  720. // or else after timeoutPropose
  721. if cs.isProposalComplete() {
  722. cs.enterPrevote(height, cs.Round)
  723. }
  724. }()
  725. // If we don't get the proposal and all block parts quick enough, enterPrevote
  726. cs.scheduleTimeout(cs.timeoutParams.Propose(round), height, round, RoundStepPropose)
  727. // Nothing more to do if we're not a validator
  728. if cs.privValidator == nil {
  729. return
  730. }
  731. if !bytes.Equal(cs.Validators.Proposer().Address, cs.privValidator.GetAddress()) {
  732. log.Info("enterPropose: Not our turn to propose", "proposer", cs.Validators.Proposer().Address, "privValidator", cs.privValidator)
  733. } else {
  734. log.Info("enterPropose: Our turn to propose", "proposer", cs.Validators.Proposer().Address, "privValidator", cs.privValidator)
  735. cs.decideProposal(height, round)
  736. }
  737. }
  738. func (cs *ConsensusState) defaultDecideProposal(height, round int) {
  739. var block *types.Block
  740. var blockParts *types.PartSet
  741. // Decide on block
  742. if cs.LockedBlock != nil {
  743. // If we're locked onto a block, just choose that.
  744. block, blockParts = cs.LockedBlock, cs.LockedBlockParts
  745. } else {
  746. // Create a new proposal block from state/txs from the mempool.
  747. block, blockParts = cs.createProposalBlock()
  748. if block == nil { // on error
  749. return
  750. }
  751. }
  752. // Make proposal
  753. polRound, polBlockID := cs.Votes.POLInfo()
  754. proposal := types.NewProposal(height, round, blockParts.Header(), polRound, polBlockID)
  755. err := cs.privValidator.SignProposal(cs.state.ChainID, proposal)
  756. if err == nil {
  757. // Set fields
  758. /* fields set by setProposal and addBlockPart
  759. cs.Proposal = proposal
  760. cs.ProposalBlock = block
  761. cs.ProposalBlockParts = blockParts
  762. */
  763. // send proposal and block parts on internal msg queue
  764. cs.sendInternalMessage(msgInfo{&ProposalMessage{proposal}, ""})
  765. for i := 0; i < blockParts.Total(); i++ {
  766. part := blockParts.GetPart(i)
  767. cs.sendInternalMessage(msgInfo{&BlockPartMessage{cs.Height, cs.Round, part}, ""})
  768. }
  769. log.Info("Signed proposal", "height", height, "round", round, "proposal", proposal)
  770. log.Debug(Fmt("Signed proposal block: %v", block))
  771. } else {
  772. if !cs.replayMode {
  773. log.Warn("enterPropose: Error signing proposal", "height", height, "round", round, "error", err)
  774. }
  775. }
  776. }
  777. // Returns true if the proposal block is complete &&
  778. // (if POLRound was proposed, we have +2/3 prevotes from there).
  779. func (cs *ConsensusState) isProposalComplete() bool {
  780. if cs.Proposal == nil || cs.ProposalBlock == nil {
  781. return false
  782. }
  783. // we have the proposal. if there's a POLRound,
  784. // make sure we have the prevotes from it too
  785. if cs.Proposal.POLRound < 0 {
  786. return true
  787. } else {
  788. // if this is false the proposer is lying or we haven't received the POL yet
  789. return cs.Votes.Prevotes(cs.Proposal.POLRound).HasTwoThirdsMajority()
  790. }
  791. }
  792. // Create the next block to propose and return it.
  793. // Returns nil block upon error.
  794. // NOTE: keep it side-effect free for clarity.
  795. func (cs *ConsensusState) createProposalBlock() (block *types.Block, blockParts *types.PartSet) {
  796. var commit *types.Commit
  797. if cs.Height == 1 {
  798. // We're creating a proposal for the first block.
  799. // The commit is empty, but not nil.
  800. commit = &types.Commit{}
  801. } else if cs.LastCommit.HasTwoThirdsMajority() {
  802. // Make the commit from LastCommit
  803. commit = cs.LastCommit.MakeCommit()
  804. } else {
  805. // This shouldn't happen.
  806. log.Error("enterPropose: Cannot propose anything: No commit for the previous block.")
  807. return
  808. }
  809. // Mempool validated transactions
  810. txs := cs.mempool.Reap(cs.config.GetInt("block_size"))
  811. return types.MakeBlock(cs.Height, cs.state.ChainID, txs, commit,
  812. cs.state.LastBlockID, cs.state.Validators.Hash(), cs.state.AppHash, cs.config.GetInt("block_part_size"))
  813. }
  814. // Enter: `timeoutPropose` after entering Propose.
  815. // Enter: proposal block and POL is ready.
  816. // Enter: any +2/3 prevotes for future round.
  817. // Prevote for LockedBlock if we're locked, or ProposalBlock if valid.
  818. // Otherwise vote nil.
  819. func (cs *ConsensusState) enterPrevote(height int, round int) {
  820. if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPrevote <= cs.Step) {
  821. log.Debug(Fmt("enterPrevote(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  822. return
  823. }
  824. defer func() {
  825. // Done enterPrevote:
  826. cs.updateRoundStep(round, RoundStepPrevote)
  827. cs.newStep()
  828. }()
  829. // fire event for how we got here
  830. if cs.isProposalComplete() {
  831. types.FireEventCompleteProposal(cs.evsw, cs.RoundStateEvent())
  832. } else {
  833. // we received +2/3 prevotes for a future round
  834. // TODO: catchup event?
  835. }
  836. // cs.stopTimer()
  837. log.Info(Fmt("enterPrevote(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  838. // Sign and broadcast vote as necessary
  839. cs.doPrevote(height, round)
  840. // Once `addVote` hits any +2/3 prevotes, we will go to PrevoteWait
  841. // (so we have more time to try and collect +2/3 prevotes for a single block)
  842. }
  843. func (cs *ConsensusState) defaultDoPrevote(height int, round int) {
  844. // If a block is locked, prevote that.
  845. if cs.LockedBlock != nil {
  846. log.Notice("enterPrevote: Block was locked")
  847. cs.signAddVote(types.VoteTypePrevote, cs.LockedBlock.Hash(), cs.LockedBlockParts.Header())
  848. return
  849. }
  850. // If ProposalBlock is nil, prevote nil.
  851. if cs.ProposalBlock == nil {
  852. log.Warn("enterPrevote: ProposalBlock is nil")
  853. cs.signAddVote(types.VoteTypePrevote, nil, types.PartSetHeader{})
  854. return
  855. }
  856. // Valdiate proposal block
  857. err := cs.state.ValidateBlock(cs.ProposalBlock)
  858. if err != nil {
  859. // ProposalBlock is invalid, prevote nil.
  860. log.Warn("enterPrevote: ProposalBlock is invalid", "error", err)
  861. cs.signAddVote(types.VoteTypePrevote, nil, types.PartSetHeader{})
  862. return
  863. }
  864. // Prevote cs.ProposalBlock
  865. // NOTE: the proposal signature is validated when it is received,
  866. // and the proposal block parts are validated as they are received (against the merkle hash in the proposal)
  867. cs.signAddVote(types.VoteTypePrevote, cs.ProposalBlock.Hash(), cs.ProposalBlockParts.Header())
  868. return
  869. }
  870. // Enter: any +2/3 prevotes at next round.
  871. func (cs *ConsensusState) enterPrevoteWait(height int, round int) {
  872. if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPrevoteWait <= cs.Step) {
  873. log.Debug(Fmt("enterPrevoteWait(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  874. return
  875. }
  876. if !cs.Votes.Prevotes(round).HasTwoThirdsAny() {
  877. PanicSanity(Fmt("enterPrevoteWait(%v/%v), but Prevotes does not have any +2/3 votes", height, round))
  878. }
  879. log.Info(Fmt("enterPrevoteWait(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  880. defer func() {
  881. // Done enterPrevoteWait:
  882. cs.updateRoundStep(round, RoundStepPrevoteWait)
  883. cs.newStep()
  884. }()
  885. // Wait for some more prevotes; enterPrecommit
  886. cs.scheduleTimeout(cs.timeoutParams.Prevote(round), height, round, RoundStepPrevoteWait)
  887. }
  888. // Enter: +2/3 precomits for block or nil.
  889. // Enter: `timeoutPrevote` after any +2/3 prevotes.
  890. // Enter: any +2/3 precommits for next round.
  891. // Lock & precommit the ProposalBlock if we have enough prevotes for it (a POL in this round)
  892. // else, unlock an existing lock and precommit nil if +2/3 of prevotes were nil,
  893. // else, precommit nil otherwise.
  894. func (cs *ConsensusState) enterPrecommit(height int, round int) {
  895. if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPrecommit <= cs.Step) {
  896. log.Debug(Fmt("enterPrecommit(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  897. return
  898. }
  899. // cs.stopTimer()
  900. log.Info(Fmt("enterPrecommit(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  901. defer func() {
  902. // Done enterPrecommit:
  903. cs.updateRoundStep(round, RoundStepPrecommit)
  904. cs.newStep()
  905. }()
  906. blockID, ok := cs.Votes.Prevotes(round).TwoThirdsMajority()
  907. // If we don't have a polka, we must precommit nil
  908. if !ok {
  909. if cs.LockedBlock != nil {
  910. log.Notice("enterPrecommit: No +2/3 prevotes during enterPrecommit while we're locked. Precommitting nil")
  911. } else {
  912. log.Notice("enterPrecommit: No +2/3 prevotes during enterPrecommit. Precommitting nil.")
  913. }
  914. cs.signAddVote(types.VoteTypePrecommit, nil, types.PartSetHeader{})
  915. return
  916. }
  917. // At this point +2/3 prevoted for a particular block or nil
  918. types.FireEventPolka(cs.evsw, cs.RoundStateEvent())
  919. // the latest POLRound should be this round
  920. polRound, _ := cs.Votes.POLInfo()
  921. if polRound < round {
  922. PanicSanity(Fmt("This POLRound should be %v but got %", round, polRound))
  923. }
  924. // +2/3 prevoted nil. Unlock and precommit nil.
  925. if len(blockID.Hash) == 0 {
  926. if cs.LockedBlock == nil {
  927. log.Notice("enterPrecommit: +2/3 prevoted for nil.")
  928. } else {
  929. log.Notice("enterPrecommit: +2/3 prevoted for nil. Unlocking")
  930. cs.LockedRound = 0
  931. cs.LockedBlock = nil
  932. cs.LockedBlockParts = nil
  933. types.FireEventUnlock(cs.evsw, cs.RoundStateEvent())
  934. }
  935. cs.signAddVote(types.VoteTypePrecommit, nil, types.PartSetHeader{})
  936. return
  937. }
  938. // At this point, +2/3 prevoted for a particular block.
  939. // If we're already locked on that block, precommit it, and update the LockedRound
  940. if cs.LockedBlock.HashesTo(blockID.Hash) {
  941. log.Notice("enterPrecommit: +2/3 prevoted locked block. Relocking")
  942. cs.LockedRound = round
  943. types.FireEventRelock(cs.evsw, cs.RoundStateEvent())
  944. cs.signAddVote(types.VoteTypePrecommit, blockID.Hash, blockID.PartsHeader)
  945. return
  946. }
  947. // If +2/3 prevoted for proposal block, stage and precommit it
  948. if cs.ProposalBlock.HashesTo(blockID.Hash) {
  949. log.Notice("enterPrecommit: +2/3 prevoted proposal block. Locking", "hash", blockID.Hash)
  950. // Validate the block.
  951. if err := cs.state.ValidateBlock(cs.ProposalBlock); err != nil {
  952. PanicConsensus(Fmt("enterPrecommit: +2/3 prevoted for an invalid block: %v", err))
  953. }
  954. cs.LockedRound = round
  955. cs.LockedBlock = cs.ProposalBlock
  956. cs.LockedBlockParts = cs.ProposalBlockParts
  957. types.FireEventLock(cs.evsw, cs.RoundStateEvent())
  958. cs.signAddVote(types.VoteTypePrecommit, blockID.Hash, blockID.PartsHeader)
  959. return
  960. }
  961. // There was a polka in this round for a block we don't have.
  962. // Fetch that block, unlock, and precommit nil.
  963. // The +2/3 prevotes for this round is the POL for our unlock.
  964. // TODO: In the future save the POL prevotes for justification.
  965. cs.LockedRound = 0
  966. cs.LockedBlock = nil
  967. cs.LockedBlockParts = nil
  968. if !cs.ProposalBlockParts.HasHeader(blockID.PartsHeader) {
  969. cs.ProposalBlock = nil
  970. cs.ProposalBlockParts = types.NewPartSetFromHeader(blockID.PartsHeader)
  971. }
  972. types.FireEventUnlock(cs.evsw, cs.RoundStateEvent())
  973. cs.signAddVote(types.VoteTypePrecommit, nil, types.PartSetHeader{})
  974. return
  975. }
  976. // Enter: any +2/3 precommits for next round.
  977. func (cs *ConsensusState) enterPrecommitWait(height int, round int) {
  978. if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPrecommitWait <= cs.Step) {
  979. log.Debug(Fmt("enterPrecommitWait(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  980. return
  981. }
  982. if !cs.Votes.Precommits(round).HasTwoThirdsAny() {
  983. PanicSanity(Fmt("enterPrecommitWait(%v/%v), but Precommits does not have any +2/3 votes", height, round))
  984. }
  985. log.Info(Fmt("enterPrecommitWait(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  986. defer func() {
  987. // Done enterPrecommitWait:
  988. cs.updateRoundStep(round, RoundStepPrecommitWait)
  989. cs.newStep()
  990. }()
  991. // Wait for some more precommits; enterNewRound
  992. cs.scheduleTimeout(cs.timeoutParams.Precommit(round), height, round, RoundStepPrecommitWait)
  993. }
  994. // Enter: +2/3 precommits for block
  995. func (cs *ConsensusState) enterCommit(height int, commitRound int) {
  996. if cs.Height != height || RoundStepCommit <= cs.Step {
  997. log.Debug(Fmt("enterCommit(%v/%v): Invalid args. Current step: %v/%v/%v", height, commitRound, cs.Height, cs.Round, cs.Step))
  998. return
  999. }
  1000. log.Info(Fmt("enterCommit(%v/%v). Current: %v/%v/%v", height, commitRound, cs.Height, cs.Round, cs.Step))
  1001. defer func() {
  1002. // Done enterCommit:
  1003. // keep cs.Round the same, commitRound points to the right Precommits set.
  1004. cs.updateRoundStep(cs.Round, RoundStepCommit)
  1005. cs.CommitRound = commitRound
  1006. cs.newStep()
  1007. // Maybe finalize immediately.
  1008. cs.tryFinalizeCommit(height)
  1009. }()
  1010. blockID, ok := cs.Votes.Precommits(commitRound).TwoThirdsMajority()
  1011. if !ok {
  1012. PanicSanity("RunActionCommit() expects +2/3 precommits")
  1013. }
  1014. // The Locked* fields no longer matter.
  1015. // Move them over to ProposalBlock if they match the commit hash,
  1016. // otherwise they'll be cleared in updateToState.
  1017. if cs.LockedBlock.HashesTo(blockID.Hash) {
  1018. cs.ProposalBlock = cs.LockedBlock
  1019. cs.ProposalBlockParts = cs.LockedBlockParts
  1020. }
  1021. // If we don't have the block being committed, set up to get it.
  1022. if !cs.ProposalBlock.HashesTo(blockID.Hash) {
  1023. if !cs.ProposalBlockParts.HasHeader(blockID.PartsHeader) {
  1024. // We're getting the wrong block.
  1025. // Set up ProposalBlockParts and keep waiting.
  1026. cs.ProposalBlock = nil
  1027. cs.ProposalBlockParts = types.NewPartSetFromHeader(blockID.PartsHeader)
  1028. } else {
  1029. // We just need to keep waiting.
  1030. }
  1031. }
  1032. }
  1033. // If we have the block AND +2/3 commits for it, finalize.
  1034. func (cs *ConsensusState) tryFinalizeCommit(height int) {
  1035. if cs.Height != height {
  1036. PanicSanity(Fmt("tryFinalizeCommit() cs.Height: %v vs height: %v", cs.Height, height))
  1037. }
  1038. blockID, ok := cs.Votes.Precommits(cs.CommitRound).TwoThirdsMajority()
  1039. if !ok || len(blockID.Hash) == 0 {
  1040. log.Warn("Attempt to finalize failed. There was no +2/3 majority, or +2/3 was for <nil>.")
  1041. return
  1042. }
  1043. if !cs.ProposalBlock.HashesTo(blockID.Hash) {
  1044. // TODO: this happens every time if we're not a validator (ugly logs)
  1045. // TODO: ^^ wait, why does it matter that we're a validator?
  1046. log.Warn("Attempt to finalize failed. We don't have the commit block.")
  1047. return
  1048. }
  1049. // go
  1050. cs.finalizeCommit(height)
  1051. }
  1052. // Increment height and goto RoundStepNewHeight
  1053. func (cs *ConsensusState) finalizeCommit(height int) {
  1054. if cs.Height != height || cs.Step != RoundStepCommit {
  1055. log.Debug(Fmt("finalizeCommit(%v): Invalid args. Current step: %v/%v/%v", height, cs.Height, cs.Round, cs.Step))
  1056. return
  1057. }
  1058. blockID, ok := cs.Votes.Precommits(cs.CommitRound).TwoThirdsMajority()
  1059. block, blockParts := cs.ProposalBlock, cs.ProposalBlockParts
  1060. if !ok {
  1061. PanicSanity(Fmt("Cannot finalizeCommit, commit does not have two thirds majority"))
  1062. }
  1063. if !blockParts.HasHeader(blockID.PartsHeader) {
  1064. PanicSanity(Fmt("Expected ProposalBlockParts header to be commit header"))
  1065. }
  1066. if !block.HashesTo(blockID.Hash) {
  1067. PanicSanity(Fmt("Cannot finalizeCommit, ProposalBlock does not hash to commit hash"))
  1068. }
  1069. if err := cs.state.ValidateBlock(block); err != nil {
  1070. PanicConsensus(Fmt("+2/3 committed an invalid block: %v", err))
  1071. }
  1072. log.Notice(Fmt("Finalizing commit of block with %d txs", block.NumTxs),
  1073. "height", block.Height, "hash", block.Hash(), "root", block.AppHash)
  1074. log.Info(Fmt("%v", block))
  1075. fail.Fail() // XXX
  1076. // Save to blockStore.
  1077. if cs.blockStore.Height() < block.Height {
  1078. precommits := cs.Votes.Precommits(cs.CommitRound)
  1079. seenCommit := precommits.MakeCommit()
  1080. cs.blockStore.SaveBlock(block, blockParts, seenCommit)
  1081. } else {
  1082. log.Warn("Why are we finalizeCommitting a block height we already have?", "height", block.Height)
  1083. }
  1084. fail.Fail() // XXX
  1085. // Create a copy of the state for staging
  1086. // and an event cache for txs
  1087. stateCopy := cs.state.Copy()
  1088. eventCache := types.NewEventCache(cs.evsw)
  1089. // Execute and commit the block, and update the mempool.
  1090. // All calls to the proxyAppConn should come here.
  1091. // NOTE: the block.AppHash wont reflect these txs until the next block
  1092. stateCopy.ApplyBlock(eventCache, cs.proxyAppConn, block, blockParts.Header(), cs.mempool)
  1093. fail.Fail() // XXX
  1094. // Fire off event for new block.
  1095. // TODO: Handle app failure. See #177
  1096. types.FireEventNewBlock(cs.evsw, types.EventDataNewBlock{block})
  1097. types.FireEventNewBlockHeader(cs.evsw, types.EventDataNewBlockHeader{block.Header})
  1098. eventCache.Flush()
  1099. // Save the state.
  1100. stateCopy.Save()
  1101. fail.Fail() // XXX
  1102. // NewHeightStep!
  1103. cs.updateToState(stateCopy)
  1104. // cs.StartTime is already set.
  1105. // Schedule Round0 to start soon.
  1106. cs.scheduleRound0(&cs.RoundState)
  1107. // By here,
  1108. // * cs.Height has been increment to height+1
  1109. // * cs.Step is now RoundStepNewHeight
  1110. // * cs.StartTime is set to when we will start round0.
  1111. return
  1112. }
  1113. //-----------------------------------------------------------------------------
  1114. func (cs *ConsensusState) defaultSetProposal(proposal *types.Proposal) error {
  1115. // Already have one
  1116. // TODO: possibly catch double proposals
  1117. if cs.Proposal != nil {
  1118. return nil
  1119. }
  1120. // Does not apply
  1121. if proposal.Height != cs.Height || proposal.Round != cs.Round {
  1122. return nil
  1123. }
  1124. // We don't care about the proposal if we're already in RoundStepCommit.
  1125. if RoundStepCommit <= cs.Step {
  1126. return nil
  1127. }
  1128. // Verify POLRound, which must be -1 or between 0 and proposal.Round exclusive.
  1129. if proposal.POLRound != -1 &&
  1130. (proposal.POLRound < 0 || proposal.Round <= proposal.POLRound) {
  1131. return ErrInvalidProposalPOLRound
  1132. }
  1133. // Verify signature
  1134. if !cs.Validators.Proposer().PubKey.VerifyBytes(types.SignBytes(cs.state.ChainID, proposal), proposal.Signature) {
  1135. return ErrInvalidProposalSignature
  1136. }
  1137. cs.Proposal = proposal
  1138. cs.ProposalBlockParts = types.NewPartSetFromHeader(proposal.BlockPartsHeader)
  1139. return nil
  1140. }
  1141. // NOTE: block is not necessarily valid.
  1142. // Asynchronously triggers either enterPrevote (before we timeout of propose) or tryFinalizeCommit, once we have the full block.
  1143. func (cs *ConsensusState) addProposalBlockPart(height int, part *types.Part, verify bool) (added bool, err error) {
  1144. // Blocks might be reused, so round mismatch is OK
  1145. if cs.Height != height {
  1146. return false, nil
  1147. }
  1148. // We're not expecting a block part.
  1149. if cs.ProposalBlockParts == nil {
  1150. return false, nil // TODO: bad peer? Return error?
  1151. }
  1152. added, err = cs.ProposalBlockParts.AddPart(part, verify)
  1153. if err != nil {
  1154. return added, err
  1155. }
  1156. if added && cs.ProposalBlockParts.IsComplete() {
  1157. // Added and completed!
  1158. var n int
  1159. var err error
  1160. cs.ProposalBlock = wire.ReadBinary(&types.Block{}, cs.ProposalBlockParts.GetReader(), types.MaxBlockSize, &n, &err).(*types.Block)
  1161. // NOTE: it's possible to receive complete proposal blocks for future rounds without having the proposal
  1162. log.Info("Received complete proposal block", "height", cs.ProposalBlock.Height, "hash", cs.ProposalBlock.Hash())
  1163. if cs.Step == RoundStepPropose && cs.isProposalComplete() {
  1164. // Move onto the next step
  1165. cs.enterPrevote(height, cs.Round)
  1166. } else if cs.Step == RoundStepCommit {
  1167. // If we're waiting on the proposal block...
  1168. cs.tryFinalizeCommit(height)
  1169. }
  1170. return true, err
  1171. }
  1172. return added, nil
  1173. }
  1174. // Attempt to add the vote. if its a duplicate signature, dupeout the validator
  1175. func (cs *ConsensusState) tryAddVote(vote *types.Vote, peerKey string) error {
  1176. _, err := cs.addVote(vote, peerKey)
  1177. if err != nil {
  1178. // If the vote height is off, we'll just ignore it,
  1179. // But if it's a conflicting sig, broadcast evidence tx for slashing.
  1180. // If it's otherwise invalid, punish peer.
  1181. if err == ErrVoteHeightMismatch {
  1182. return err
  1183. } else if _, ok := err.(*types.ErrVoteConflictingVotes); ok {
  1184. if peerKey == "" {
  1185. log.Warn("Found conflicting vote from ourselves. Did you unsafe_reset a validator?", "height", vote.Height, "round", vote.Round, "type", vote.Type)
  1186. return err
  1187. }
  1188. log.Warn("Found conflicting vote. Publish evidence (TODO)")
  1189. /* TODO
  1190. evidenceTx := &types.DupeoutTx{
  1191. Address: address,
  1192. VoteA: *errDupe.VoteA,
  1193. VoteB: *errDupe.VoteB,
  1194. }
  1195. cs.mempool.BroadcastTx(struct{???}{evidenceTx}) // shouldn't need to check returned err
  1196. */
  1197. return err
  1198. } else {
  1199. // Probably an invalid signature. Bad peer.
  1200. log.Warn("Error attempting to add vote", "error", err)
  1201. return ErrAddingVote
  1202. }
  1203. }
  1204. return nil
  1205. }
  1206. //-----------------------------------------------------------------------------
  1207. func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool, err error) {
  1208. log.Debug("addVote", "voteHeight", vote.Height, "voteType", vote.Type, "csHeight", cs.Height)
  1209. // A precommit for the previous height?
  1210. if vote.Height+1 == cs.Height {
  1211. if !(cs.Step == RoundStepNewHeight && vote.Type == types.VoteTypePrecommit) {
  1212. // TODO: give the reason ..
  1213. // fmt.Errorf("tryAddVote: Wrong height, not a LastCommit straggler commit.")
  1214. return added, ErrVoteHeightMismatch
  1215. }
  1216. added, err = cs.LastCommit.AddVote(vote)
  1217. if added {
  1218. log.Info(Fmt("Added to lastPrecommits: %v", cs.LastCommit.StringShort()))
  1219. types.FireEventVote(cs.evsw, types.EventDataVote{vote})
  1220. }
  1221. return
  1222. }
  1223. // A prevote/precommit for this height?
  1224. if vote.Height == cs.Height {
  1225. height := cs.Height
  1226. added, err = cs.Votes.AddVote(vote, peerKey)
  1227. if added {
  1228. types.FireEventVote(cs.evsw, types.EventDataVote{vote})
  1229. switch vote.Type {
  1230. case types.VoteTypePrevote:
  1231. prevotes := cs.Votes.Prevotes(vote.Round)
  1232. log.Info("Added to prevote", "vote", vote, "prevotes", prevotes.StringShort())
  1233. // First, unlock if prevotes is a valid POL.
  1234. // >> lockRound < POLRound <= unlockOrChangeLockRound (see spec)
  1235. // NOTE: If (lockRound < POLRound) but !(POLRound <= unlockOrChangeLockRound),
  1236. // we'll still enterNewRound(H,vote.R) and enterPrecommit(H,vote.R) to process it
  1237. // there.
  1238. if (cs.LockedBlock != nil) && (cs.LockedRound < vote.Round) && (vote.Round <= cs.Round) {
  1239. blockID, ok := prevotes.TwoThirdsMajority()
  1240. if ok && !cs.LockedBlock.HashesTo(blockID.Hash) {
  1241. log.Notice("Unlocking because of POL.", "lockedRound", cs.LockedRound, "POLRound", vote.Round)
  1242. cs.LockedRound = 0
  1243. cs.LockedBlock = nil
  1244. cs.LockedBlockParts = nil
  1245. types.FireEventUnlock(cs.evsw, cs.RoundStateEvent())
  1246. }
  1247. }
  1248. if cs.Round <= vote.Round && prevotes.HasTwoThirdsAny() {
  1249. // Round-skip over to PrevoteWait or goto Precommit.
  1250. cs.enterNewRound(height, vote.Round) // if the vote is ahead of us
  1251. if prevotes.HasTwoThirdsMajority() {
  1252. cs.enterPrecommit(height, vote.Round)
  1253. } else {
  1254. cs.enterPrevote(height, vote.Round) // if the vote is ahead of us
  1255. cs.enterPrevoteWait(height, vote.Round)
  1256. }
  1257. } else if cs.Proposal != nil && 0 <= cs.Proposal.POLRound && cs.Proposal.POLRound == vote.Round {
  1258. // If the proposal is now complete, enter prevote of cs.Round.
  1259. if cs.isProposalComplete() {
  1260. cs.enterPrevote(height, cs.Round)
  1261. }
  1262. }
  1263. case types.VoteTypePrecommit:
  1264. precommits := cs.Votes.Precommits(vote.Round)
  1265. log.Info("Added to precommit", "vote", vote, "precommits", precommits.StringShort())
  1266. blockID, ok := precommits.TwoThirdsMajority()
  1267. if ok {
  1268. if len(blockID.Hash) == 0 {
  1269. cs.enterNewRound(height, vote.Round+1)
  1270. } else {
  1271. cs.enterNewRound(height, vote.Round)
  1272. cs.enterPrecommit(height, vote.Round)
  1273. cs.enterCommit(height, vote.Round)
  1274. }
  1275. } else if cs.Round <= vote.Round && precommits.HasTwoThirdsAny() {
  1276. cs.enterNewRound(height, vote.Round)
  1277. cs.enterPrecommit(height, vote.Round)
  1278. cs.enterPrecommitWait(height, vote.Round)
  1279. //}()
  1280. }
  1281. default:
  1282. PanicSanity(Fmt("Unexpected vote type %X", vote.Type)) // Should not happen.
  1283. }
  1284. }
  1285. // Either duplicate, or error upon cs.Votes.AddByIndex()
  1286. return
  1287. } else {
  1288. err = ErrVoteHeightMismatch
  1289. }
  1290. // Height mismatch, bad peer?
  1291. log.Info("Vote ignored and not added", "voteHeight", vote.Height, "csHeight", cs.Height, "err", err)
  1292. return
  1293. }
  1294. func (cs *ConsensusState) signVote(type_ byte, hash []byte, header types.PartSetHeader) (*types.Vote, error) {
  1295. // TODO: store our index in the cs so we don't have to do this every time
  1296. addr := cs.privValidator.GetAddress()
  1297. valIndex, _ := cs.Validators.GetByAddress(addr)
  1298. vote := &types.Vote{
  1299. ValidatorAddress: addr,
  1300. ValidatorIndex: valIndex,
  1301. Height: cs.Height,
  1302. Round: cs.Round,
  1303. Type: type_,
  1304. BlockID: types.BlockID{hash, header},
  1305. }
  1306. err := cs.privValidator.SignVote(cs.state.ChainID, vote)
  1307. return vote, err
  1308. }
  1309. // sign the vote and publish on internalMsgQueue
  1310. func (cs *ConsensusState) signAddVote(type_ byte, hash []byte, header types.PartSetHeader) *types.Vote {
  1311. if cs.privValidator == nil || !cs.Validators.HasAddress(cs.privValidator.GetAddress()) {
  1312. return nil
  1313. }
  1314. vote, err := cs.signVote(type_, hash, header)
  1315. if err == nil {
  1316. cs.sendInternalMessage(msgInfo{&VoteMessage{vote}, ""})
  1317. log.Info("Signed and pushed vote", "height", cs.Height, "round", cs.Round, "vote", vote, "error", err)
  1318. return vote
  1319. } else {
  1320. //if !cs.replayMode {
  1321. log.Warn("Error signing vote", "height", cs.Height, "round", cs.Round, "vote", vote, "error", err)
  1322. //}
  1323. return nil
  1324. }
  1325. }
  1326. //---------------------------------------------------------
  1327. func CompareHRS(h1, r1 int, s1 RoundStepType, h2, r2 int, s2 RoundStepType) int {
  1328. if h1 < h2 {
  1329. return -1
  1330. } else if h1 > h2 {
  1331. return 1
  1332. }
  1333. if r1 < r2 {
  1334. return -1
  1335. } else if r1 > r2 {
  1336. return 1
  1337. }
  1338. if s1 < s2 {
  1339. return -1
  1340. } else if s1 > s2 {
  1341. return 1
  1342. }
  1343. return 0
  1344. }