You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1517 lines
50 KiB

10 years ago
10 years ago
10 years ago
10 years ago
9 years ago
10 years ago
10 years ago
10 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
9 years ago
9 years ago
10 years ago
10 years ago
10 years ago
10 years ago
9 years ago
9 years ago
9 years ago
9 years ago
10 years ago
10 years ago
9 years ago
9 years ago
9 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
9 years ago
9 years ago
10 years ago
9 years ago
9 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
10 years ago
10 years ago
9 years ago
9 years ago
9 years ago
10 years ago
10 years ago
10 years ago
9 years ago
10 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
10 years ago
10 years ago
  1. package consensus
  2. import (
  3. "bytes"
  4. "errors"
  5. "fmt"
  6. "reflect"
  7. "sync"
  8. "time"
  9. . "github.com/tendermint/go-common"
  10. "github.com/tendermint/go-events"
  11. "github.com/tendermint/go-wire"
  12. bc "github.com/tendermint/tendermint/blockchain"
  13. mempl "github.com/tendermint/tendermint/mempool"
  14. "github.com/tendermint/tendermint/proxy"
  15. sm "github.com/tendermint/tendermint/state"
  16. "github.com/tendermint/tendermint/types"
  17. )
  18. //-----------------------------------------------------------------------------
  19. // Timeout Parameters
  20. // All in milliseconds
  21. type TimeoutParams struct {
  22. Propose0 int
  23. ProposeDelta int
  24. Prevote0 int
  25. PrevoteDelta int
  26. Precommit0 int
  27. PrecommitDelta int
  28. Commit0 int
  29. }
  30. // Wait this long for a proposal
  31. func (tp *TimeoutParams) Propose(round int) time.Duration {
  32. return time.Duration(tp.Propose0+tp.ProposeDelta*round) * time.Millisecond
  33. }
  34. // After receiving any +2/3 prevote, wait this long for stragglers
  35. func (tp *TimeoutParams) Prevote(round int) time.Duration {
  36. return time.Duration(tp.Prevote0+tp.PrevoteDelta*round) * time.Millisecond
  37. }
  38. // After receiving any +2/3 precommits, wait this long for stragglers
  39. func (tp *TimeoutParams) Precommit(round int) time.Duration {
  40. return time.Duration(tp.Precommit0+tp.PrecommitDelta*round) * time.Millisecond
  41. }
  42. // After receiving +2/3 precommits for a single block (a commit), wait this long for stragglers in the next height's RoundStepNewHeight
  43. func (tp *TimeoutParams) Commit(t time.Time) time.Time {
  44. return t.Add(time.Duration(tp.Commit0) * time.Millisecond)
  45. }
  46. // Initialize parameters from config file
  47. func InitTimeoutParamsFromConfig() *TimeoutParams {
  48. return &TimeoutParams{
  49. Propose0: config.GetInt("timeout_propose"),
  50. ProposeDelta: config.GetInt("timeout_propose_delta"),
  51. Prevote0: config.GetInt("timeout_prevote"),
  52. PrevoteDelta: config.GetInt("timeout_prevote_delta"),
  53. Precommit0: config.GetInt("timeout_precommit"),
  54. PrecommitDelta: config.GetInt("timeout_precommit_delta"),
  55. Commit0: config.GetInt("timeout_commit"),
  56. }
  57. }
  58. //-----------------------------------------------------------------------------
  59. // Errors
  60. var (
  61. ErrInvalidProposalSignature = errors.New("Error invalid proposal signature")
  62. ErrInvalidProposalPOLRound = errors.New("Error invalid proposal POL round")
  63. ErrAddingVote = errors.New("Error adding vote")
  64. ErrVoteHeightMismatch = errors.New("Error vote height mismatch")
  65. )
  66. //-----------------------------------------------------------------------------
  67. // RoundStepType enum type
  68. type RoundStepType uint8 // These must be numeric, ordered.
  69. const (
  70. RoundStepNewHeight = RoundStepType(0x01) // Wait til CommitTime + timeoutCommit
  71. RoundStepNewRound = RoundStepType(0x02) // Setup new round and go to RoundStepPropose
  72. RoundStepPropose = RoundStepType(0x03) // Did propose, gossip proposal
  73. RoundStepPrevote = RoundStepType(0x04) // Did prevote, gossip prevotes
  74. RoundStepPrevoteWait = RoundStepType(0x05) // Did receive any +2/3 prevotes, start timeout
  75. RoundStepPrecommit = RoundStepType(0x06) // Did precommit, gossip precommits
  76. RoundStepPrecommitWait = RoundStepType(0x07) // Did receive any +2/3 precommits, start timeout
  77. RoundStepCommit = RoundStepType(0x08) // Entered commit state machine
  78. // NOTE: RoundStepNewHeight acts as RoundStepCommitWait.
  79. )
  80. func (rs RoundStepType) String() string {
  81. switch rs {
  82. case RoundStepNewHeight:
  83. return "RoundStepNewHeight"
  84. case RoundStepNewRound:
  85. return "RoundStepNewRound"
  86. case RoundStepPropose:
  87. return "RoundStepPropose"
  88. case RoundStepPrevote:
  89. return "RoundStepPrevote"
  90. case RoundStepPrevoteWait:
  91. return "RoundStepPrevoteWait"
  92. case RoundStepPrecommit:
  93. return "RoundStepPrecommit"
  94. case RoundStepPrecommitWait:
  95. return "RoundStepPrecommitWait"
  96. case RoundStepCommit:
  97. return "RoundStepCommit"
  98. default:
  99. return "RoundStepUnknown" // Cannot panic.
  100. }
  101. }
  102. //-----------------------------------------------------------------------------
  103. // Immutable when returned from ConsensusState.GetRoundState()
  104. type RoundState struct {
  105. Height int // Height we are working on
  106. Round int
  107. Step RoundStepType
  108. StartTime time.Time
  109. CommitTime time.Time // Subjective time when +2/3 precommits for Block at Round were found
  110. Validators *types.ValidatorSet
  111. Proposal *types.Proposal
  112. ProposalBlock *types.Block
  113. ProposalBlockParts *types.PartSet
  114. LockedRound int
  115. LockedBlock *types.Block
  116. LockedBlockParts *types.PartSet
  117. Votes *HeightVoteSet
  118. CommitRound int //
  119. LastCommit *types.VoteSet // Last precommits at Height-1
  120. LastValidators *types.ValidatorSet
  121. }
  122. func (rs *RoundState) RoundStateEvent() types.EventDataRoundState {
  123. edrs := types.EventDataRoundState{
  124. Height: rs.Height,
  125. Round: rs.Round,
  126. Step: rs.Step.String(),
  127. RoundState: rs,
  128. }
  129. return edrs
  130. }
  131. func (rs *RoundState) String() string {
  132. return rs.StringIndented("")
  133. }
  134. func (rs *RoundState) StringIndented(indent string) string {
  135. return fmt.Sprintf(`RoundState{
  136. %s H:%v R:%v S:%v
  137. %s StartTime: %v
  138. %s CommitTime: %v
  139. %s Validators: %v
  140. %s Proposal: %v
  141. %s ProposalBlock: %v %v
  142. %s LockedRound: %v
  143. %s LockedBlock: %v %v
  144. %s Votes: %v
  145. %s LastCommit: %v
  146. %s LastValidators: %v
  147. %s}`,
  148. indent, rs.Height, rs.Round, rs.Step,
  149. indent, rs.StartTime,
  150. indent, rs.CommitTime,
  151. indent, rs.Validators.StringIndented(indent+" "),
  152. indent, rs.Proposal,
  153. indent, rs.ProposalBlockParts.StringShort(), rs.ProposalBlock.StringShort(),
  154. indent, rs.LockedRound,
  155. indent, rs.LockedBlockParts.StringShort(), rs.LockedBlock.StringShort(),
  156. indent, rs.Votes.StringIndented(indent+" "),
  157. indent, rs.LastCommit.StringShort(),
  158. indent, rs.LastValidators.StringIndented(indent+" "),
  159. indent)
  160. }
  161. func (rs *RoundState) StringShort() string {
  162. return fmt.Sprintf(`RoundState{H:%v R:%v S:%v ST:%v}`,
  163. rs.Height, rs.Round, rs.Step, rs.StartTime)
  164. }
  165. //-----------------------------------------------------------------------------
  166. var (
  167. msgQueueSize = 1000
  168. tickTockBufferSize = 10
  169. )
  170. // msgs from the reactor which may update the state
  171. type msgInfo struct {
  172. Msg ConsensusMessage `json:"msg"`
  173. PeerKey string `json:"peer_key"`
  174. }
  175. // internally generated messages which may update the state
  176. type timeoutInfo struct {
  177. Duration time.Duration `json:"duration"`
  178. Height int `json:"height"`
  179. Round int `json:"round"`
  180. Step RoundStepType `json:"step"`
  181. }
  182. func (ti *timeoutInfo) String() string {
  183. return fmt.Sprintf("%v ; %d/%d %v", ti.Duration, ti.Height, ti.Round, ti.Step)
  184. }
  185. // Tracks consensus state across block heights and rounds.
  186. type ConsensusState struct {
  187. QuitService
  188. proxyAppConn proxy.AppConn
  189. blockStore *bc.BlockStore
  190. mempool *mempl.Mempool
  191. privValidator *types.PrivValidator
  192. mtx sync.Mutex
  193. RoundState
  194. state *sm.State // State until height-1.
  195. peerMsgQueue chan msgInfo // serializes msgs affecting state (proposals, block parts, votes)
  196. internalMsgQueue chan msgInfo // like peerMsgQueue but for our own proposals, parts, votes
  197. timeoutTicker *time.Ticker // ticker for timeouts
  198. tickChan chan timeoutInfo // start the timeoutTicker in the timeoutRoutine
  199. tockChan chan timeoutInfo // timeouts are relayed on tockChan to the receiveRoutine
  200. timeoutParams *TimeoutParams // parameters and functions for timeout intervals
  201. evsw *events.EventSwitch
  202. wal *WAL
  203. nSteps int // used for testing to limit the number of transitions the state makes
  204. }
  205. func NewConsensusState(state *sm.State, proxyAppConn proxy.AppConn, blockStore *bc.BlockStore, mempool *mempl.Mempool) *ConsensusState {
  206. cs := &ConsensusState{
  207. proxyAppConn: proxyAppConn,
  208. blockStore: blockStore,
  209. mempool: mempool,
  210. peerMsgQueue: make(chan msgInfo, msgQueueSize),
  211. internalMsgQueue: make(chan msgInfo, msgQueueSize),
  212. timeoutTicker: new(time.Ticker),
  213. tickChan: make(chan timeoutInfo, tickTockBufferSize),
  214. tockChan: make(chan timeoutInfo, tickTockBufferSize),
  215. timeoutParams: InitTimeoutParamsFromConfig(),
  216. }
  217. cs.updateToState(state)
  218. // Don't call scheduleRound0 yet.
  219. // We do that upon Start().
  220. cs.reconstructLastCommit(state)
  221. cs.QuitService = *NewQuitService(log, "ConsensusState", cs)
  222. return cs
  223. }
  224. //----------------------------------------
  225. // Public interface
  226. // implements events.Eventable
  227. func (cs *ConsensusState) SetEventSwitch(evsw *events.EventSwitch) {
  228. cs.evsw = evsw
  229. }
  230. func (cs *ConsensusState) String() string {
  231. return Fmt("ConsensusState(H:%v R:%v S:%v", cs.Height, cs.Round, cs.Step)
  232. }
  233. func (cs *ConsensusState) GetState() *sm.State {
  234. cs.mtx.Lock()
  235. defer cs.mtx.Unlock()
  236. return cs.state.Copy()
  237. }
  238. func (cs *ConsensusState) GetRoundState() *RoundState {
  239. cs.mtx.Lock()
  240. defer cs.mtx.Unlock()
  241. return cs.getRoundState()
  242. }
  243. func (cs *ConsensusState) getRoundState() *RoundState {
  244. rs := cs.RoundState // copy
  245. return &rs
  246. }
  247. func (cs *ConsensusState) SetPrivValidator(priv *types.PrivValidator) {
  248. cs.mtx.Lock()
  249. defer cs.mtx.Unlock()
  250. cs.privValidator = priv
  251. }
  252. func (cs *ConsensusState) OnStart() error {
  253. cs.QuitService.OnStart()
  254. // start timeout and receive routines
  255. cs.startRoutines(0)
  256. // we may have lost some votes if the process crashed
  257. // reload from consensus log to catchup
  258. if err := cs.catchupReplay(cs.Height); err != nil {
  259. log.Error("Error on catchup replay", "error", err.Error())
  260. // let's go for it anyways, maybe we're fine
  261. }
  262. // schedule the first round!
  263. cs.scheduleRound0(cs.Height)
  264. return nil
  265. }
  266. // timeoutRoutine: receive requests for timeouts on tickChan and fire timeouts on tockChan
  267. // receiveRoutine: serializes processing of proposoals, block parts, votes; coordinates state transitions
  268. func (cs *ConsensusState) startRoutines(maxSteps int) {
  269. go cs.timeoutRoutine()
  270. go cs.receiveRoutine(maxSteps)
  271. }
  272. func (cs *ConsensusState) OnStop() {
  273. cs.QuitService.OnStop()
  274. if cs.wal != nil && cs.IsRunning() {
  275. cs.wal.Wait()
  276. }
  277. }
  278. // Open file to log all consensus messages and timeouts for deterministic accountability
  279. func (cs *ConsensusState) OpenWAL(file string) (err error) {
  280. cs.mtx.Lock()
  281. defer cs.mtx.Unlock()
  282. wal, err := NewWAL(file, config.GetBool("cswal_light"))
  283. if err != nil {
  284. return err
  285. }
  286. cs.wal = wal
  287. return nil
  288. }
  289. //------------------------------------------------------------
  290. // Public interface for passing messages into the consensus state,
  291. // possibly causing a state transition
  292. // TODO: should these return anything or let callers just use events?
  293. // May block on send if queue is full.
  294. func (cs *ConsensusState) AddVote(valIndex int, vote *types.Vote, peerKey string) (added bool, address []byte, err error) {
  295. if peerKey == "" {
  296. cs.internalMsgQueue <- msgInfo{&VoteMessage{valIndex, vote}, ""}
  297. } else {
  298. cs.peerMsgQueue <- msgInfo{&VoteMessage{valIndex, vote}, peerKey}
  299. }
  300. // TODO: wait for event?!
  301. return false, nil, nil
  302. }
  303. // May block on send if queue is full.
  304. func (cs *ConsensusState) SetProposal(proposal *types.Proposal, peerKey string) error {
  305. if peerKey == "" {
  306. cs.internalMsgQueue <- msgInfo{&ProposalMessage{proposal}, ""}
  307. } else {
  308. cs.peerMsgQueue <- msgInfo{&ProposalMessage{proposal}, peerKey}
  309. }
  310. // TODO: wait for event?!
  311. return nil
  312. }
  313. // May block on send if queue is full.
  314. func (cs *ConsensusState) AddProposalBlockPart(height, round int, part *types.Part, peerKey string) error {
  315. if peerKey == "" {
  316. cs.internalMsgQueue <- msgInfo{&BlockPartMessage{height, round, part}, ""}
  317. } else {
  318. cs.peerMsgQueue <- msgInfo{&BlockPartMessage{height, round, part}, peerKey}
  319. }
  320. // TODO: wait for event?!
  321. return nil
  322. }
  323. // May block on send if queue is full.
  324. func (cs *ConsensusState) SetProposalAndBlock(proposal *types.Proposal, block *types.Block, parts *types.PartSet, peerKey string) error {
  325. cs.SetProposal(proposal, peerKey)
  326. for i := 0; i < parts.Total(); i++ {
  327. part := parts.GetPart(i)
  328. cs.AddProposalBlockPart(proposal.Height, proposal.Round, part, peerKey)
  329. }
  330. return nil // TODO errors
  331. }
  332. //------------------------------------------------------------
  333. // internal functions for managing the state
  334. func (cs *ConsensusState) updateHeight(height int) {
  335. cs.Height = height
  336. }
  337. func (cs *ConsensusState) updateRoundStep(round int, step RoundStepType) {
  338. cs.Round = round
  339. cs.Step = step
  340. }
  341. // enterNewRound(height, 0) at cs.StartTime.
  342. func (cs *ConsensusState) scheduleRound0(height int) {
  343. //log.Info("scheduleRound0", "now", time.Now(), "startTime", cs.StartTime)
  344. sleepDuration := cs.StartTime.Sub(time.Now())
  345. if sleepDuration < time.Duration(0) {
  346. sleepDuration = time.Duration(0)
  347. }
  348. cs.scheduleTimeout(sleepDuration, height, 0, RoundStepNewHeight)
  349. }
  350. // Attempt to schedule a timeout by sending timeoutInfo on the tickChan.
  351. // The timeoutRoutine is alwaya available to read from tickChan (it won't block).
  352. // The scheduling may fail if the timeoutRoutine has already scheduled a timeout for a later height/round/step.
  353. func (cs *ConsensusState) scheduleTimeout(duration time.Duration, height, round int, step RoundStepType) {
  354. cs.tickChan <- timeoutInfo{duration, height, round, step}
  355. }
  356. // send a msg into the receiveRoutine regarding our own proposal, block part, or vote
  357. func (cs *ConsensusState) sendInternalMessage(mi msgInfo) {
  358. select {
  359. case cs.internalMsgQueue <- mi:
  360. default:
  361. // NOTE: using the go-routine means our votes can
  362. // be processed out of order.
  363. // TODO: use CList here for strict determinism and
  364. // attempt push to internalMsgQueue in receiveRoutine
  365. log.Debug("Internal msg queue is full. Using a go-routine")
  366. go func() { cs.internalMsgQueue <- mi }()
  367. }
  368. }
  369. // Reconstruct LastCommit from SeenValidation, which we saved along with the block,
  370. // (which happens even before saving the state)
  371. func (cs *ConsensusState) reconstructLastCommit(state *sm.State) {
  372. if state.LastBlockHeight == 0 {
  373. return
  374. }
  375. seenValidation := cs.blockStore.LoadSeenValidation(state.LastBlockHeight)
  376. lastPrecommits := types.NewVoteSet(state.LastBlockHeight, seenValidation.Round(), types.VoteTypePrecommit, state.LastValidators)
  377. for idx, precommit := range seenValidation.Precommits {
  378. if precommit == nil {
  379. continue
  380. }
  381. added, _, err := lastPrecommits.AddByIndex(idx, precommit)
  382. if !added || err != nil {
  383. PanicCrisis(Fmt("Failed to reconstruct LastCommit: %v", err))
  384. }
  385. }
  386. if !lastPrecommits.HasTwoThirdsMajority() {
  387. PanicSanity("Failed to reconstruct LastCommit: Does not have +2/3 maj")
  388. }
  389. cs.LastCommit = lastPrecommits
  390. }
  391. // Updates ConsensusState and increments height to match that of state.
  392. // The round becomes 0 and cs.Step becomes RoundStepNewHeight.
  393. func (cs *ConsensusState) updateToState(state *sm.State) {
  394. if cs.CommitRound > -1 && 0 < cs.Height && cs.Height != state.LastBlockHeight {
  395. PanicSanity(Fmt("updateToState() expected state height of %v but found %v",
  396. cs.Height, state.LastBlockHeight))
  397. }
  398. if cs.state != nil && cs.state.LastBlockHeight+1 != cs.Height {
  399. // This might happen when someone else is mutating cs.state.
  400. // Someone forgot to pass in state.Copy() somewhere?!
  401. PanicSanity(Fmt("Inconsistent cs.state.LastBlockHeight+1 %v vs cs.Height %v",
  402. cs.state.LastBlockHeight+1, cs.Height))
  403. }
  404. // If state isn't further out than cs.state, just ignore.
  405. // This happens when SwitchToConsensus() is called in the reactor.
  406. // We don't want to reset e.g. the Votes.
  407. if cs.state != nil && (state.LastBlockHeight <= cs.state.LastBlockHeight) {
  408. log.Notice("Ignoring updateToState()", "newHeight", state.LastBlockHeight+1, "oldHeight", cs.state.LastBlockHeight+1)
  409. return
  410. }
  411. // Reset fields based on state.
  412. validators := state.Validators
  413. height := state.LastBlockHeight + 1 // Next desired block height
  414. lastPrecommits := (*types.VoteSet)(nil)
  415. if cs.CommitRound > -1 && cs.Votes != nil {
  416. if !cs.Votes.Precommits(cs.CommitRound).HasTwoThirdsMajority() {
  417. PanicSanity("updateToState(state) called but last Precommit round didn't have +2/3")
  418. }
  419. lastPrecommits = cs.Votes.Precommits(cs.CommitRound)
  420. }
  421. // RoundState fields
  422. cs.updateHeight(height)
  423. cs.updateRoundStep(0, RoundStepNewHeight)
  424. if cs.CommitTime.IsZero() {
  425. // "Now" makes it easier to sync up dev nodes.
  426. // We add timeoutCommit to allow transactions
  427. // to be gathered for the first block.
  428. // And alternative solution that relies on clocks:
  429. // cs.StartTime = state.LastBlockTime.Add(timeoutCommit)
  430. cs.StartTime = cs.timeoutParams.Commit(time.Now())
  431. } else {
  432. cs.StartTime = cs.timeoutParams.Commit(cs.CommitTime)
  433. }
  434. cs.CommitTime = time.Time{}
  435. cs.Validators = validators
  436. cs.Proposal = nil
  437. cs.ProposalBlock = nil
  438. cs.ProposalBlockParts = nil
  439. cs.LockedRound = 0
  440. cs.LockedBlock = nil
  441. cs.LockedBlockParts = nil
  442. cs.Votes = NewHeightVoteSet(height, validators)
  443. cs.CommitRound = -1
  444. cs.LastCommit = lastPrecommits
  445. cs.LastValidators = state.LastValidators
  446. cs.state = state
  447. // Finally, broadcast RoundState
  448. cs.newStep()
  449. }
  450. func (cs *ConsensusState) newStep() {
  451. rs := cs.RoundStateEvent()
  452. cs.wal.Save(rs)
  453. cs.nSteps += 1
  454. // newStep is called by updateToStep in NewConsensusState before the evsw is set!
  455. if cs.evsw != nil {
  456. cs.evsw.FireEvent(types.EventStringNewRoundStep(), rs)
  457. }
  458. }
  459. //-----------------------------------------
  460. // the main go routines
  461. // the state machine sends on tickChan to start a new timer.
  462. // timers are interupted and replaced by new ticks from later steps
  463. // timeouts of 0 on the tickChan will be immediately relayed to the tockChan
  464. func (cs *ConsensusState) timeoutRoutine() {
  465. log.Debug("Starting timeout routine")
  466. var ti timeoutInfo
  467. for {
  468. select {
  469. case newti := <-cs.tickChan:
  470. log.Debug("Received tick", "old_ti", ti, "new_ti", newti)
  471. // ignore tickers for old height/round/step
  472. if newti.Height < ti.Height {
  473. continue
  474. } else if newti.Height == ti.Height {
  475. if newti.Round < ti.Round {
  476. continue
  477. } else if newti.Round == ti.Round {
  478. if ti.Step > 0 && newti.Step <= ti.Step {
  479. continue
  480. }
  481. }
  482. }
  483. ti = newti
  484. // if the newti has duration == 0, we relay to the tockChan immediately (no timeout)
  485. if ti.Duration == time.Duration(0) {
  486. go func(t timeoutInfo) { cs.tockChan <- t }(ti)
  487. continue
  488. }
  489. log.Debug("Scheduling timeout", "dur", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step)
  490. cs.timeoutTicker.Stop()
  491. cs.timeoutTicker = time.NewTicker(ti.Duration)
  492. case <-cs.timeoutTicker.C:
  493. log.Info("Timed out", "dur", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step)
  494. cs.timeoutTicker.Stop()
  495. // go routine here gaurantees timeoutRoutine doesn't block.
  496. // Determinism comes from playback in the receiveRoutine.
  497. // We can eliminate it by merging the timeoutRoutine into receiveRoutine
  498. // and managing the timeouts ourselves with a millisecond ticker
  499. go func(t timeoutInfo) { cs.tockChan <- t }(ti)
  500. case <-cs.Quit:
  501. return
  502. }
  503. }
  504. }
  505. // a nice idea but probably more trouble than its worth
  506. func (cs *ConsensusState) stopTimer() {
  507. cs.timeoutTicker.Stop()
  508. }
  509. // receiveRoutine handles messages which may cause state transitions.
  510. // it's argument (n) is the number of messages to process before exiting - use 0 to run forever
  511. // It keeps the RoundState and is the only thing that updates it.
  512. // Updates (state transitions) happen on timeouts, complete proposals, and 2/3 majorities
  513. func (cs *ConsensusState) receiveRoutine(maxSteps int) {
  514. for {
  515. if maxSteps > 0 {
  516. if cs.nSteps >= maxSteps {
  517. log.Warn("reached max steps. exiting receive routine")
  518. cs.nSteps = 0
  519. return
  520. }
  521. }
  522. rs := cs.RoundState
  523. var mi msgInfo
  524. select {
  525. case mi = <-cs.peerMsgQueue:
  526. cs.wal.Save(mi)
  527. // handles proposals, block parts, votes
  528. // may generate internal events (votes, complete proposals, 2/3 majorities)
  529. cs.handleMsg(mi, rs)
  530. case mi = <-cs.internalMsgQueue:
  531. cs.wal.Save(mi)
  532. // handles proposals, block parts, votes
  533. cs.handleMsg(mi, rs)
  534. case ti := <-cs.tockChan:
  535. cs.wal.Save(ti)
  536. // if the timeout is relevant to the rs
  537. // go to the next step
  538. cs.handleTimeout(ti, rs)
  539. case <-cs.Quit:
  540. // drain the internalMsgQueue in case we eg. signed a proposal but it didn't hit the wal
  541. FLUSH:
  542. for {
  543. select {
  544. case mi = <-cs.internalMsgQueue:
  545. cs.wal.Save(mi)
  546. cs.handleMsg(mi, rs)
  547. default:
  548. break FLUSH
  549. }
  550. }
  551. // close wal now that we're done writing to it
  552. if cs.wal != nil {
  553. cs.wal.Close()
  554. }
  555. return
  556. }
  557. }
  558. }
  559. // state transitions on complete-proposal, 2/3-any, 2/3-one
  560. func (cs *ConsensusState) handleMsg(mi msgInfo, rs RoundState) {
  561. cs.mtx.Lock()
  562. defer cs.mtx.Unlock()
  563. var err error
  564. msg, peerKey := mi.Msg, mi.PeerKey
  565. switch msg := msg.(type) {
  566. case *ProposalMessage:
  567. // will not cause transition.
  568. // once proposal is set, we can receive block parts
  569. err = cs.setProposal(msg.Proposal)
  570. case *BlockPartMessage:
  571. // if the proposal is complete, we'll enterPrevote or tryFinalizeCommit
  572. _, err = cs.addProposalBlockPart(msg.Height, msg.Part)
  573. if err != nil && msg.Round != cs.Round {
  574. err = nil
  575. }
  576. case *VoteMessage:
  577. // attempt to add the vote and dupeout the validator if its a duplicate signature
  578. // if the vote gives us a 2/3-any or 2/3-one, we transition
  579. err := cs.tryAddVote(msg.ValidatorIndex, msg.Vote, peerKey)
  580. if err == ErrAddingVote {
  581. // TODO: punish peer
  582. }
  583. // NOTE: the vote is broadcast to peers by the reactor listening
  584. // for vote events
  585. // TODO: If rs.Height == vote.Height && rs.Round < vote.Round,
  586. // the peer is sending us CatchupCommit precommits.
  587. // We could make note of this and help filter in broadcastHasVoteMessage().
  588. default:
  589. log.Warn("Unknown msg type", reflect.TypeOf(msg))
  590. }
  591. if err != nil {
  592. log.Error("Error with msg", "type", reflect.TypeOf(msg), "peer", peerKey, "error", err, "msg", msg)
  593. }
  594. }
  595. func (cs *ConsensusState) handleTimeout(ti timeoutInfo, rs RoundState) {
  596. log.Debug("Received tock", "timeout", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step)
  597. // timeouts must be for current height, round, step
  598. if ti.Height != rs.Height || ti.Round < rs.Round || (ti.Round == rs.Round && ti.Step < rs.Step) {
  599. log.Debug("Ignoring tock because we're ahead", "height", rs.Height, "round", rs.Round, "step", rs.Step)
  600. return
  601. }
  602. // the timeout will now cause a state transition
  603. cs.mtx.Lock()
  604. defer cs.mtx.Unlock()
  605. switch ti.Step {
  606. case RoundStepNewHeight:
  607. // NewRound event fired from enterNewRound.
  608. // XXX: should we fire timeout here?
  609. cs.enterNewRound(ti.Height, 0)
  610. case RoundStepPropose:
  611. cs.evsw.FireEvent(types.EventStringTimeoutPropose(), cs.RoundStateEvent())
  612. cs.enterPrevote(ti.Height, ti.Round)
  613. case RoundStepPrevoteWait:
  614. cs.evsw.FireEvent(types.EventStringTimeoutWait(), cs.RoundStateEvent())
  615. cs.enterPrecommit(ti.Height, ti.Round)
  616. case RoundStepPrecommitWait:
  617. cs.evsw.FireEvent(types.EventStringTimeoutWait(), cs.RoundStateEvent())
  618. cs.enterNewRound(ti.Height, ti.Round+1)
  619. default:
  620. panic(Fmt("Invalid timeout step: %v", ti.Step))
  621. }
  622. }
  623. //-----------------------------------------------------------------------------
  624. // State functions
  625. // Used internally by handleTimeout and handleMsg to make state transitions
  626. // Enter: +2/3 precommits for nil at (height,round-1)
  627. // Enter: `timeoutPrecommits` after any +2/3 precommits from (height,round-1)
  628. // Enter: `startTime = commitTime+timeoutCommit` from NewHeight(height)
  629. // NOTE: cs.StartTime was already set for height.
  630. func (cs *ConsensusState) enterNewRound(height int, round int) {
  631. if cs.Height != height || round < cs.Round || (cs.Round == round && cs.Step != RoundStepNewHeight) {
  632. log.Debug(Fmt("enterNewRound(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  633. return
  634. }
  635. if now := time.Now(); cs.StartTime.After(now) {
  636. log.Warn("Need to set a buffer and log.Warn() here for sanity.", "startTime", cs.StartTime, "now", now)
  637. }
  638. // cs.stopTimer()
  639. log.Notice(Fmt("enterNewRound(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  640. // Increment validators if necessary
  641. validators := cs.Validators
  642. if cs.Round < round {
  643. validators = validators.Copy()
  644. validators.IncrementAccum(round - cs.Round)
  645. }
  646. // Setup new round
  647. // we don't fire newStep for this step,
  648. // but we fire an event, so update the round step first
  649. cs.updateRoundStep(round, RoundStepNewRound)
  650. cs.Validators = validators
  651. if round == 0 {
  652. // We've already reset these upon new height,
  653. // and meanwhile we might have received a proposal
  654. // for round 0.
  655. } else {
  656. cs.Proposal = nil
  657. cs.ProposalBlock = nil
  658. cs.ProposalBlockParts = nil
  659. }
  660. cs.Votes.SetRound(round + 1) // also track next round (round+1) to allow round-skipping
  661. cs.evsw.FireEvent(types.EventStringNewRound(), cs.RoundStateEvent())
  662. // Immediately go to enterPropose.
  663. cs.enterPropose(height, round)
  664. }
  665. // Enter: from NewRound(height,round).
  666. func (cs *ConsensusState) enterPropose(height int, round int) {
  667. if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPropose <= cs.Step) {
  668. log.Debug(Fmt("enterPropose(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  669. return
  670. }
  671. log.Info(Fmt("enterPropose(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  672. defer func() {
  673. // Done enterPropose:
  674. cs.updateRoundStep(round, RoundStepPropose)
  675. cs.newStep()
  676. // If we have the whole proposal + POL, then goto Prevote now.
  677. // else, we'll enterPrevote when the rest of the proposal is received (in AddProposalBlockPart),
  678. // or else after timeoutPropose
  679. if cs.isProposalComplete() {
  680. cs.enterPrevote(height, cs.Round)
  681. }
  682. }()
  683. // If we don't get the proposal and all block parts quick enough, enterPrevote
  684. cs.scheduleTimeout(cs.timeoutParams.Propose(round), height, round, RoundStepPropose)
  685. // Nothing more to do if we're not a validator
  686. if cs.privValidator == nil {
  687. return
  688. }
  689. if !bytes.Equal(cs.Validators.Proposer().Address, cs.privValidator.Address) {
  690. log.Info("enterPropose: Not our turn to propose", "proposer", cs.Validators.Proposer().Address, "privValidator", cs.privValidator)
  691. } else {
  692. log.Info("enterPropose: Our turn to propose", "proposer", cs.Validators.Proposer().Address, "privValidator", cs.privValidator)
  693. cs.decideProposal(height, round)
  694. }
  695. }
  696. func (cs *ConsensusState) decideProposal(height, round int) {
  697. var block *types.Block
  698. var blockParts *types.PartSet
  699. // Decide on block
  700. if cs.LockedBlock != nil {
  701. // If we're locked onto a block, just choose that.
  702. block, blockParts = cs.LockedBlock, cs.LockedBlockParts
  703. } else {
  704. // Create a new proposal block from state/txs from the mempool.
  705. block, blockParts = cs.createProposalBlock()
  706. if block == nil { // on error
  707. return
  708. }
  709. }
  710. // Make proposal
  711. proposal := types.NewProposal(height, round, blockParts.Header(), cs.Votes.POLRound())
  712. err := cs.privValidator.SignProposal(cs.state.ChainID, proposal)
  713. if err == nil {
  714. // Set fields
  715. /* fields set by setProposal and addBlockPart
  716. cs.Proposal = proposal
  717. cs.ProposalBlock = block
  718. cs.ProposalBlockParts = blockParts
  719. */
  720. // send proposal and block parts on internal msg queue
  721. cs.sendInternalMessage(msgInfo{&ProposalMessage{proposal}, ""})
  722. for i := 0; i < blockParts.Total(); i++ {
  723. part := blockParts.GetPart(i)
  724. cs.sendInternalMessage(msgInfo{&BlockPartMessage{cs.Height, cs.Round, part}, ""})
  725. }
  726. log.Info("Signed and sent proposal", "height", height, "round", round, "proposal", proposal)
  727. log.Debug(Fmt("Signed and sent proposal block: %v", block))
  728. } else {
  729. log.Warn("enterPropose: Error signing proposal", "height", height, "round", round, "error", err)
  730. }
  731. }
  732. // Returns true if the proposal block is complete &&
  733. // (if POLRound was proposed, we have +2/3 prevotes from there).
  734. func (cs *ConsensusState) isProposalComplete() bool {
  735. if cs.Proposal == nil || cs.ProposalBlock == nil {
  736. return false
  737. }
  738. // we have the proposal. if there's a POLRound,
  739. // make sure we have the prevotes from it too
  740. if cs.Proposal.POLRound < 0 {
  741. return true
  742. } else {
  743. // if this is false the proposer is lying or we haven't received the POL yet
  744. return cs.Votes.Prevotes(cs.Proposal.POLRound).HasTwoThirdsMajority()
  745. }
  746. }
  747. // Create the next block to propose and return it.
  748. // Returns nil block upon error.
  749. // NOTE: keep it side-effect free for clarity.
  750. func (cs *ConsensusState) createProposalBlock() (block *types.Block, blockParts *types.PartSet) {
  751. var validation *types.Validation
  752. if cs.Height == 1 {
  753. // We're creating a proposal for the first block.
  754. // The validation is empty, but not nil.
  755. validation = &types.Validation{}
  756. } else if cs.LastCommit.HasTwoThirdsMajority() {
  757. // Make the validation from LastCommit
  758. validation = cs.LastCommit.MakeValidation()
  759. } else {
  760. // This shouldn't happen.
  761. log.Error("enterPropose: Cannot propose anything: No validation for the previous block.")
  762. return
  763. }
  764. maxBlockSize := config.GetInt("block_size")
  765. // Mempool validated transactions
  766. // if block_size < 0, no txs will be included
  767. var txs []types.Tx
  768. if maxBlockSize >= 0 {
  769. txs = cs.mempool.Reap()
  770. }
  771. // Cap the number of txs in a block
  772. if maxBlockSize > 0 && maxBlockSize < len(txs) {
  773. txs = txs[:maxBlockSize]
  774. }
  775. block = &types.Block{
  776. Header: &types.Header{
  777. ChainID: cs.state.ChainID,
  778. Height: cs.Height,
  779. Time: time.Now(),
  780. Fees: 0, // TODO fees
  781. NumTxs: len(txs),
  782. LastBlockHash: cs.state.LastBlockHash,
  783. LastBlockParts: cs.state.LastBlockParts,
  784. ValidatorsHash: cs.state.Validators.Hash(),
  785. AppHash: cs.state.AppHash, // state merkle root of txs from the previous block.
  786. },
  787. LastValidation: validation,
  788. Data: &types.Data{
  789. Txs: txs,
  790. },
  791. }
  792. block.FillHeader()
  793. blockParts = block.MakePartSet()
  794. return block, blockParts
  795. }
  796. // Enter: `timeoutPropose` after entering Propose.
  797. // Enter: proposal block and POL is ready.
  798. // Enter: any +2/3 prevotes for future round.
  799. // Prevote for LockedBlock if we're locked, or ProposalBlock if valid.
  800. // Otherwise vote nil.
  801. func (cs *ConsensusState) enterPrevote(height int, round int) {
  802. if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPrevote <= cs.Step) {
  803. log.Debug(Fmt("enterPrevote(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  804. return
  805. }
  806. defer func() {
  807. // Done enterPrevote:
  808. cs.updateRoundStep(round, RoundStepPrevote)
  809. cs.newStep()
  810. }()
  811. // fire event for how we got here
  812. if cs.isProposalComplete() {
  813. cs.evsw.FireEvent(types.EventStringCompleteProposal(), cs.RoundStateEvent())
  814. } else {
  815. // we received +2/3 prevotes for a future round
  816. // TODO: catchup event?
  817. }
  818. // cs.stopTimer()
  819. log.Info(Fmt("enterPrevote(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  820. // Sign and broadcast vote as necessary
  821. cs.doPrevote(height, round)
  822. // Once `addVote` hits any +2/3 prevotes, we will go to PrevoteWait
  823. // (so we have more time to try and collect +2/3 prevotes for a single block)
  824. }
  825. func (cs *ConsensusState) doPrevote(height int, round int) {
  826. // If a block is locked, prevote that.
  827. if cs.LockedBlock != nil {
  828. log.Info("enterPrevote: Block was locked")
  829. cs.signAddVote(types.VoteTypePrevote, cs.LockedBlock.Hash(), cs.LockedBlockParts.Header())
  830. return
  831. }
  832. // If ProposalBlock is nil, prevote nil.
  833. if cs.ProposalBlock == nil {
  834. log.Warn("enterPrevote: ProposalBlock is nil")
  835. cs.signAddVote(types.VoteTypePrevote, nil, types.PartSetHeader{})
  836. return
  837. }
  838. // Valdiate proposal block
  839. err := cs.state.ValidateBlock(cs.ProposalBlock)
  840. if err != nil {
  841. // ProposalBlock is invalid, prevote nil.
  842. log.Warn("enterPrevote: ProposalBlock is invalid", "error", err)
  843. cs.signAddVote(types.VoteTypePrevote, nil, types.PartSetHeader{})
  844. return
  845. }
  846. // Prevote cs.ProposalBlock
  847. // NOTE: the proposal signature is validated when it is received,
  848. // and the proposal block parts are validated as they are received (against the merkle hash in the proposal)
  849. cs.signAddVote(types.VoteTypePrevote, cs.ProposalBlock.Hash(), cs.ProposalBlockParts.Header())
  850. return
  851. }
  852. // Enter: any +2/3 prevotes at next round.
  853. func (cs *ConsensusState) enterPrevoteWait(height int, round int) {
  854. if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPrevoteWait <= cs.Step) {
  855. log.Debug(Fmt("enterPrevoteWait(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  856. return
  857. }
  858. if !cs.Votes.Prevotes(round).HasTwoThirdsAny() {
  859. PanicSanity(Fmt("enterPrevoteWait(%v/%v), but Prevotes does not have any +2/3 votes", height, round))
  860. }
  861. log.Info(Fmt("enterPrevoteWait(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  862. defer func() {
  863. // Done enterPrevoteWait:
  864. cs.updateRoundStep(round, RoundStepPrevoteWait)
  865. cs.newStep()
  866. }()
  867. // Wait for some more prevotes; enterPrecommit
  868. cs.scheduleTimeout(cs.timeoutParams.Prevote(round), height, round, RoundStepPrevoteWait)
  869. }
  870. // Enter: +2/3 precomits for block or nil.
  871. // Enter: `timeoutPrevote` after any +2/3 prevotes.
  872. // Enter: any +2/3 precommits for next round.
  873. // Lock & precommit the ProposalBlock if we have enough prevotes for it (a POL in this round)
  874. // else, unlock an existing lock and precommit nil if +2/3 of prevotes were nil,
  875. // else, precommit nil otherwise.
  876. func (cs *ConsensusState) enterPrecommit(height int, round int) {
  877. if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPrecommit <= cs.Step) {
  878. log.Debug(Fmt("enterPrecommit(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  879. return
  880. }
  881. // cs.stopTimer()
  882. log.Info(Fmt("enterPrecommit(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  883. defer func() {
  884. // Done enterPrecommit:
  885. cs.updateRoundStep(round, RoundStepPrecommit)
  886. cs.newStep()
  887. }()
  888. hash, partsHeader, ok := cs.Votes.Prevotes(round).TwoThirdsMajority()
  889. // If we don't have a polka, we must precommit nil
  890. if !ok {
  891. if cs.LockedBlock != nil {
  892. log.Info("enterPrecommit: No +2/3 prevotes during enterPrecommit while we're locked. Precommitting nil")
  893. } else {
  894. log.Info("enterPrecommit: No +2/3 prevotes during enterPrecommit. Precommitting nil.")
  895. }
  896. cs.signAddVote(types.VoteTypePrecommit, nil, types.PartSetHeader{})
  897. return
  898. }
  899. // At this point +2/3 prevoted for a particular block or nil
  900. cs.evsw.FireEvent(types.EventStringPolka(), cs.RoundStateEvent())
  901. // the latest POLRound should be this round
  902. if cs.Votes.POLRound() < round {
  903. PanicSanity(Fmt("This POLRound should be %v but got %", round, cs.Votes.POLRound()))
  904. }
  905. // +2/3 prevoted nil. Unlock and precommit nil.
  906. if len(hash) == 0 {
  907. if cs.LockedBlock == nil {
  908. log.Notice("enterPrecommit: +2/3 prevoted for nil.")
  909. } else {
  910. log.Notice("enterPrecommit: +2/3 prevoted for nil. Unlocking")
  911. cs.LockedRound = 0
  912. cs.LockedBlock = nil
  913. cs.LockedBlockParts = nil
  914. cs.evsw.FireEvent(types.EventStringUnlock(), cs.RoundStateEvent())
  915. }
  916. cs.signAddVote(types.VoteTypePrecommit, nil, types.PartSetHeader{})
  917. return
  918. }
  919. // At this point, +2/3 prevoted for a particular block.
  920. // If we're already locked on that block, precommit it, and update the LockedRound
  921. if cs.LockedBlock.HashesTo(hash) {
  922. log.Notice("enterPrecommit: +2/3 prevoted locked block. Relocking")
  923. cs.LockedRound = round
  924. cs.evsw.FireEvent(types.EventStringRelock(), cs.RoundStateEvent())
  925. cs.signAddVote(types.VoteTypePrecommit, hash, partsHeader)
  926. return
  927. }
  928. // If +2/3 prevoted for proposal block, stage and precommit it
  929. if cs.ProposalBlock.HashesTo(hash) {
  930. log.Notice("enterPrecommit: +2/3 prevoted proposal block. Locking", "hash", hash)
  931. // Validate the block.
  932. if err := cs.state.ValidateBlock(cs.ProposalBlock); err != nil {
  933. PanicConsensus(Fmt("enterPrecommit: +2/3 prevoted for an invalid block: %v", err))
  934. }
  935. cs.LockedRound = round
  936. cs.LockedBlock = cs.ProposalBlock
  937. cs.LockedBlockParts = cs.ProposalBlockParts
  938. cs.evsw.FireEvent(types.EventStringLock(), cs.RoundStateEvent())
  939. cs.signAddVote(types.VoteTypePrecommit, hash, partsHeader)
  940. return
  941. }
  942. // There was a polka in this round for a block we don't have.
  943. // Fetch that block, unlock, and precommit nil.
  944. // The +2/3 prevotes for this round is the POL for our unlock.
  945. // TODO: In the future save the POL prevotes for justification.
  946. cs.LockedRound = 0
  947. cs.LockedBlock = nil
  948. cs.LockedBlockParts = nil
  949. if !cs.ProposalBlockParts.HasHeader(partsHeader) {
  950. cs.ProposalBlock = nil
  951. cs.ProposalBlockParts = types.NewPartSetFromHeader(partsHeader)
  952. }
  953. cs.evsw.FireEvent(types.EventStringUnlock(), cs.RoundStateEvent())
  954. cs.signAddVote(types.VoteTypePrecommit, nil, types.PartSetHeader{})
  955. return
  956. }
  957. // Enter: any +2/3 precommits for next round.
  958. func (cs *ConsensusState) enterPrecommitWait(height int, round int) {
  959. if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPrecommitWait <= cs.Step) {
  960. log.Debug(Fmt("enterPrecommitWait(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  961. return
  962. }
  963. if !cs.Votes.Precommits(round).HasTwoThirdsAny() {
  964. PanicSanity(Fmt("enterPrecommitWait(%v/%v), but Precommits does not have any +2/3 votes", height, round))
  965. }
  966. log.Info(Fmt("enterPrecommitWait(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  967. defer func() {
  968. // Done enterPrecommitWait:
  969. cs.updateRoundStep(round, RoundStepPrecommitWait)
  970. cs.newStep()
  971. }()
  972. // Wait for some more precommits; enterNewRound
  973. cs.scheduleTimeout(cs.timeoutParams.Precommit(round), height, round, RoundStepPrecommitWait)
  974. }
  975. // Enter: +2/3 precommits for block
  976. func (cs *ConsensusState) enterCommit(height int, commitRound int) {
  977. if cs.Height != height || RoundStepCommit <= cs.Step {
  978. log.Debug(Fmt("enterCommit(%v/%v): Invalid args. Current step: %v/%v/%v", height, commitRound, cs.Height, cs.Round, cs.Step))
  979. return
  980. }
  981. log.Info(Fmt("enterCommit(%v/%v). Current: %v/%v/%v", height, commitRound, cs.Height, cs.Round, cs.Step))
  982. defer func() {
  983. // Done enterCommit:
  984. // keep cs.Round the same, commitRound points to the right Precommits set.
  985. cs.updateRoundStep(cs.Round, RoundStepCommit)
  986. cs.CommitRound = commitRound
  987. cs.newStep()
  988. // Maybe finalize immediately.
  989. cs.tryFinalizeCommit(height)
  990. }()
  991. hash, partsHeader, ok := cs.Votes.Precommits(commitRound).TwoThirdsMajority()
  992. if !ok {
  993. PanicSanity("RunActionCommit() expects +2/3 precommits")
  994. }
  995. // The Locked* fields no longer matter.
  996. // Move them over to ProposalBlock if they match the commit hash,
  997. // otherwise they'll be cleared in updateToState.
  998. if cs.LockedBlock.HashesTo(hash) {
  999. cs.ProposalBlock = cs.LockedBlock
  1000. cs.ProposalBlockParts = cs.LockedBlockParts
  1001. }
  1002. // If we don't have the block being committed, set up to get it.
  1003. if !cs.ProposalBlock.HashesTo(hash) {
  1004. if !cs.ProposalBlockParts.HasHeader(partsHeader) {
  1005. // We're getting the wrong block.
  1006. // Set up ProposalBlockParts and keep waiting.
  1007. cs.ProposalBlock = nil
  1008. cs.ProposalBlockParts = types.NewPartSetFromHeader(partsHeader)
  1009. } else {
  1010. // We just need to keep waiting.
  1011. }
  1012. }
  1013. }
  1014. // If we have the block AND +2/3 commits for it, finalize.
  1015. func (cs *ConsensusState) tryFinalizeCommit(height int) {
  1016. if cs.Height != height {
  1017. PanicSanity(Fmt("tryFinalizeCommit() cs.Height: %v vs height: %v", cs.Height, height))
  1018. }
  1019. hash, _, ok := cs.Votes.Precommits(cs.CommitRound).TwoThirdsMajority()
  1020. if !ok || len(hash) == 0 {
  1021. log.Warn("Attempt to finalize failed. There was no +2/3 majority, or +2/3 was for <nil>.")
  1022. return
  1023. }
  1024. if !cs.ProposalBlock.HashesTo(hash) {
  1025. // TODO: this happens every time if we're not a validator (ugly logs)
  1026. log.Warn("Attempt to finalize failed. We don't have the commit block.")
  1027. return
  1028. }
  1029. // go
  1030. cs.finalizeCommit(height)
  1031. }
  1032. // Increment height and goto RoundStepNewHeight
  1033. func (cs *ConsensusState) finalizeCommit(height int) {
  1034. if cs.Height != height || cs.Step != RoundStepCommit {
  1035. log.Debug(Fmt("finalizeCommit(%v): Invalid args. Current step: %v/%v/%v", height, cs.Height, cs.Round, cs.Step))
  1036. return
  1037. }
  1038. hash, header, ok := cs.Votes.Precommits(cs.CommitRound).TwoThirdsMajority()
  1039. block, blockParts := cs.ProposalBlock, cs.ProposalBlockParts
  1040. if !ok {
  1041. PanicSanity(Fmt("Cannot finalizeCommit, commit does not have two thirds majority"))
  1042. }
  1043. if !blockParts.HasHeader(header) {
  1044. PanicSanity(Fmt("Expected ProposalBlockParts header to be commit header"))
  1045. }
  1046. if !block.HashesTo(hash) {
  1047. PanicSanity(Fmt("Cannot finalizeCommit, ProposalBlock does not hash to commit hash"))
  1048. }
  1049. if err := cs.state.ValidateBlock(block); err != nil {
  1050. PanicConsensus(Fmt("+2/3 committed an invalid block: %v", err))
  1051. }
  1052. log.Notice(Fmt("Finalizing commit of block with %d txs", block.NumTxs), "height", block.Height, "hash", block.Hash())
  1053. log.Info(Fmt("%v", block))
  1054. // Fire off event for new block.
  1055. // TODO: Handle app failure. See #177
  1056. cs.evsw.FireEvent(types.EventStringNewBlock(), types.EventDataNewBlock{block})
  1057. // Create a copy of the state for staging
  1058. stateCopy := cs.state.Copy()
  1059. // Run the block on the State:
  1060. // + update validator sets
  1061. // + run txs on the proxyAppConn
  1062. err := stateCopy.ExecBlock(cs.evsw, cs.proxyAppConn, block, blockParts.Header())
  1063. if err != nil {
  1064. // TODO: handle this gracefully.
  1065. PanicQ(Fmt("Exec failed for application"))
  1066. }
  1067. // Save to blockStore.
  1068. if cs.blockStore.Height() < block.Height {
  1069. commits := cs.Votes.Precommits(cs.CommitRound)
  1070. seenValidation := commits.MakeValidation()
  1071. cs.blockStore.SaveBlock(block, blockParts, seenValidation)
  1072. }
  1073. /*
  1074. // Commit to proxyAppConn
  1075. err = cs.proxyAppConn.CommitSync()
  1076. if err != nil {
  1077. // TODO: handle this gracefully.
  1078. PanicQ(Fmt("Commit failed for application"))
  1079. }
  1080. */
  1081. // Save the state.
  1082. stateCopy.Save()
  1083. // Update mempool.
  1084. cs.mempool.Update(block.Height, block.Txs)
  1085. // NewHeightStep!
  1086. cs.updateToState(stateCopy)
  1087. // cs.StartTime is already set.
  1088. // Schedule Round0 to start soon.
  1089. cs.scheduleRound0(height + 1)
  1090. // By here,
  1091. // * cs.Height has been increment to height+1
  1092. // * cs.Step is now RoundStepNewHeight
  1093. // * cs.StartTime is set to when we will start round0.
  1094. return
  1095. }
  1096. //-----------------------------------------------------------------------------
  1097. func (cs *ConsensusState) setProposal(proposal *types.Proposal) error {
  1098. // Already have one
  1099. if cs.Proposal != nil {
  1100. return nil
  1101. }
  1102. // Does not apply
  1103. if proposal.Height != cs.Height || proposal.Round != cs.Round {
  1104. return nil
  1105. }
  1106. // We don't care about the proposal if we're already in RoundStepCommit.
  1107. if RoundStepCommit <= cs.Step {
  1108. return nil
  1109. }
  1110. // Verify POLRound, which must be -1 or between 0 and proposal.Round exclusive.
  1111. if proposal.POLRound != -1 &&
  1112. (proposal.POLRound < 0 || proposal.Round <= proposal.POLRound) {
  1113. return ErrInvalidProposalPOLRound
  1114. }
  1115. // Verify signature
  1116. if !cs.Validators.Proposer().PubKey.VerifyBytes(types.SignBytes(cs.state.ChainID, proposal), proposal.Signature) {
  1117. return ErrInvalidProposalSignature
  1118. }
  1119. cs.Proposal = proposal
  1120. cs.ProposalBlockParts = types.NewPartSetFromHeader(proposal.BlockPartsHeader)
  1121. return nil
  1122. }
  1123. // NOTE: block is not necessarily valid.
  1124. // Asynchronously triggers either enterPrevote (before we timeout of propose) or tryFinalizeCommit, once we have the full block.
  1125. func (cs *ConsensusState) addProposalBlockPart(height int, part *types.Part) (added bool, err error) {
  1126. // Blocks might be reused, so round mismatch is OK
  1127. if cs.Height != height {
  1128. return false, nil
  1129. }
  1130. // We're not expecting a block part.
  1131. if cs.ProposalBlockParts == nil {
  1132. return false, nil // TODO: bad peer? Return error?
  1133. }
  1134. added, err = cs.ProposalBlockParts.AddPart(part)
  1135. if err != nil {
  1136. return added, err
  1137. }
  1138. if added && cs.ProposalBlockParts.IsComplete() {
  1139. // Added and completed!
  1140. var n int
  1141. var err error
  1142. cs.ProposalBlock = wire.ReadBinary(&types.Block{}, cs.ProposalBlockParts.GetReader(), types.MaxBlockSize, &n, &err).(*types.Block)
  1143. // NOTE: it's possible to receive complete proposal blocks for future rounds without having the proposal
  1144. log.Info("Received complete proposal block", "height", cs.ProposalBlock.Height, "hash", cs.ProposalBlock.Hash())
  1145. if cs.Step == RoundStepPropose && cs.isProposalComplete() {
  1146. // Move onto the next step
  1147. cs.enterPrevote(height, cs.Round)
  1148. } else if cs.Step == RoundStepCommit {
  1149. // If we're waiting on the proposal block...
  1150. cs.tryFinalizeCommit(height)
  1151. }
  1152. return true, err
  1153. }
  1154. return added, nil
  1155. }
  1156. // Attempt to add the vote. if its a duplicate signature, dupeout the validator
  1157. func (cs *ConsensusState) tryAddVote(valIndex int, vote *types.Vote, peerKey string) error {
  1158. _, _, err := cs.addVote(valIndex, vote, peerKey)
  1159. if err != nil {
  1160. // If the vote height is off, we'll just ignore it,
  1161. // But if it's a conflicting sig, broadcast evidence tx for slashing.
  1162. // If it's otherwise invalid, punish peer.
  1163. if err == ErrVoteHeightMismatch {
  1164. return err
  1165. } else if _, ok := err.(*types.ErrVoteConflictingSignature); ok {
  1166. if peerKey == "" {
  1167. log.Warn("Found conflicting vote from ourselves. Did you unsafe_reset a validator?", "height", vote.Height, "round", vote.Round, "type", vote.Type)
  1168. return err
  1169. }
  1170. log.Warn("Found conflicting vote. Publish evidence (TODO)")
  1171. /* TODO
  1172. evidenceTx := &types.DupeoutTx{
  1173. Address: address,
  1174. VoteA: *errDupe.VoteA,
  1175. VoteB: *errDupe.VoteB,
  1176. }
  1177. cs.mempool.BroadcastTx(struct{???}{evidenceTx}) // shouldn't need to check returned err
  1178. */
  1179. return err
  1180. } else {
  1181. // Probably an invalid signature. Bad peer.
  1182. log.Warn("Error attempting to add vote", "error", err)
  1183. return ErrAddingVote
  1184. }
  1185. }
  1186. return nil
  1187. }
  1188. //-----------------------------------------------------------------------------
  1189. func (cs *ConsensusState) addVote(valIndex int, vote *types.Vote, peerKey string) (added bool, address []byte, err error) {
  1190. log.Debug("addVote", "voteHeight", vote.Height, "voteType", vote.Type, "csHeight", cs.Height)
  1191. // A precommit for the previous height?
  1192. if vote.Height+1 == cs.Height {
  1193. if !(cs.Step == RoundStepNewHeight && vote.Type == types.VoteTypePrecommit) {
  1194. // TODO: give the reason ..
  1195. // fmt.Errorf("tryAddVote: Wrong height, not a LastCommit straggler commit.")
  1196. return added, nil, ErrVoteHeightMismatch
  1197. }
  1198. added, address, err = cs.LastCommit.AddByIndex(valIndex, vote)
  1199. if added {
  1200. log.Info(Fmt("Added to lastPrecommits: %v", cs.LastCommit.StringShort()))
  1201. cs.evsw.FireEvent(types.EventStringVote(), types.EventDataVote{valIndex, address, vote})
  1202. }
  1203. return
  1204. }
  1205. // A prevote/precommit for this height?
  1206. if vote.Height == cs.Height {
  1207. height := cs.Height
  1208. added, address, err = cs.Votes.AddByIndex(valIndex, vote, peerKey)
  1209. if added {
  1210. cs.evsw.FireEvent(types.EventStringVote(), types.EventDataVote{valIndex, address, vote})
  1211. switch vote.Type {
  1212. case types.VoteTypePrevote:
  1213. prevotes := cs.Votes.Prevotes(vote.Round)
  1214. log.Info("Added to prevote", "vote", vote, "prevotes", prevotes.StringShort())
  1215. // First, unlock if prevotes is a valid POL.
  1216. // >> lockRound < POLRound <= unlockOrChangeLockRound (see spec)
  1217. // NOTE: If (lockRound < POLRound) but !(POLRound <= unlockOrChangeLockRound),
  1218. // we'll still enterNewRound(H,vote.R) and enterPrecommit(H,vote.R) to process it
  1219. // there.
  1220. if (cs.LockedBlock != nil) && (cs.LockedRound < vote.Round) && (vote.Round <= cs.Round) {
  1221. hash, _, ok := prevotes.TwoThirdsMajority()
  1222. if ok && !cs.LockedBlock.HashesTo(hash) {
  1223. log.Notice("Unlocking because of POL.", "lockedRound", cs.LockedRound, "POLRound", vote.Round)
  1224. cs.LockedRound = 0
  1225. cs.LockedBlock = nil
  1226. cs.LockedBlockParts = nil
  1227. cs.evsw.FireEvent(types.EventStringUnlock(), cs.RoundStateEvent())
  1228. }
  1229. }
  1230. if cs.Round <= vote.Round && prevotes.HasTwoThirdsAny() {
  1231. // Round-skip over to PrevoteWait or goto Precommit.
  1232. cs.enterNewRound(height, vote.Round) // if the vote is ahead of us
  1233. if prevotes.HasTwoThirdsMajority() {
  1234. cs.enterPrecommit(height, vote.Round)
  1235. } else {
  1236. cs.enterPrevote(height, vote.Round) // if the vote is ahead of us
  1237. cs.enterPrevoteWait(height, vote.Round)
  1238. }
  1239. } else if cs.Proposal != nil && 0 <= cs.Proposal.POLRound && cs.Proposal.POLRound == vote.Round {
  1240. // If the proposal is now complete, enter prevote of cs.Round.
  1241. if cs.isProposalComplete() {
  1242. cs.enterPrevote(height, cs.Round)
  1243. }
  1244. }
  1245. case types.VoteTypePrecommit:
  1246. precommits := cs.Votes.Precommits(vote.Round)
  1247. log.Info("Added to precommit", "vote", vote, "precommits", precommits.StringShort())
  1248. hash, _, ok := precommits.TwoThirdsMajority()
  1249. if ok {
  1250. if len(hash) == 0 {
  1251. cs.enterNewRound(height, vote.Round+1)
  1252. } else {
  1253. cs.enterNewRound(height, vote.Round)
  1254. cs.enterPrecommit(height, vote.Round)
  1255. cs.enterCommit(height, vote.Round)
  1256. }
  1257. } else if cs.Round <= vote.Round && precommits.HasTwoThirdsAny() {
  1258. cs.enterNewRound(height, vote.Round)
  1259. cs.enterPrecommit(height, vote.Round)
  1260. cs.enterPrecommitWait(height, vote.Round)
  1261. //}()
  1262. }
  1263. default:
  1264. PanicSanity(Fmt("Unexpected vote type %X", vote.Type)) // Should not happen.
  1265. }
  1266. }
  1267. // Either duplicate, or error upon cs.Votes.AddByIndex()
  1268. return
  1269. } else {
  1270. err = ErrVoteHeightMismatch
  1271. }
  1272. // Height mismatch, bad peer?
  1273. log.Info("Vote ignored and not added", "voteHeight", vote.Height, "csHeight", cs.Height)
  1274. return
  1275. }
  1276. func (cs *ConsensusState) signVote(type_ byte, hash []byte, header types.PartSetHeader) (*types.Vote, error) {
  1277. vote := &types.Vote{
  1278. Height: cs.Height,
  1279. Round: cs.Round,
  1280. Type: type_,
  1281. BlockHash: hash,
  1282. BlockPartsHeader: header,
  1283. }
  1284. err := cs.privValidator.SignVote(cs.state.ChainID, vote)
  1285. return vote, err
  1286. }
  1287. // signs the vote, publishes on internalMsgQueue
  1288. func (cs *ConsensusState) signAddVote(type_ byte, hash []byte, header types.PartSetHeader) *types.Vote {
  1289. if cs.privValidator == nil || !cs.Validators.HasAddress(cs.privValidator.Address) {
  1290. return nil
  1291. }
  1292. vote, err := cs.signVote(type_, hash, header)
  1293. if err == nil {
  1294. // TODO: store our index in the cs so we don't have to do this every time
  1295. valIndex, _ := cs.Validators.GetByAddress(cs.privValidator.Address)
  1296. cs.sendInternalMessage(msgInfo{&VoteMessage{valIndex, vote}, ""})
  1297. log.Info("Signed and pushed vote", "height", cs.Height, "round", cs.Round, "vote", vote, "error", err)
  1298. return vote
  1299. } else {
  1300. log.Warn("Error signing vote", "height", cs.Height, "round", cs.Round, "vote", vote, "error", err)
  1301. return nil
  1302. }
  1303. }
  1304. //---------------------------------------------------------
  1305. func CompareHRS(h1, r1 int, s1 RoundStepType, h2, r2 int, s2 RoundStepType) int {
  1306. if h1 < h2 {
  1307. return -1
  1308. } else if h1 > h2 {
  1309. return 1
  1310. }
  1311. if r1 < r2 {
  1312. return -1
  1313. } else if r1 > r2 {
  1314. return 1
  1315. }
  1316. if s1 < s2 {
  1317. return -1
  1318. } else if s1 > s2 {
  1319. return 1
  1320. }
  1321. return 0
  1322. }