You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1506 lines
50 KiB

10 years ago
10 years ago
10 years ago
10 years ago
9 years ago
10 years ago
10 years ago
10 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
9 years ago
9 years ago
10 years ago
10 years ago
10 years ago
10 years ago
9 years ago
9 years ago
9 years ago
9 years ago
10 years ago
10 years ago
9 years ago
9 years ago
9 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
9 years ago
9 years ago
10 years ago
9 years ago
9 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
10 years ago
10 years ago
9 years ago
9 years ago
9 years ago
10 years ago
10 years ago
10 years ago
9 years ago
10 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
10 years ago
10 years ago
  1. package consensus
  2. import (
  3. "bytes"
  4. "errors"
  5. "fmt"
  6. "reflect"
  7. "sync"
  8. "time"
  9. . "github.com/tendermint/go-common"
  10. "github.com/tendermint/go-events"
  11. "github.com/tendermint/go-wire"
  12. bc "github.com/tendermint/tendermint/blockchain"
  13. mempl "github.com/tendermint/tendermint/mempool"
  14. "github.com/tendermint/tendermint/proxy"
  15. sm "github.com/tendermint/tendermint/state"
  16. "github.com/tendermint/tendermint/types"
  17. )
  18. //-----------------------------------------------------------------------------
  19. // Timeout Parameters
  20. // All in milliseconds
  21. type TimeoutParams struct {
  22. Propose0 int
  23. ProposeDelta int
  24. Prevote0 int
  25. PrevoteDelta int
  26. Precommit0 int
  27. PrecommitDelta int
  28. Commit0 int
  29. }
  30. // Wait this long for a proposal
  31. func (tp *TimeoutParams) Propose(round int) time.Duration {
  32. return time.Duration(tp.Propose0+tp.ProposeDelta*round) * time.Millisecond
  33. }
  34. // After receiving any +2/3 prevote, wait this long for stragglers
  35. func (tp *TimeoutParams) Prevote(round int) time.Duration {
  36. return time.Duration(tp.Prevote0+tp.PrevoteDelta*round) * time.Millisecond
  37. }
  38. // After receiving any +2/3 precommits, wait this long for stragglers
  39. func (tp *TimeoutParams) Precommit(round int) time.Duration {
  40. return time.Duration(tp.Precommit0+tp.PrecommitDelta*round) * time.Millisecond
  41. }
  42. // After receiving +2/3 precommits for a single block (a commit), wait this long for stragglers in the next height's RoundStepNewHeight
  43. func (tp *TimeoutParams) Commit(t time.Time) time.Time {
  44. return t.Add(time.Duration(tp.Commit0) * time.Millisecond)
  45. }
  46. // Initialize parameters from config file
  47. func InitTimeoutParamsFromConfig() *TimeoutParams {
  48. return &TimeoutParams{
  49. Propose0: config.GetInt("timeout_propose"),
  50. ProposeDelta: config.GetInt("timeout_propose_delta"),
  51. Prevote0: config.GetInt("timeout_prevote"),
  52. PrevoteDelta: config.GetInt("timeout_prevote_delta"),
  53. Precommit0: config.GetInt("timeout_precommit"),
  54. PrecommitDelta: config.GetInt("timeout_precommit_delta"),
  55. Commit0: config.GetInt("timeout_commit"),
  56. }
  57. }
  58. //-----------------------------------------------------------------------------
  59. // Errors
  60. var (
  61. ErrInvalidProposalSignature = errors.New("Error invalid proposal signature")
  62. ErrInvalidProposalPOLRound = errors.New("Error invalid proposal POL round")
  63. ErrAddingVote = errors.New("Error adding vote")
  64. ErrVoteHeightMismatch = errors.New("Error vote height mismatch")
  65. )
  66. //-----------------------------------------------------------------------------
  67. // RoundStepType enum type
  68. type RoundStepType uint8 // These must be numeric, ordered.
  69. const (
  70. RoundStepNewHeight = RoundStepType(0x01) // Wait til CommitTime + timeoutCommit
  71. RoundStepNewRound = RoundStepType(0x02) // Setup new round and go to RoundStepPropose
  72. RoundStepPropose = RoundStepType(0x03) // Did propose, gossip proposal
  73. RoundStepPrevote = RoundStepType(0x04) // Did prevote, gossip prevotes
  74. RoundStepPrevoteWait = RoundStepType(0x05) // Did receive any +2/3 prevotes, start timeout
  75. RoundStepPrecommit = RoundStepType(0x06) // Did precommit, gossip precommits
  76. RoundStepPrecommitWait = RoundStepType(0x07) // Did receive any +2/3 precommits, start timeout
  77. RoundStepCommit = RoundStepType(0x08) // Entered commit state machine
  78. // NOTE: RoundStepNewHeight acts as RoundStepCommitWait.
  79. )
  80. func (rs RoundStepType) String() string {
  81. switch rs {
  82. case RoundStepNewHeight:
  83. return "RoundStepNewHeight"
  84. case RoundStepNewRound:
  85. return "RoundStepNewRound"
  86. case RoundStepPropose:
  87. return "RoundStepPropose"
  88. case RoundStepPrevote:
  89. return "RoundStepPrevote"
  90. case RoundStepPrevoteWait:
  91. return "RoundStepPrevoteWait"
  92. case RoundStepPrecommit:
  93. return "RoundStepPrecommit"
  94. case RoundStepPrecommitWait:
  95. return "RoundStepPrecommitWait"
  96. case RoundStepCommit:
  97. return "RoundStepCommit"
  98. default:
  99. return "RoundStepUnknown" // Cannot panic.
  100. }
  101. }
  102. //-----------------------------------------------------------------------------
  103. // Immutable when returned from ConsensusState.GetRoundState()
  104. type RoundState struct {
  105. Height int // Height we are working on
  106. Round int
  107. Step RoundStepType
  108. StartTime time.Time
  109. CommitTime time.Time // Subjective time when +2/3 precommits for Block at Round were found
  110. Validators *types.ValidatorSet
  111. Proposal *types.Proposal
  112. ProposalBlock *types.Block
  113. ProposalBlockParts *types.PartSet
  114. LockedRound int
  115. LockedBlock *types.Block
  116. LockedBlockParts *types.PartSet
  117. Votes *HeightVoteSet
  118. CommitRound int //
  119. LastCommit *types.VoteSet // Last precommits at Height-1
  120. LastValidators *types.ValidatorSet
  121. }
  122. func (rs *RoundState) RoundStateEvent() types.EventDataRoundState {
  123. edrs := types.EventDataRoundState{
  124. Height: rs.Height,
  125. Round: rs.Round,
  126. Step: rs.Step.String(),
  127. RoundState: rs,
  128. }
  129. return edrs
  130. }
  131. func (rs *RoundState) String() string {
  132. return rs.StringIndented("")
  133. }
  134. func (rs *RoundState) StringIndented(indent string) string {
  135. return fmt.Sprintf(`RoundState{
  136. %s H:%v R:%v S:%v
  137. %s StartTime: %v
  138. %s CommitTime: %v
  139. %s Validators: %v
  140. %s Proposal: %v
  141. %s ProposalBlock: %v %v
  142. %s LockedRound: %v
  143. %s LockedBlock: %v %v
  144. %s Votes: %v
  145. %s LastCommit: %v
  146. %s LastValidators: %v
  147. %s}`,
  148. indent, rs.Height, rs.Round, rs.Step,
  149. indent, rs.StartTime,
  150. indent, rs.CommitTime,
  151. indent, rs.Validators.StringIndented(indent+" "),
  152. indent, rs.Proposal,
  153. indent, rs.ProposalBlockParts.StringShort(), rs.ProposalBlock.StringShort(),
  154. indent, rs.LockedRound,
  155. indent, rs.LockedBlockParts.StringShort(), rs.LockedBlock.StringShort(),
  156. indent, rs.Votes.StringIndented(indent+" "),
  157. indent, rs.LastCommit.StringShort(),
  158. indent, rs.LastValidators.StringIndented(indent+" "),
  159. indent)
  160. }
  161. func (rs *RoundState) StringShort() string {
  162. return fmt.Sprintf(`RoundState{H:%v R:%v S:%v ST:%v}`,
  163. rs.Height, rs.Round, rs.Step, rs.StartTime)
  164. }
  165. //-----------------------------------------------------------------------------
  166. var (
  167. msgQueueSize = 1000
  168. tickTockBufferSize = 10
  169. )
  170. // msgs from the reactor which may update the state
  171. type msgInfo struct {
  172. Msg ConsensusMessage `json:"msg"`
  173. PeerKey string `json:"peer_key"`
  174. }
  175. // internally generated messages which may update the state
  176. type timeoutInfo struct {
  177. Duration time.Duration `json:"duration"`
  178. Height int `json:"height"`
  179. Round int `json:"round"`
  180. Step RoundStepType `json:"step"`
  181. }
  182. func (ti *timeoutInfo) String() string {
  183. return fmt.Sprintf("%v ; %d/%d %v", ti.Duration, ti.Height, ti.Round, ti.Step)
  184. }
  185. // Tracks consensus state across block heights and rounds.
  186. type ConsensusState struct {
  187. QuitService
  188. proxyAppConn proxy.AppConn
  189. blockStore *bc.BlockStore
  190. mempool *mempl.Mempool
  191. privValidator *types.PrivValidator
  192. mtx sync.Mutex
  193. RoundState
  194. state *sm.State // State until height-1.
  195. peerMsgQueue chan msgInfo // serializes msgs affecting state (proposals, block parts, votes)
  196. internalMsgQueue chan msgInfo // like peerMsgQueue but for our own proposals, parts, votes
  197. timeoutTicker *time.Ticker // ticker for timeouts
  198. tickChan chan timeoutInfo // start the timeoutTicker in the timeoutRoutine
  199. tockChan chan timeoutInfo // timeouts are relayed on tockChan to the receiveRoutine
  200. timeoutParams *TimeoutParams // parameters and functions for timeout intervals
  201. evsw *events.EventSwitch
  202. wal *WAL
  203. nSteps int // used for testing to limit the number of transitions the state makes
  204. }
  205. func NewConsensusState(state *sm.State, proxyAppConn proxy.AppConn, blockStore *bc.BlockStore, mempool *mempl.Mempool) *ConsensusState {
  206. cs := &ConsensusState{
  207. proxyAppConn: proxyAppConn,
  208. blockStore: blockStore,
  209. mempool: mempool,
  210. peerMsgQueue: make(chan msgInfo, msgQueueSize),
  211. internalMsgQueue: make(chan msgInfo, msgQueueSize),
  212. timeoutTicker: new(time.Ticker),
  213. tickChan: make(chan timeoutInfo, tickTockBufferSize),
  214. tockChan: make(chan timeoutInfo, tickTockBufferSize),
  215. timeoutParams: InitTimeoutParamsFromConfig(),
  216. }
  217. cs.updateToState(state)
  218. // Don't call scheduleRound0 yet.
  219. // We do that upon Start().
  220. cs.reconstructLastCommit(state)
  221. cs.QuitService = *NewQuitService(log, "ConsensusState", cs)
  222. return cs
  223. }
  224. //----------------------------------------
  225. // Public interface
  226. // implements events.Eventable
  227. func (cs *ConsensusState) SetEventSwitch(evsw *events.EventSwitch) {
  228. cs.evsw = evsw
  229. }
  230. func (cs *ConsensusState) String() string {
  231. return Fmt("ConsensusState(H:%v R:%v S:%v", cs.Height, cs.Round, cs.Step)
  232. }
  233. func (cs *ConsensusState) GetState() *sm.State {
  234. cs.mtx.Lock()
  235. defer cs.mtx.Unlock()
  236. return cs.state.Copy()
  237. }
  238. func (cs *ConsensusState) GetRoundState() *RoundState {
  239. cs.mtx.Lock()
  240. defer cs.mtx.Unlock()
  241. return cs.getRoundState()
  242. }
  243. func (cs *ConsensusState) getRoundState() *RoundState {
  244. rs := cs.RoundState // copy
  245. return &rs
  246. }
  247. func (cs *ConsensusState) SetPrivValidator(priv *types.PrivValidator) {
  248. cs.mtx.Lock()
  249. defer cs.mtx.Unlock()
  250. cs.privValidator = priv
  251. }
  252. func (cs *ConsensusState) OnStart() error {
  253. cs.QuitService.OnStart()
  254. // start timeout and receive routines
  255. cs.startRoutines(0)
  256. // we may have lost some votes if the process crashed
  257. // reload from consensus log to catchup
  258. if err := cs.catchupReplay(cs.Height); err != nil {
  259. log.Error("Error on catchup replay", "error", err.Error())
  260. // let's go for it anyways, maybe we're fine
  261. }
  262. // schedule the first round!
  263. cs.scheduleRound0(cs.Height)
  264. return nil
  265. }
  266. // timeoutRoutine: receive requests for timeouts on tickChan and fire timeouts on tockChan
  267. // receiveRoutine: serializes processing of proposoals, block parts, votes; coordinates state transitions
  268. func (cs *ConsensusState) startRoutines(maxSteps int) {
  269. go cs.timeoutRoutine()
  270. go cs.receiveRoutine(maxSteps)
  271. }
  272. func (cs *ConsensusState) OnStop() {
  273. cs.QuitService.OnStop()
  274. if cs.wal != nil && cs.IsRunning() {
  275. cs.wal.Wait()
  276. }
  277. }
  278. // Open file to log all consensus messages and timeouts for deterministic accountability
  279. func (cs *ConsensusState) OpenWAL(file string) (err error) {
  280. cs.mtx.Lock()
  281. defer cs.mtx.Unlock()
  282. wal, err := NewWAL(file, config.GetBool("cswal_light"))
  283. if err != nil {
  284. return err
  285. }
  286. cs.wal = wal
  287. return nil
  288. }
  289. //------------------------------------------------------------
  290. // Public interface for passing messages into the consensus state,
  291. // possibly causing a state transition
  292. // TODO: should these return anything or let callers just use events?
  293. // May block on send if queue is full.
  294. func (cs *ConsensusState) AddVote(valIndex int, vote *types.Vote, peerKey string) (added bool, address []byte, err error) {
  295. if peerKey == "" {
  296. cs.internalMsgQueue <- msgInfo{&VoteMessage{valIndex, vote}, ""}
  297. } else {
  298. cs.peerMsgQueue <- msgInfo{&VoteMessage{valIndex, vote}, peerKey}
  299. }
  300. // TODO: wait for event?!
  301. return false, nil, nil
  302. }
  303. // May block on send if queue is full.
  304. func (cs *ConsensusState) SetProposal(proposal *types.Proposal, peerKey string) error {
  305. if peerKey == "" {
  306. cs.internalMsgQueue <- msgInfo{&ProposalMessage{proposal}, ""}
  307. } else {
  308. cs.peerMsgQueue <- msgInfo{&ProposalMessage{proposal}, peerKey}
  309. }
  310. // TODO: wait for event?!
  311. return nil
  312. }
  313. // May block on send if queue is full.
  314. func (cs *ConsensusState) AddProposalBlockPart(height, round int, part *types.Part, peerKey string) error {
  315. if peerKey == "" {
  316. cs.internalMsgQueue <- msgInfo{&BlockPartMessage{height, round, part}, ""}
  317. } else {
  318. cs.peerMsgQueue <- msgInfo{&BlockPartMessage{height, round, part}, peerKey}
  319. }
  320. // TODO: wait for event?!
  321. return nil
  322. }
  323. // May block on send if queue is full.
  324. func (cs *ConsensusState) SetProposalAndBlock(proposal *types.Proposal, block *types.Block, parts *types.PartSet, peerKey string) error {
  325. cs.SetProposal(proposal, peerKey)
  326. for i := 0; i < parts.Total(); i++ {
  327. part := parts.GetPart(i)
  328. cs.AddProposalBlockPart(proposal.Height, proposal.Round, part, peerKey)
  329. }
  330. return nil // TODO errors
  331. }
  332. //------------------------------------------------------------
  333. // internal functions for managing the state
  334. func (cs *ConsensusState) updateHeight(height int) {
  335. cs.Height = height
  336. }
  337. func (cs *ConsensusState) updateRoundStep(round int, step RoundStepType) {
  338. cs.Round = round
  339. cs.Step = step
  340. }
  341. // enterNewRound(height, 0) at cs.StartTime.
  342. func (cs *ConsensusState) scheduleRound0(height int) {
  343. //log.Info("scheduleRound0", "now", time.Now(), "startTime", cs.StartTime)
  344. sleepDuration := cs.StartTime.Sub(time.Now())
  345. if sleepDuration < time.Duration(0) {
  346. sleepDuration = time.Duration(0)
  347. }
  348. cs.scheduleTimeout(sleepDuration, height, 0, RoundStepNewHeight)
  349. }
  350. // Attempt to schedule a timeout by sending timeoutInfo on the tickChan.
  351. // The timeoutRoutine is alwaya available to read from tickChan (it won't block).
  352. // The scheduling may fail if the timeoutRoutine has already scheduled a timeout for a later height/round/step.
  353. func (cs *ConsensusState) scheduleTimeout(duration time.Duration, height, round int, step RoundStepType) {
  354. cs.tickChan <- timeoutInfo{duration, height, round, step}
  355. }
  356. // send a msg into the receiveRoutine regarding our own proposal, block part, or vote
  357. func (cs *ConsensusState) sendInternalMessage(mi msgInfo) {
  358. select {
  359. case cs.internalMsgQueue <- mi:
  360. default:
  361. // NOTE: using the go-routine means our votes can
  362. // be processed out of order.
  363. // TODO: use CList here for strict determinism and
  364. // attempt push to internalMsgQueue in receiveRoutine
  365. log.Debug("Internal msg queue is full. Using a go-routine")
  366. go func() { cs.internalMsgQueue <- mi }()
  367. }
  368. }
  369. // Reconstruct LastCommit from SeenValidation, which we saved along with the block,
  370. // (which happens even before saving the state)
  371. func (cs *ConsensusState) reconstructLastCommit(state *sm.State) {
  372. if state.LastBlockHeight == 0 {
  373. return
  374. }
  375. seenValidation := cs.blockStore.LoadSeenValidation(state.LastBlockHeight)
  376. lastPrecommits := types.NewVoteSet(state.LastBlockHeight, seenValidation.Round(), types.VoteTypePrecommit, state.LastValidators)
  377. for idx, precommit := range seenValidation.Precommits {
  378. if precommit == nil {
  379. continue
  380. }
  381. added, _, err := lastPrecommits.AddByIndex(idx, precommit)
  382. if !added || err != nil {
  383. PanicCrisis(Fmt("Failed to reconstruct LastCommit: %v", err))
  384. }
  385. }
  386. if !lastPrecommits.HasTwoThirdsMajority() {
  387. PanicSanity("Failed to reconstruct LastCommit: Does not have +2/3 maj")
  388. }
  389. cs.LastCommit = lastPrecommits
  390. }
  391. // Updates ConsensusState and increments height to match that of state.
  392. // The round becomes 0 and cs.Step becomes RoundStepNewHeight.
  393. func (cs *ConsensusState) updateToState(state *sm.State) {
  394. if cs.CommitRound > -1 && 0 < cs.Height && cs.Height != state.LastBlockHeight {
  395. PanicSanity(Fmt("updateToState() expected state height of %v but found %v",
  396. cs.Height, state.LastBlockHeight))
  397. }
  398. if cs.state != nil && cs.state.LastBlockHeight+1 != cs.Height {
  399. // This might happen when someone else is mutating cs.state.
  400. // Someone forgot to pass in state.Copy() somewhere?!
  401. PanicSanity(Fmt("Inconsistent cs.state.LastBlockHeight+1 %v vs cs.Height %v",
  402. cs.state.LastBlockHeight+1, cs.Height))
  403. }
  404. // If state isn't further out than cs.state, just ignore.
  405. // This happens when SwitchToConsensus() is called in the reactor.
  406. // We don't want to reset e.g. the Votes.
  407. if cs.state != nil && (state.LastBlockHeight <= cs.state.LastBlockHeight) {
  408. log.Notice("Ignoring updateToState()", "newHeight", state.LastBlockHeight+1, "oldHeight", cs.state.LastBlockHeight+1)
  409. return
  410. }
  411. // Reset fields based on state.
  412. validators := state.Validators
  413. height := state.LastBlockHeight + 1 // Next desired block height
  414. lastPrecommits := (*types.VoteSet)(nil)
  415. if cs.CommitRound > -1 && cs.Votes != nil {
  416. if !cs.Votes.Precommits(cs.CommitRound).HasTwoThirdsMajority() {
  417. PanicSanity("updateToState(state) called but last Precommit round didn't have +2/3")
  418. }
  419. lastPrecommits = cs.Votes.Precommits(cs.CommitRound)
  420. }
  421. // RoundState fields
  422. cs.updateHeight(height)
  423. cs.updateRoundStep(0, RoundStepNewHeight)
  424. if cs.CommitTime.IsZero() {
  425. // "Now" makes it easier to sync up dev nodes.
  426. // We add timeoutCommit to allow transactions
  427. // to be gathered for the first block.
  428. // And alternative solution that relies on clocks:
  429. // cs.StartTime = state.LastBlockTime.Add(timeoutCommit)
  430. cs.StartTime = cs.timeoutParams.Commit(time.Now())
  431. } else {
  432. cs.StartTime = cs.timeoutParams.Commit(cs.CommitTime)
  433. }
  434. cs.CommitTime = time.Time{}
  435. cs.Validators = validators
  436. cs.Proposal = nil
  437. cs.ProposalBlock = nil
  438. cs.ProposalBlockParts = nil
  439. cs.LockedRound = 0
  440. cs.LockedBlock = nil
  441. cs.LockedBlockParts = nil
  442. cs.Votes = NewHeightVoteSet(height, validators)
  443. cs.CommitRound = -1
  444. cs.LastCommit = lastPrecommits
  445. cs.LastValidators = state.LastValidators
  446. cs.state = state
  447. // Finally, broadcast RoundState
  448. cs.newStep()
  449. }
  450. func (cs *ConsensusState) newStep() {
  451. rs := cs.RoundStateEvent()
  452. cs.wal.Save(rs)
  453. cs.nSteps += 1
  454. // newStep is called by updateToStep in NewConsensusState before the evsw is set!
  455. if cs.evsw != nil {
  456. cs.evsw.FireEvent(types.EventStringNewRoundStep(), rs)
  457. }
  458. }
  459. //-----------------------------------------
  460. // the main go routines
  461. // the state machine sends on tickChan to start a new timer.
  462. // timers are interupted and replaced by new ticks from later steps
  463. // timeouts of 0 on the tickChan will be immediately relayed to the tockChan
  464. func (cs *ConsensusState) timeoutRoutine() {
  465. log.Debug("Starting timeout routine")
  466. var ti timeoutInfo
  467. for {
  468. select {
  469. case newti := <-cs.tickChan:
  470. log.Debug("Received tick", "old_ti", ti, "new_ti", newti)
  471. // ignore tickers for old height/round/step
  472. if newti.Height < ti.Height {
  473. continue
  474. } else if newti.Height == ti.Height {
  475. if newti.Round < ti.Round {
  476. continue
  477. } else if newti.Round == ti.Round {
  478. if ti.Step > 0 && newti.Step <= ti.Step {
  479. continue
  480. }
  481. }
  482. }
  483. ti = newti
  484. // if the newti has duration == 0, we relay to the tockChan immediately (no timeout)
  485. if ti.Duration == time.Duration(0) {
  486. go func(t timeoutInfo) { cs.tockChan <- t }(ti)
  487. continue
  488. }
  489. log.Debug("Scheduling timeout", "dur", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step)
  490. cs.timeoutTicker.Stop()
  491. cs.timeoutTicker = time.NewTicker(ti.Duration)
  492. case <-cs.timeoutTicker.C:
  493. log.Info("Timed out", "dur", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step)
  494. cs.timeoutTicker.Stop()
  495. // go routine here gaurantees timeoutRoutine doesn't block.
  496. // Determinism comes from playback in the receiveRoutine.
  497. // We can eliminate it by merging the timeoutRoutine into receiveRoutine
  498. // and managing the timeouts ourselves with a millisecond ticker
  499. go func(t timeoutInfo) { cs.tockChan <- t }(ti)
  500. case <-cs.Quit:
  501. return
  502. }
  503. }
  504. }
  505. // a nice idea but probably more trouble than its worth
  506. func (cs *ConsensusState) stopTimer() {
  507. cs.timeoutTicker.Stop()
  508. }
  509. // receiveRoutine handles messages which may cause state transitions.
  510. // it's argument (n) is the number of messages to process before exiting - use 0 to run forever
  511. // It keeps the RoundState and is the only thing that updates it.
  512. // Updates (state transitions) happen on timeouts, complete proposals, and 2/3 majorities
  513. func (cs *ConsensusState) receiveRoutine(maxSteps int) {
  514. for {
  515. if maxSteps > 0 {
  516. if cs.nSteps >= maxSteps {
  517. log.Warn("reached max steps. exiting receive routine")
  518. cs.nSteps = 0
  519. return
  520. }
  521. }
  522. rs := cs.RoundState
  523. var mi msgInfo
  524. select {
  525. case mi = <-cs.peerMsgQueue:
  526. cs.wal.Save(mi)
  527. // handles proposals, block parts, votes
  528. // may generate internal events (votes, complete proposals, 2/3 majorities)
  529. cs.handleMsg(mi, rs)
  530. case mi = <-cs.internalMsgQueue:
  531. cs.wal.Save(mi)
  532. // handles proposals, block parts, votes
  533. cs.handleMsg(mi, rs)
  534. case ti := <-cs.tockChan:
  535. cs.wal.Save(ti)
  536. // if the timeout is relevant to the rs
  537. // go to the next step
  538. cs.handleTimeout(ti, rs)
  539. case <-cs.Quit:
  540. // drain the internalMsgQueue in case we eg. signed a proposal but it didn't hit the wal
  541. FLUSH:
  542. for {
  543. select {
  544. case mi = <-cs.internalMsgQueue:
  545. cs.wal.Save(mi)
  546. cs.handleMsg(mi, rs)
  547. default:
  548. break FLUSH
  549. }
  550. }
  551. // close wal now that we're done writing to it
  552. if cs.wal != nil {
  553. cs.wal.Close()
  554. }
  555. return
  556. }
  557. }
  558. }
  559. // state transitions on complete-proposal, 2/3-any, 2/3-one
  560. func (cs *ConsensusState) handleMsg(mi msgInfo, rs RoundState) {
  561. cs.mtx.Lock()
  562. defer cs.mtx.Unlock()
  563. var err error
  564. msg, peerKey := mi.Msg, mi.PeerKey
  565. switch msg := msg.(type) {
  566. case *ProposalMessage:
  567. // will not cause transition.
  568. // once proposal is set, we can receive block parts
  569. err = cs.setProposal(msg.Proposal)
  570. case *BlockPartMessage:
  571. // if the proposal is complete, we'll enterPrevote or tryFinalizeCommit
  572. _, err = cs.addProposalBlockPart(msg.Height, msg.Part)
  573. if err != nil && msg.Round != cs.Round {
  574. err = nil
  575. }
  576. case *VoteMessage:
  577. // attempt to add the vote and dupeout the validator if its a duplicate signature
  578. // if the vote gives us a 2/3-any or 2/3-one, we transition
  579. err := cs.tryAddVote(msg.ValidatorIndex, msg.Vote, peerKey)
  580. if err == ErrAddingVote {
  581. // TODO: punish peer
  582. }
  583. // NOTE: the vote is broadcast to peers by the reactor listening
  584. // for vote events
  585. // TODO: If rs.Height == vote.Height && rs.Round < vote.Round,
  586. // the peer is sending us CatchupCommit precommits.
  587. // We could make note of this and help filter in broadcastHasVoteMessage().
  588. default:
  589. log.Warn("Unknown msg type", reflect.TypeOf(msg))
  590. }
  591. if err != nil {
  592. log.Error("Error with msg", "type", reflect.TypeOf(msg), "peer", peerKey, "error", err, "msg", msg)
  593. }
  594. }
  595. func (cs *ConsensusState) handleTimeout(ti timeoutInfo, rs RoundState) {
  596. log.Debug("Received tock", "timeout", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step)
  597. // timeouts must be for current height, round, step
  598. if ti.Height != rs.Height || ti.Round < rs.Round || (ti.Round == rs.Round && ti.Step < rs.Step) {
  599. log.Debug("Ignoring tock because we're ahead", "height", rs.Height, "round", rs.Round, "step", rs.Step)
  600. return
  601. }
  602. // the timeout will now cause a state transition
  603. cs.mtx.Lock()
  604. defer cs.mtx.Unlock()
  605. switch ti.Step {
  606. case RoundStepNewHeight:
  607. // NewRound event fired from enterNewRound.
  608. // XXX: should we fire timeout here?
  609. cs.enterNewRound(ti.Height, 0)
  610. case RoundStepPropose:
  611. cs.evsw.FireEvent(types.EventStringTimeoutPropose(), cs.RoundStateEvent())
  612. cs.enterPrevote(ti.Height, ti.Round)
  613. case RoundStepPrevoteWait:
  614. cs.evsw.FireEvent(types.EventStringTimeoutWait(), cs.RoundStateEvent())
  615. cs.enterPrecommit(ti.Height, ti.Round)
  616. case RoundStepPrecommitWait:
  617. cs.evsw.FireEvent(types.EventStringTimeoutWait(), cs.RoundStateEvent())
  618. cs.enterNewRound(ti.Height, ti.Round+1)
  619. default:
  620. panic(Fmt("Invalid timeout step: %v", ti.Step))
  621. }
  622. }
  623. //-----------------------------------------------------------------------------
  624. // State functions
  625. // Used internally by handleTimeout and handleMsg to make state transitions
  626. // Enter: +2/3 precommits for nil at (height,round-1)
  627. // Enter: `timeoutPrecommits` after any +2/3 precommits from (height,round-1)
  628. // Enter: `startTime = commitTime+timeoutCommit` from NewHeight(height)
  629. // NOTE: cs.StartTime was already set for height.
  630. func (cs *ConsensusState) enterNewRound(height int, round int) {
  631. if cs.Height != height || round < cs.Round || (cs.Round == round && cs.Step != RoundStepNewHeight) {
  632. log.Debug(Fmt("enterNewRound(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  633. return
  634. }
  635. if now := time.Now(); cs.StartTime.After(now) {
  636. log.Warn("Need to set a buffer and log.Warn() here for sanity.", "startTime", cs.StartTime, "now", now)
  637. }
  638. // cs.stopTimer()
  639. log.Notice(Fmt("enterNewRound(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  640. // Increment validators if necessary
  641. validators := cs.Validators
  642. if cs.Round < round {
  643. validators = validators.Copy()
  644. validators.IncrementAccum(round - cs.Round)
  645. }
  646. // Setup new round
  647. // we don't fire newStep for this step,
  648. // but we fire an event, so update the round step first
  649. cs.updateRoundStep(round, RoundStepNewRound)
  650. cs.Validators = validators
  651. if round == 0 {
  652. // We've already reset these upon new height,
  653. // and meanwhile we might have received a proposal
  654. // for round 0.
  655. } else {
  656. cs.Proposal = nil
  657. cs.ProposalBlock = nil
  658. cs.ProposalBlockParts = nil
  659. }
  660. cs.Votes.SetRound(round + 1) // also track next round (round+1) to allow round-skipping
  661. cs.evsw.FireEvent(types.EventStringNewRound(), cs.RoundStateEvent())
  662. // Immediately go to enterPropose.
  663. cs.enterPropose(height, round)
  664. }
  665. // Enter: from NewRound(height,round).
  666. func (cs *ConsensusState) enterPropose(height int, round int) {
  667. if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPropose <= cs.Step) {
  668. log.Debug(Fmt("enterPropose(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  669. return
  670. }
  671. log.Info(Fmt("enterPropose(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  672. defer func() {
  673. // Done enterPropose:
  674. cs.updateRoundStep(round, RoundStepPropose)
  675. cs.newStep()
  676. // If we have the whole proposal + POL, then goto Prevote now.
  677. // else, we'll enterPrevote when the rest of the proposal is received (in AddProposalBlockPart),
  678. // or else after timeoutPropose
  679. if cs.isProposalComplete() {
  680. cs.enterPrevote(height, cs.Round)
  681. }
  682. }()
  683. // If we don't get the proposal and all block parts quick enough, enterPrevote
  684. cs.scheduleTimeout(cs.timeoutParams.Propose(round), height, round, RoundStepPropose)
  685. // Nothing more to do if we're not a validator
  686. if cs.privValidator == nil {
  687. return
  688. }
  689. if !bytes.Equal(cs.Validators.Proposer().Address, cs.privValidator.Address) {
  690. log.Info("enterPropose: Not our turn to propose", "proposer", cs.Validators.Proposer().Address, "privValidator", cs.privValidator)
  691. } else {
  692. log.Info("enterPropose: Our turn to propose", "proposer", cs.Validators.Proposer().Address, "privValidator", cs.privValidator)
  693. cs.decideProposal(height, round)
  694. }
  695. }
  696. func (cs *ConsensusState) decideProposal(height, round int) {
  697. var block *types.Block
  698. var blockParts *types.PartSet
  699. // Decide on block
  700. if cs.LockedBlock != nil {
  701. // If we're locked onto a block, just choose that.
  702. block, blockParts = cs.LockedBlock, cs.LockedBlockParts
  703. } else {
  704. // Create a new proposal block from state/txs from the mempool.
  705. block, blockParts = cs.createProposalBlock()
  706. if block == nil { // on error
  707. return
  708. }
  709. }
  710. // Make proposal
  711. proposal := types.NewProposal(height, round, blockParts.Header(), cs.Votes.POLRound())
  712. err := cs.privValidator.SignProposal(cs.state.ChainID, proposal)
  713. if err == nil {
  714. // Set fields
  715. /* fields set by setProposal and addBlockPart
  716. cs.Proposal = proposal
  717. cs.ProposalBlock = block
  718. cs.ProposalBlockParts = blockParts
  719. */
  720. // send proposal and block parts on internal msg queue
  721. cs.sendInternalMessage(msgInfo{&ProposalMessage{proposal}, ""})
  722. for i := 0; i < blockParts.Total(); i++ {
  723. part := blockParts.GetPart(i)
  724. cs.sendInternalMessage(msgInfo{&BlockPartMessage{cs.Height, cs.Round, part}, ""})
  725. }
  726. log.Info("Signed and sent proposal", "height", height, "round", round, "proposal", proposal)
  727. log.Debug(Fmt("Signed and sent proposal block: %v", block))
  728. } else {
  729. log.Warn("enterPropose: Error signing proposal", "height", height, "round", round, "error", err)
  730. }
  731. }
  732. // Returns true if the proposal block is complete &&
  733. // (if POLRound was proposed, we have +2/3 prevotes from there).
  734. func (cs *ConsensusState) isProposalComplete() bool {
  735. if cs.Proposal == nil || cs.ProposalBlock == nil {
  736. return false
  737. }
  738. // we have the proposal. if there's a POLRound,
  739. // make sure we have the prevotes from it too
  740. if cs.Proposal.POLRound < 0 {
  741. return true
  742. } else {
  743. // if this is false the proposer is lying or we haven't received the POL yet
  744. return cs.Votes.Prevotes(cs.Proposal.POLRound).HasTwoThirdsMajority()
  745. }
  746. }
  747. // Create the next block to propose and return it.
  748. // Returns nil block upon error.
  749. // NOTE: keep it side-effect free for clarity.
  750. func (cs *ConsensusState) createProposalBlock() (block *types.Block, blockParts *types.PartSet) {
  751. var validation *types.Validation
  752. if cs.Height == 1 {
  753. // We're creating a proposal for the first block.
  754. // The validation is empty, but not nil.
  755. validation = &types.Validation{}
  756. } else if cs.LastCommit.HasTwoThirdsMajority() {
  757. // Make the validation from LastCommit
  758. validation = cs.LastCommit.MakeValidation()
  759. } else {
  760. // This shouldn't happen.
  761. log.Error("enterPropose: Cannot propose anything: No validation for the previous block.")
  762. return
  763. }
  764. // Mempool validated transactions
  765. txs := cs.mempool.Reap(config.GetInt("block_size"))
  766. block = &types.Block{
  767. Header: &types.Header{
  768. ChainID: cs.state.ChainID,
  769. Height: cs.Height,
  770. Time: time.Now(),
  771. Fees: 0, // TODO fees
  772. NumTxs: len(txs),
  773. LastBlockHash: cs.state.LastBlockHash,
  774. LastBlockParts: cs.state.LastBlockParts,
  775. ValidatorsHash: cs.state.Validators.Hash(),
  776. AppHash: cs.state.AppHash, // state merkle root of txs from the previous block.
  777. },
  778. LastValidation: validation,
  779. Data: &types.Data{
  780. Txs: txs,
  781. },
  782. }
  783. block.FillHeader()
  784. blockParts = block.MakePartSet()
  785. return block, blockParts
  786. }
  787. // Enter: `timeoutPropose` after entering Propose.
  788. // Enter: proposal block and POL is ready.
  789. // Enter: any +2/3 prevotes for future round.
  790. // Prevote for LockedBlock if we're locked, or ProposalBlock if valid.
  791. // Otherwise vote nil.
  792. func (cs *ConsensusState) enterPrevote(height int, round int) {
  793. if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPrevote <= cs.Step) {
  794. log.Debug(Fmt("enterPrevote(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  795. return
  796. }
  797. defer func() {
  798. // Done enterPrevote:
  799. cs.updateRoundStep(round, RoundStepPrevote)
  800. cs.newStep()
  801. }()
  802. // fire event for how we got here
  803. if cs.isProposalComplete() {
  804. cs.evsw.FireEvent(types.EventStringCompleteProposal(), cs.RoundStateEvent())
  805. } else {
  806. // we received +2/3 prevotes for a future round
  807. // TODO: catchup event?
  808. }
  809. // cs.stopTimer()
  810. log.Info(Fmt("enterPrevote(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  811. // Sign and broadcast vote as necessary
  812. cs.doPrevote(height, round)
  813. // Once `addVote` hits any +2/3 prevotes, we will go to PrevoteWait
  814. // (so we have more time to try and collect +2/3 prevotes for a single block)
  815. }
  816. func (cs *ConsensusState) doPrevote(height int, round int) {
  817. // If a block is locked, prevote that.
  818. if cs.LockedBlock != nil {
  819. log.Info("enterPrevote: Block was locked")
  820. cs.signAddVote(types.VoteTypePrevote, cs.LockedBlock.Hash(), cs.LockedBlockParts.Header())
  821. return
  822. }
  823. // If ProposalBlock is nil, prevote nil.
  824. if cs.ProposalBlock == nil {
  825. log.Warn("enterPrevote: ProposalBlock is nil")
  826. cs.signAddVote(types.VoteTypePrevote, nil, types.PartSetHeader{})
  827. return
  828. }
  829. // Valdiate proposal block
  830. err := cs.state.ValidateBlock(cs.ProposalBlock)
  831. if err != nil {
  832. // ProposalBlock is invalid, prevote nil.
  833. log.Warn("enterPrevote: ProposalBlock is invalid", "error", err)
  834. cs.signAddVote(types.VoteTypePrevote, nil, types.PartSetHeader{})
  835. return
  836. }
  837. // Prevote cs.ProposalBlock
  838. // NOTE: the proposal signature is validated when it is received,
  839. // and the proposal block parts are validated as they are received (against the merkle hash in the proposal)
  840. cs.signAddVote(types.VoteTypePrevote, cs.ProposalBlock.Hash(), cs.ProposalBlockParts.Header())
  841. return
  842. }
  843. // Enter: any +2/3 prevotes at next round.
  844. func (cs *ConsensusState) enterPrevoteWait(height int, round int) {
  845. if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPrevoteWait <= cs.Step) {
  846. log.Debug(Fmt("enterPrevoteWait(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  847. return
  848. }
  849. if !cs.Votes.Prevotes(round).HasTwoThirdsAny() {
  850. PanicSanity(Fmt("enterPrevoteWait(%v/%v), but Prevotes does not have any +2/3 votes", height, round))
  851. }
  852. log.Info(Fmt("enterPrevoteWait(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  853. defer func() {
  854. // Done enterPrevoteWait:
  855. cs.updateRoundStep(round, RoundStepPrevoteWait)
  856. cs.newStep()
  857. }()
  858. // Wait for some more prevotes; enterPrecommit
  859. cs.scheduleTimeout(cs.timeoutParams.Prevote(round), height, round, RoundStepPrevoteWait)
  860. }
  861. // Enter: +2/3 precomits for block or nil.
  862. // Enter: `timeoutPrevote` after any +2/3 prevotes.
  863. // Enter: any +2/3 precommits for next round.
  864. // Lock & precommit the ProposalBlock if we have enough prevotes for it (a POL in this round)
  865. // else, unlock an existing lock and precommit nil if +2/3 of prevotes were nil,
  866. // else, precommit nil otherwise.
  867. func (cs *ConsensusState) enterPrecommit(height int, round int) {
  868. if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPrecommit <= cs.Step) {
  869. log.Debug(Fmt("enterPrecommit(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  870. return
  871. }
  872. // cs.stopTimer()
  873. log.Info(Fmt("enterPrecommit(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  874. defer func() {
  875. // Done enterPrecommit:
  876. cs.updateRoundStep(round, RoundStepPrecommit)
  877. cs.newStep()
  878. }()
  879. hash, partsHeader, ok := cs.Votes.Prevotes(round).TwoThirdsMajority()
  880. // If we don't have a polka, we must precommit nil
  881. if !ok {
  882. if cs.LockedBlock != nil {
  883. log.Info("enterPrecommit: No +2/3 prevotes during enterPrecommit while we're locked. Precommitting nil")
  884. } else {
  885. log.Info("enterPrecommit: No +2/3 prevotes during enterPrecommit. Precommitting nil.")
  886. }
  887. cs.signAddVote(types.VoteTypePrecommit, nil, types.PartSetHeader{})
  888. return
  889. }
  890. // At this point +2/3 prevoted for a particular block or nil
  891. cs.evsw.FireEvent(types.EventStringPolka(), cs.RoundStateEvent())
  892. // the latest POLRound should be this round
  893. if cs.Votes.POLRound() < round {
  894. PanicSanity(Fmt("This POLRound should be %v but got %", round, cs.Votes.POLRound()))
  895. }
  896. // +2/3 prevoted nil. Unlock and precommit nil.
  897. if len(hash) == 0 {
  898. if cs.LockedBlock == nil {
  899. log.Notice("enterPrecommit: +2/3 prevoted for nil.")
  900. } else {
  901. log.Notice("enterPrecommit: +2/3 prevoted for nil. Unlocking")
  902. cs.LockedRound = 0
  903. cs.LockedBlock = nil
  904. cs.LockedBlockParts = nil
  905. cs.evsw.FireEvent(types.EventStringUnlock(), cs.RoundStateEvent())
  906. }
  907. cs.signAddVote(types.VoteTypePrecommit, nil, types.PartSetHeader{})
  908. return
  909. }
  910. // At this point, +2/3 prevoted for a particular block.
  911. // If we're already locked on that block, precommit it, and update the LockedRound
  912. if cs.LockedBlock.HashesTo(hash) {
  913. log.Notice("enterPrecommit: +2/3 prevoted locked block. Relocking")
  914. cs.LockedRound = round
  915. cs.evsw.FireEvent(types.EventStringRelock(), cs.RoundStateEvent())
  916. cs.signAddVote(types.VoteTypePrecommit, hash, partsHeader)
  917. return
  918. }
  919. // If +2/3 prevoted for proposal block, stage and precommit it
  920. if cs.ProposalBlock.HashesTo(hash) {
  921. log.Notice("enterPrecommit: +2/3 prevoted proposal block. Locking", "hash", hash)
  922. // Validate the block.
  923. if err := cs.state.ValidateBlock(cs.ProposalBlock); err != nil {
  924. PanicConsensus(Fmt("enterPrecommit: +2/3 prevoted for an invalid block: %v", err))
  925. }
  926. cs.LockedRound = round
  927. cs.LockedBlock = cs.ProposalBlock
  928. cs.LockedBlockParts = cs.ProposalBlockParts
  929. cs.evsw.FireEvent(types.EventStringLock(), cs.RoundStateEvent())
  930. cs.signAddVote(types.VoteTypePrecommit, hash, partsHeader)
  931. return
  932. }
  933. // There was a polka in this round for a block we don't have.
  934. // Fetch that block, unlock, and precommit nil.
  935. // The +2/3 prevotes for this round is the POL for our unlock.
  936. // TODO: In the future save the POL prevotes for justification.
  937. cs.LockedRound = 0
  938. cs.LockedBlock = nil
  939. cs.LockedBlockParts = nil
  940. if !cs.ProposalBlockParts.HasHeader(partsHeader) {
  941. cs.ProposalBlock = nil
  942. cs.ProposalBlockParts = types.NewPartSetFromHeader(partsHeader)
  943. }
  944. cs.evsw.FireEvent(types.EventStringUnlock(), cs.RoundStateEvent())
  945. cs.signAddVote(types.VoteTypePrecommit, nil, types.PartSetHeader{})
  946. return
  947. }
  948. // Enter: any +2/3 precommits for next round.
  949. func (cs *ConsensusState) enterPrecommitWait(height int, round int) {
  950. if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPrecommitWait <= cs.Step) {
  951. log.Debug(Fmt("enterPrecommitWait(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  952. return
  953. }
  954. if !cs.Votes.Precommits(round).HasTwoThirdsAny() {
  955. PanicSanity(Fmt("enterPrecommitWait(%v/%v), but Precommits does not have any +2/3 votes", height, round))
  956. }
  957. log.Info(Fmt("enterPrecommitWait(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  958. defer func() {
  959. // Done enterPrecommitWait:
  960. cs.updateRoundStep(round, RoundStepPrecommitWait)
  961. cs.newStep()
  962. }()
  963. // Wait for some more precommits; enterNewRound
  964. cs.scheduleTimeout(cs.timeoutParams.Precommit(round), height, round, RoundStepPrecommitWait)
  965. }
  966. // Enter: +2/3 precommits for block
  967. func (cs *ConsensusState) enterCommit(height int, commitRound int) {
  968. if cs.Height != height || RoundStepCommit <= cs.Step {
  969. log.Debug(Fmt("enterCommit(%v/%v): Invalid args. Current step: %v/%v/%v", height, commitRound, cs.Height, cs.Round, cs.Step))
  970. return
  971. }
  972. log.Info(Fmt("enterCommit(%v/%v). Current: %v/%v/%v", height, commitRound, cs.Height, cs.Round, cs.Step))
  973. defer func() {
  974. // Done enterCommit:
  975. // keep cs.Round the same, commitRound points to the right Precommits set.
  976. cs.updateRoundStep(cs.Round, RoundStepCommit)
  977. cs.CommitRound = commitRound
  978. cs.newStep()
  979. // Maybe finalize immediately.
  980. cs.tryFinalizeCommit(height)
  981. }()
  982. hash, partsHeader, ok := cs.Votes.Precommits(commitRound).TwoThirdsMajority()
  983. if !ok {
  984. PanicSanity("RunActionCommit() expects +2/3 precommits")
  985. }
  986. // The Locked* fields no longer matter.
  987. // Move them over to ProposalBlock if they match the commit hash,
  988. // otherwise they'll be cleared in updateToState.
  989. if cs.LockedBlock.HashesTo(hash) {
  990. cs.ProposalBlock = cs.LockedBlock
  991. cs.ProposalBlockParts = cs.LockedBlockParts
  992. }
  993. // If we don't have the block being committed, set up to get it.
  994. if !cs.ProposalBlock.HashesTo(hash) {
  995. if !cs.ProposalBlockParts.HasHeader(partsHeader) {
  996. // We're getting the wrong block.
  997. // Set up ProposalBlockParts and keep waiting.
  998. cs.ProposalBlock = nil
  999. cs.ProposalBlockParts = types.NewPartSetFromHeader(partsHeader)
  1000. } else {
  1001. // We just need to keep waiting.
  1002. }
  1003. }
  1004. }
  1005. // If we have the block AND +2/3 commits for it, finalize.
  1006. func (cs *ConsensusState) tryFinalizeCommit(height int) {
  1007. if cs.Height != height {
  1008. PanicSanity(Fmt("tryFinalizeCommit() cs.Height: %v vs height: %v", cs.Height, height))
  1009. }
  1010. hash, _, ok := cs.Votes.Precommits(cs.CommitRound).TwoThirdsMajority()
  1011. if !ok || len(hash) == 0 {
  1012. log.Warn("Attempt to finalize failed. There was no +2/3 majority, or +2/3 was for <nil>.")
  1013. return
  1014. }
  1015. if !cs.ProposalBlock.HashesTo(hash) {
  1016. // TODO: this happens every time if we're not a validator (ugly logs)
  1017. log.Warn("Attempt to finalize failed. We don't have the commit block.")
  1018. return
  1019. }
  1020. // go
  1021. cs.finalizeCommit(height)
  1022. }
  1023. // Increment height and goto RoundStepNewHeight
  1024. func (cs *ConsensusState) finalizeCommit(height int) {
  1025. if cs.Height != height || cs.Step != RoundStepCommit {
  1026. log.Debug(Fmt("finalizeCommit(%v): Invalid args. Current step: %v/%v/%v", height, cs.Height, cs.Round, cs.Step))
  1027. return
  1028. }
  1029. hash, header, ok := cs.Votes.Precommits(cs.CommitRound).TwoThirdsMajority()
  1030. block, blockParts := cs.ProposalBlock, cs.ProposalBlockParts
  1031. if !ok {
  1032. PanicSanity(Fmt("Cannot finalizeCommit, commit does not have two thirds majority"))
  1033. }
  1034. if !blockParts.HasHeader(header) {
  1035. PanicSanity(Fmt("Expected ProposalBlockParts header to be commit header"))
  1036. }
  1037. if !block.HashesTo(hash) {
  1038. PanicSanity(Fmt("Cannot finalizeCommit, ProposalBlock does not hash to commit hash"))
  1039. }
  1040. if err := cs.state.ValidateBlock(block); err != nil {
  1041. PanicConsensus(Fmt("+2/3 committed an invalid block: %v", err))
  1042. }
  1043. log.Notice(Fmt("Finalizing commit of block with %d txs", block.NumTxs), "height", block.Height, "hash", block.Hash())
  1044. log.Info(Fmt("%v", block))
  1045. // Fire off event for new block.
  1046. // TODO: Handle app failure. See #177
  1047. cs.evsw.FireEvent(types.EventStringNewBlock(), types.EventDataNewBlock{block})
  1048. // Create a copy of the state for staging
  1049. stateCopy := cs.state.Copy()
  1050. // Run the block on the State:
  1051. // + update validator sets
  1052. // + run txs on the proxyAppConn
  1053. err := stateCopy.ExecBlock(cs.evsw, cs.proxyAppConn, block, blockParts.Header())
  1054. if err != nil {
  1055. // TODO: handle this gracefully.
  1056. PanicQ(Fmt("Exec failed for application"))
  1057. }
  1058. // Save to blockStore.
  1059. if cs.blockStore.Height() < block.Height {
  1060. commits := cs.Votes.Precommits(cs.CommitRound)
  1061. seenValidation := commits.MakeValidation()
  1062. cs.blockStore.SaveBlock(block, blockParts, seenValidation)
  1063. }
  1064. /*
  1065. // Commit to proxyAppConn
  1066. err = cs.proxyAppConn.CommitSync()
  1067. if err != nil {
  1068. // TODO: handle this gracefully.
  1069. PanicQ(Fmt("Commit failed for application"))
  1070. }
  1071. */
  1072. // Save the state.
  1073. stateCopy.Save()
  1074. // Update mempool.
  1075. cs.mempool.Update(block.Height, block.Txs)
  1076. // NewHeightStep!
  1077. cs.updateToState(stateCopy)
  1078. // cs.StartTime is already set.
  1079. // Schedule Round0 to start soon.
  1080. cs.scheduleRound0(height + 1)
  1081. // By here,
  1082. // * cs.Height has been increment to height+1
  1083. // * cs.Step is now RoundStepNewHeight
  1084. // * cs.StartTime is set to when we will start round0.
  1085. return
  1086. }
  1087. //-----------------------------------------------------------------------------
  1088. func (cs *ConsensusState) setProposal(proposal *types.Proposal) error {
  1089. // Already have one
  1090. if cs.Proposal != nil {
  1091. return nil
  1092. }
  1093. // Does not apply
  1094. if proposal.Height != cs.Height || proposal.Round != cs.Round {
  1095. return nil
  1096. }
  1097. // We don't care about the proposal if we're already in RoundStepCommit.
  1098. if RoundStepCommit <= cs.Step {
  1099. return nil
  1100. }
  1101. // Verify POLRound, which must be -1 or between 0 and proposal.Round exclusive.
  1102. if proposal.POLRound != -1 &&
  1103. (proposal.POLRound < 0 || proposal.Round <= proposal.POLRound) {
  1104. return ErrInvalidProposalPOLRound
  1105. }
  1106. // Verify signature
  1107. if !cs.Validators.Proposer().PubKey.VerifyBytes(types.SignBytes(cs.state.ChainID, proposal), proposal.Signature) {
  1108. return ErrInvalidProposalSignature
  1109. }
  1110. cs.Proposal = proposal
  1111. cs.ProposalBlockParts = types.NewPartSetFromHeader(proposal.BlockPartsHeader)
  1112. return nil
  1113. }
  1114. // NOTE: block is not necessarily valid.
  1115. // Asynchronously triggers either enterPrevote (before we timeout of propose) or tryFinalizeCommit, once we have the full block.
  1116. func (cs *ConsensusState) addProposalBlockPart(height int, part *types.Part) (added bool, err error) {
  1117. // Blocks might be reused, so round mismatch is OK
  1118. if cs.Height != height {
  1119. return false, nil
  1120. }
  1121. // We're not expecting a block part.
  1122. if cs.ProposalBlockParts == nil {
  1123. return false, nil // TODO: bad peer? Return error?
  1124. }
  1125. added, err = cs.ProposalBlockParts.AddPart(part)
  1126. if err != nil {
  1127. return added, err
  1128. }
  1129. if added && cs.ProposalBlockParts.IsComplete() {
  1130. // Added and completed!
  1131. var n int
  1132. var err error
  1133. cs.ProposalBlock = wire.ReadBinary(&types.Block{}, cs.ProposalBlockParts.GetReader(), types.MaxBlockSize, &n, &err).(*types.Block)
  1134. // NOTE: it's possible to receive complete proposal blocks for future rounds without having the proposal
  1135. log.Info("Received complete proposal block", "height", cs.ProposalBlock.Height, "hash", cs.ProposalBlock.Hash())
  1136. if cs.Step == RoundStepPropose && cs.isProposalComplete() {
  1137. // Move onto the next step
  1138. cs.enterPrevote(height, cs.Round)
  1139. } else if cs.Step == RoundStepCommit {
  1140. // If we're waiting on the proposal block...
  1141. cs.tryFinalizeCommit(height)
  1142. }
  1143. return true, err
  1144. }
  1145. return added, nil
  1146. }
  1147. // Attempt to add the vote. if its a duplicate signature, dupeout the validator
  1148. func (cs *ConsensusState) tryAddVote(valIndex int, vote *types.Vote, peerKey string) error {
  1149. _, _, err := cs.addVote(valIndex, vote, peerKey)
  1150. if err != nil {
  1151. // If the vote height is off, we'll just ignore it,
  1152. // But if it's a conflicting sig, broadcast evidence tx for slashing.
  1153. // If it's otherwise invalid, punish peer.
  1154. if err == ErrVoteHeightMismatch {
  1155. return err
  1156. } else if _, ok := err.(*types.ErrVoteConflictingSignature); ok {
  1157. if peerKey == "" {
  1158. log.Warn("Found conflicting vote from ourselves. Did you unsafe_reset a validator?", "height", vote.Height, "round", vote.Round, "type", vote.Type)
  1159. return err
  1160. }
  1161. log.Warn("Found conflicting vote. Publish evidence (TODO)")
  1162. /* TODO
  1163. evidenceTx := &types.DupeoutTx{
  1164. Address: address,
  1165. VoteA: *errDupe.VoteA,
  1166. VoteB: *errDupe.VoteB,
  1167. }
  1168. cs.mempool.BroadcastTx(struct{???}{evidenceTx}) // shouldn't need to check returned err
  1169. */
  1170. return err
  1171. } else {
  1172. // Probably an invalid signature. Bad peer.
  1173. log.Warn("Error attempting to add vote", "error", err)
  1174. return ErrAddingVote
  1175. }
  1176. }
  1177. return nil
  1178. }
  1179. //-----------------------------------------------------------------------------
  1180. func (cs *ConsensusState) addVote(valIndex int, vote *types.Vote, peerKey string) (added bool, address []byte, err error) {
  1181. log.Debug("addVote", "voteHeight", vote.Height, "voteType", vote.Type, "csHeight", cs.Height)
  1182. // A precommit for the previous height?
  1183. if vote.Height+1 == cs.Height {
  1184. if !(cs.Step == RoundStepNewHeight && vote.Type == types.VoteTypePrecommit) {
  1185. // TODO: give the reason ..
  1186. // fmt.Errorf("tryAddVote: Wrong height, not a LastCommit straggler commit.")
  1187. return added, nil, ErrVoteHeightMismatch
  1188. }
  1189. added, address, err = cs.LastCommit.AddByIndex(valIndex, vote)
  1190. if added {
  1191. log.Info(Fmt("Added to lastPrecommits: %v", cs.LastCommit.StringShort()))
  1192. cs.evsw.FireEvent(types.EventStringVote(), types.EventDataVote{valIndex, address, vote})
  1193. }
  1194. return
  1195. }
  1196. // A prevote/precommit for this height?
  1197. if vote.Height == cs.Height {
  1198. height := cs.Height
  1199. added, address, err = cs.Votes.AddByIndex(valIndex, vote, peerKey)
  1200. if added {
  1201. cs.evsw.FireEvent(types.EventStringVote(), types.EventDataVote{valIndex, address, vote})
  1202. switch vote.Type {
  1203. case types.VoteTypePrevote:
  1204. prevotes := cs.Votes.Prevotes(vote.Round)
  1205. log.Info("Added to prevote", "vote", vote, "prevotes", prevotes.StringShort())
  1206. // First, unlock if prevotes is a valid POL.
  1207. // >> lockRound < POLRound <= unlockOrChangeLockRound (see spec)
  1208. // NOTE: If (lockRound < POLRound) but !(POLRound <= unlockOrChangeLockRound),
  1209. // we'll still enterNewRound(H,vote.R) and enterPrecommit(H,vote.R) to process it
  1210. // there.
  1211. if (cs.LockedBlock != nil) && (cs.LockedRound < vote.Round) && (vote.Round <= cs.Round) {
  1212. hash, _, ok := prevotes.TwoThirdsMajority()
  1213. if ok && !cs.LockedBlock.HashesTo(hash) {
  1214. log.Notice("Unlocking because of POL.", "lockedRound", cs.LockedRound, "POLRound", vote.Round)
  1215. cs.LockedRound = 0
  1216. cs.LockedBlock = nil
  1217. cs.LockedBlockParts = nil
  1218. cs.evsw.FireEvent(types.EventStringUnlock(), cs.RoundStateEvent())
  1219. }
  1220. }
  1221. if cs.Round <= vote.Round && prevotes.HasTwoThirdsAny() {
  1222. // Round-skip over to PrevoteWait or goto Precommit.
  1223. cs.enterNewRound(height, vote.Round) // if the vote is ahead of us
  1224. if prevotes.HasTwoThirdsMajority() {
  1225. cs.enterPrecommit(height, vote.Round)
  1226. } else {
  1227. cs.enterPrevote(height, vote.Round) // if the vote is ahead of us
  1228. cs.enterPrevoteWait(height, vote.Round)
  1229. }
  1230. } else if cs.Proposal != nil && 0 <= cs.Proposal.POLRound && cs.Proposal.POLRound == vote.Round {
  1231. // If the proposal is now complete, enter prevote of cs.Round.
  1232. if cs.isProposalComplete() {
  1233. cs.enterPrevote(height, cs.Round)
  1234. }
  1235. }
  1236. case types.VoteTypePrecommit:
  1237. precommits := cs.Votes.Precommits(vote.Round)
  1238. log.Info("Added to precommit", "vote", vote, "precommits", precommits.StringShort())
  1239. hash, _, ok := precommits.TwoThirdsMajority()
  1240. if ok {
  1241. if len(hash) == 0 {
  1242. cs.enterNewRound(height, vote.Round+1)
  1243. } else {
  1244. cs.enterNewRound(height, vote.Round)
  1245. cs.enterPrecommit(height, vote.Round)
  1246. cs.enterCommit(height, vote.Round)
  1247. }
  1248. } else if cs.Round <= vote.Round && precommits.HasTwoThirdsAny() {
  1249. cs.enterNewRound(height, vote.Round)
  1250. cs.enterPrecommit(height, vote.Round)
  1251. cs.enterPrecommitWait(height, vote.Round)
  1252. //}()
  1253. }
  1254. default:
  1255. PanicSanity(Fmt("Unexpected vote type %X", vote.Type)) // Should not happen.
  1256. }
  1257. }
  1258. // Either duplicate, or error upon cs.Votes.AddByIndex()
  1259. return
  1260. } else {
  1261. err = ErrVoteHeightMismatch
  1262. }
  1263. // Height mismatch, bad peer?
  1264. log.Info("Vote ignored and not added", "voteHeight", vote.Height, "csHeight", cs.Height)
  1265. return
  1266. }
  1267. func (cs *ConsensusState) signVote(type_ byte, hash []byte, header types.PartSetHeader) (*types.Vote, error) {
  1268. vote := &types.Vote{
  1269. Height: cs.Height,
  1270. Round: cs.Round,
  1271. Type: type_,
  1272. BlockHash: hash,
  1273. BlockPartsHeader: header,
  1274. }
  1275. err := cs.privValidator.SignVote(cs.state.ChainID, vote)
  1276. return vote, err
  1277. }
  1278. // signs the vote, publishes on internalMsgQueue
  1279. func (cs *ConsensusState) signAddVote(type_ byte, hash []byte, header types.PartSetHeader) *types.Vote {
  1280. if cs.privValidator == nil || !cs.Validators.HasAddress(cs.privValidator.Address) {
  1281. return nil
  1282. }
  1283. vote, err := cs.signVote(type_, hash, header)
  1284. if err == nil {
  1285. // TODO: store our index in the cs so we don't have to do this every time
  1286. valIndex, _ := cs.Validators.GetByAddress(cs.privValidator.Address)
  1287. cs.sendInternalMessage(msgInfo{&VoteMessage{valIndex, vote}, ""})
  1288. log.Info("Signed and pushed vote", "height", cs.Height, "round", cs.Round, "vote", vote, "error", err)
  1289. return vote
  1290. } else {
  1291. log.Warn("Error signing vote", "height", cs.Height, "round", cs.Round, "vote", vote, "error", err)
  1292. return nil
  1293. }
  1294. }
  1295. //---------------------------------------------------------
  1296. func CompareHRS(h1, r1 int, s1 RoundStepType, h2, r2 int, s2 RoundStepType) int {
  1297. if h1 < h2 {
  1298. return -1
  1299. } else if h1 > h2 {
  1300. return 1
  1301. }
  1302. if r1 < r2 {
  1303. return -1
  1304. } else if r1 > r2 {
  1305. return 1
  1306. }
  1307. if s1 < s2 {
  1308. return -1
  1309. } else if s1 > s2 {
  1310. return 1
  1311. }
  1312. return 0
  1313. }