You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1515 lines
50 KiB

10 years ago
10 years ago
10 years ago
10 years ago
9 years ago
10 years ago
10 years ago
10 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
9 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
9 years ago
9 years ago
9 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
9 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
9 years ago
10 years ago
9 years ago
9 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
9 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
8 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
9 years ago
9 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
  1. package consensus
  2. import (
  3. "bytes"
  4. "errors"
  5. "fmt"
  6. "reflect"
  7. "sync"
  8. "time"
  9. "github.com/ebuchman/fail-test"
  10. . "github.com/tendermint/go-common"
  11. cfg "github.com/tendermint/go-config"
  12. "github.com/tendermint/go-wire"
  13. bc "github.com/tendermint/tendermint/blockchain"
  14. mempl "github.com/tendermint/tendermint/mempool"
  15. "github.com/tendermint/tendermint/proxy"
  16. sm "github.com/tendermint/tendermint/state"
  17. "github.com/tendermint/tendermint/types"
  18. )
  19. //-----------------------------------------------------------------------------
  20. // Timeout Parameters
  21. // All in milliseconds
  22. type TimeoutParams struct {
  23. Propose0 int
  24. ProposeDelta int
  25. Prevote0 int
  26. PrevoteDelta int
  27. Precommit0 int
  28. PrecommitDelta int
  29. Commit0 int
  30. }
  31. // Wait this long for a proposal
  32. func (tp *TimeoutParams) Propose(round int) time.Duration {
  33. return time.Duration(tp.Propose0+tp.ProposeDelta*round) * time.Millisecond
  34. }
  35. // After receiving any +2/3 prevote, wait this long for stragglers
  36. func (tp *TimeoutParams) Prevote(round int) time.Duration {
  37. return time.Duration(tp.Prevote0+tp.PrevoteDelta*round) * time.Millisecond
  38. }
  39. // After receiving any +2/3 precommits, wait this long for stragglers
  40. func (tp *TimeoutParams) Precommit(round int) time.Duration {
  41. return time.Duration(tp.Precommit0+tp.PrecommitDelta*round) * time.Millisecond
  42. }
  43. // After receiving +2/3 precommits for a single block (a commit), wait this long for stragglers in the next height's RoundStepNewHeight
  44. func (tp *TimeoutParams) Commit(t time.Time) time.Time {
  45. return t.Add(time.Duration(tp.Commit0) * time.Millisecond)
  46. }
  47. // Initialize parameters from config
  48. func InitTimeoutParamsFromConfig(config cfg.Config) *TimeoutParams {
  49. return &TimeoutParams{
  50. Propose0: config.GetInt("timeout_propose"),
  51. ProposeDelta: config.GetInt("timeout_propose_delta"),
  52. Prevote0: config.GetInt("timeout_prevote"),
  53. PrevoteDelta: config.GetInt("timeout_prevote_delta"),
  54. Precommit0: config.GetInt("timeout_precommit"),
  55. PrecommitDelta: config.GetInt("timeout_precommit_delta"),
  56. Commit0: config.GetInt("timeout_commit"),
  57. }
  58. }
  59. //-----------------------------------------------------------------------------
  60. // Errors
  61. var (
  62. ErrInvalidProposalSignature = errors.New("Error invalid proposal signature")
  63. ErrInvalidProposalPOLRound = errors.New("Error invalid proposal POL round")
  64. ErrAddingVote = errors.New("Error adding vote")
  65. ErrVoteHeightMismatch = errors.New("Error vote height mismatch")
  66. )
  67. //-----------------------------------------------------------------------------
  68. // RoundStepType enum type
  69. type RoundStepType uint8 // These must be numeric, ordered.
  70. const (
  71. RoundStepNewHeight = RoundStepType(0x01) // Wait til CommitTime + timeoutCommit
  72. RoundStepNewRound = RoundStepType(0x02) // Setup new round and go to RoundStepPropose
  73. RoundStepPropose = RoundStepType(0x03) // Did propose, gossip proposal
  74. RoundStepPrevote = RoundStepType(0x04) // Did prevote, gossip prevotes
  75. RoundStepPrevoteWait = RoundStepType(0x05) // Did receive any +2/3 prevotes, start timeout
  76. RoundStepPrecommit = RoundStepType(0x06) // Did precommit, gossip precommits
  77. RoundStepPrecommitWait = RoundStepType(0x07) // Did receive any +2/3 precommits, start timeout
  78. RoundStepCommit = RoundStepType(0x08) // Entered commit state machine
  79. // NOTE: RoundStepNewHeight acts as RoundStepCommitWait.
  80. )
  81. func (rs RoundStepType) String() string {
  82. switch rs {
  83. case RoundStepNewHeight:
  84. return "RoundStepNewHeight"
  85. case RoundStepNewRound:
  86. return "RoundStepNewRound"
  87. case RoundStepPropose:
  88. return "RoundStepPropose"
  89. case RoundStepPrevote:
  90. return "RoundStepPrevote"
  91. case RoundStepPrevoteWait:
  92. return "RoundStepPrevoteWait"
  93. case RoundStepPrecommit:
  94. return "RoundStepPrecommit"
  95. case RoundStepPrecommitWait:
  96. return "RoundStepPrecommitWait"
  97. case RoundStepCommit:
  98. return "RoundStepCommit"
  99. default:
  100. return "RoundStepUnknown" // Cannot panic.
  101. }
  102. }
  103. //-----------------------------------------------------------------------------
  104. // Immutable when returned from ConsensusState.GetRoundState()
  105. type RoundState struct {
  106. Height int // Height we are working on
  107. Round int
  108. Step RoundStepType
  109. StartTime time.Time
  110. CommitTime time.Time // Subjective time when +2/3 precommits for Block at Round were found
  111. Validators *types.ValidatorSet
  112. Proposal *types.Proposal
  113. ProposalBlock *types.Block
  114. ProposalBlockParts *types.PartSet
  115. LockedRound int
  116. LockedBlock *types.Block
  117. LockedBlockParts *types.PartSet
  118. Votes *HeightVoteSet
  119. CommitRound int //
  120. LastCommit *types.VoteSet // Last precommits at Height-1
  121. LastValidators *types.ValidatorSet
  122. }
  123. func (rs *RoundState) RoundStateEvent() types.EventDataRoundState {
  124. edrs := types.EventDataRoundState{
  125. Height: rs.Height,
  126. Round: rs.Round,
  127. Step: rs.Step.String(),
  128. RoundState: rs,
  129. }
  130. return edrs
  131. }
  132. func (rs *RoundState) String() string {
  133. return rs.StringIndented("")
  134. }
  135. func (rs *RoundState) StringIndented(indent string) string {
  136. return fmt.Sprintf(`RoundState{
  137. %s H:%v R:%v S:%v
  138. %s StartTime: %v
  139. %s CommitTime: %v
  140. %s Validators: %v
  141. %s Proposal: %v
  142. %s ProposalBlock: %v %v
  143. %s LockedRound: %v
  144. %s LockedBlock: %v %v
  145. %s Votes: %v
  146. %s LastCommit: %v
  147. %s LastValidators: %v
  148. %s}`,
  149. indent, rs.Height, rs.Round, rs.Step,
  150. indent, rs.StartTime,
  151. indent, rs.CommitTime,
  152. indent, rs.Validators.StringIndented(indent+" "),
  153. indent, rs.Proposal,
  154. indent, rs.ProposalBlockParts.StringShort(), rs.ProposalBlock.StringShort(),
  155. indent, rs.LockedRound,
  156. indent, rs.LockedBlockParts.StringShort(), rs.LockedBlock.StringShort(),
  157. indent, rs.Votes.StringIndented(indent+" "),
  158. indent, rs.LastCommit.StringShort(),
  159. indent, rs.LastValidators.StringIndented(indent+" "),
  160. indent)
  161. }
  162. func (rs *RoundState) StringShort() string {
  163. return fmt.Sprintf(`RoundState{H:%v R:%v S:%v ST:%v}`,
  164. rs.Height, rs.Round, rs.Step, rs.StartTime)
  165. }
  166. //-----------------------------------------------------------------------------
  167. var (
  168. msgQueueSize = 1000
  169. tickTockBufferSize = 10
  170. )
  171. // msgs from the reactor which may update the state
  172. type msgInfo struct {
  173. Msg ConsensusMessage `json:"msg"`
  174. PeerKey string `json:"peer_key"`
  175. }
  176. // internally generated messages which may update the state
  177. type timeoutInfo struct {
  178. Duration time.Duration `json:"duration"`
  179. Height int `json:"height"`
  180. Round int `json:"round"`
  181. Step RoundStepType `json:"step"`
  182. }
  183. func (ti *timeoutInfo) String() string {
  184. return fmt.Sprintf("%v ; %d/%d %v", ti.Duration, ti.Height, ti.Round, ti.Step)
  185. }
  186. type PrivValidator interface {
  187. GetAddress() []byte
  188. SignVote(chainID string, vote *types.Vote) error
  189. SignProposal(chainID string, proposal *types.Proposal) error
  190. }
  191. // Tracks consensus state across block heights and rounds.
  192. type ConsensusState struct {
  193. BaseService
  194. config cfg.Config
  195. proxyAppConn proxy.AppConnConsensus
  196. blockStore *bc.BlockStore
  197. mempool *mempl.Mempool
  198. privValidator PrivValidator // for signing votes
  199. mtx sync.Mutex
  200. RoundState
  201. state *sm.State // State until height-1.
  202. peerMsgQueue chan msgInfo // serializes msgs affecting state (proposals, block parts, votes)
  203. internalMsgQueue chan msgInfo // like peerMsgQueue but for our own proposals, parts, votes
  204. timeoutTicker TimeoutTicker // ticker for timeouts
  205. timeoutParams *TimeoutParams // parameters and functions for timeout intervals
  206. evsw types.EventSwitch
  207. wal *WAL
  208. replayMode bool // so we don't log signing errors during replay
  209. nSteps int // used for testing to limit the number of transitions the state makes
  210. // allow certain function to be overwritten for testing
  211. decideProposal func(height, round int)
  212. doPrevote func(height, round int)
  213. setProposal func(proposal *types.Proposal) error
  214. }
  215. func NewConsensusState(config cfg.Config, state *sm.State, proxyAppConn proxy.AppConnConsensus, blockStore *bc.BlockStore, mempool *mempl.Mempool) *ConsensusState {
  216. cs := &ConsensusState{
  217. config: config,
  218. proxyAppConn: proxyAppConn,
  219. blockStore: blockStore,
  220. mempool: mempool,
  221. peerMsgQueue: make(chan msgInfo, msgQueueSize),
  222. internalMsgQueue: make(chan msgInfo, msgQueueSize),
  223. timeoutTicker: NewTimeoutTicker(),
  224. timeoutParams: InitTimeoutParamsFromConfig(config),
  225. }
  226. // set function defaults (may be overwritten before calling Start)
  227. cs.decideProposal = cs.defaultDecideProposal
  228. cs.doPrevote = cs.defaultDoPrevote
  229. cs.setProposal = cs.defaultSetProposal
  230. cs.updateToState(state)
  231. // Don't call scheduleRound0 yet.
  232. // We do that upon Start().
  233. cs.reconstructLastCommit(state)
  234. cs.BaseService = *NewBaseService(log, "ConsensusState", cs)
  235. return cs
  236. }
  237. //----------------------------------------
  238. // Public interface
  239. // implements events.Eventable
  240. func (cs *ConsensusState) SetEventSwitch(evsw types.EventSwitch) {
  241. cs.evsw = evsw
  242. }
  243. func (cs *ConsensusState) String() string {
  244. // better not to access shared variables
  245. return Fmt("ConsensusState") //(H:%v R:%v S:%v", cs.Height, cs.Round, cs.Step)
  246. }
  247. func (cs *ConsensusState) GetState() *sm.State {
  248. cs.mtx.Lock()
  249. defer cs.mtx.Unlock()
  250. return cs.state.Copy()
  251. }
  252. func (cs *ConsensusState) GetRoundState() *RoundState {
  253. cs.mtx.Lock()
  254. defer cs.mtx.Unlock()
  255. return cs.getRoundState()
  256. }
  257. func (cs *ConsensusState) getRoundState() *RoundState {
  258. rs := cs.RoundState // copy
  259. return &rs
  260. }
  261. func (cs *ConsensusState) GetValidators() (int, []*types.Validator) {
  262. cs.mtx.Lock()
  263. defer cs.mtx.Unlock()
  264. return cs.state.LastBlockHeight, cs.state.Validators.Copy().Validators
  265. }
  266. // Sets our private validator account for signing votes.
  267. func (cs *ConsensusState) SetPrivValidator(priv PrivValidator) {
  268. cs.mtx.Lock()
  269. defer cs.mtx.Unlock()
  270. cs.privValidator = priv
  271. }
  272. // Set the local timer
  273. func (cs *ConsensusState) SetTimeoutTicker(timeoutTicker TimeoutTicker) {
  274. cs.mtx.Lock()
  275. defer cs.mtx.Unlock()
  276. cs.timeoutTicker = timeoutTicker
  277. }
  278. func (cs *ConsensusState) LoadCommit(height int) *types.Commit {
  279. cs.mtx.Lock()
  280. defer cs.mtx.Unlock()
  281. if height == cs.blockStore.Height() {
  282. return cs.blockStore.LoadSeenCommit(height)
  283. }
  284. return cs.blockStore.LoadBlockCommit(height)
  285. }
  286. func (cs *ConsensusState) OnStart() error {
  287. cs.BaseService.OnStart()
  288. walDir := cs.config.GetString("cs_wal_dir")
  289. err := EnsureDir(walDir, 0700)
  290. if err != nil {
  291. log.Error("Error ensuring ConsensusState wal dir", "error", err.Error())
  292. return err
  293. }
  294. err = cs.OpenWAL(walDir)
  295. if err != nil {
  296. log.Error("Error loading ConsensusState wal", "error", err.Error())
  297. return err
  298. }
  299. // we need the timeoutRoutine for replay so
  300. // we don't block on the tick chan.
  301. // NOTE: we will get a build up of garbage go routines
  302. // firing on the tockChan until the receiveRoutine is started
  303. // to deal with them (by that point, at most one will be valid)
  304. cs.timeoutTicker.Start()
  305. // we may have lost some votes if the process crashed
  306. // reload from consensus log to catchup
  307. if err := cs.catchupReplay(cs.Height); err != nil {
  308. log.Error("Error on catchup replay", "error", err.Error())
  309. // let's go for it anyways, maybe we're fine
  310. }
  311. // now start the receiveRoutine
  312. go cs.receiveRoutine(0)
  313. // schedule the first round!
  314. // use GetRoundState so we don't race the receiveRoutine for access
  315. cs.scheduleRound0(cs.GetRoundState())
  316. return nil
  317. }
  318. // timeoutRoutine: receive requests for timeouts on tickChan and fire timeouts on tockChan
  319. // receiveRoutine: serializes processing of proposoals, block parts, votes; coordinates state transitions
  320. func (cs *ConsensusState) startRoutines(maxSteps int) {
  321. cs.timeoutTicker.Start()
  322. go cs.receiveRoutine(maxSteps)
  323. }
  324. func (cs *ConsensusState) OnStop() {
  325. cs.BaseService.OnStop()
  326. cs.timeoutTicker.Stop()
  327. // Make BaseService.Wait() wait until cs.wal.Wait()
  328. if cs.wal != nil && cs.IsRunning() {
  329. cs.wal.Wait()
  330. }
  331. }
  332. // Open file to log all consensus messages and timeouts for deterministic accountability
  333. func (cs *ConsensusState) OpenWAL(walDir string) (err error) {
  334. cs.mtx.Lock()
  335. defer cs.mtx.Unlock()
  336. wal, err := NewWAL(walDir, cs.config.GetBool("cs_wal_light"))
  337. if err != nil {
  338. return err
  339. }
  340. cs.wal = wal
  341. return nil
  342. }
  343. //------------------------------------------------------------
  344. // Public interface for passing messages into the consensus state,
  345. // possibly causing a state transition
  346. // TODO: should these return anything or let callers just use events?
  347. // May block on send if queue is full.
  348. func (cs *ConsensusState) AddVote(vote *types.Vote, peerKey string) (added bool, err error) {
  349. if peerKey == "" {
  350. cs.internalMsgQueue <- msgInfo{&VoteMessage{vote}, ""}
  351. } else {
  352. cs.peerMsgQueue <- msgInfo{&VoteMessage{vote}, peerKey}
  353. }
  354. // TODO: wait for event?!
  355. return false, nil
  356. }
  357. // May block on send if queue is full.
  358. func (cs *ConsensusState) SetProposal(proposal *types.Proposal, peerKey string) error {
  359. if peerKey == "" {
  360. cs.internalMsgQueue <- msgInfo{&ProposalMessage{proposal}, ""}
  361. } else {
  362. cs.peerMsgQueue <- msgInfo{&ProposalMessage{proposal}, peerKey}
  363. }
  364. // TODO: wait for event?!
  365. return nil
  366. }
  367. // May block on send if queue is full.
  368. func (cs *ConsensusState) AddProposalBlockPart(height, round int, part *types.Part, peerKey string) error {
  369. if peerKey == "" {
  370. cs.internalMsgQueue <- msgInfo{&BlockPartMessage{height, round, part}, ""}
  371. } else {
  372. cs.peerMsgQueue <- msgInfo{&BlockPartMessage{height, round, part}, peerKey}
  373. }
  374. // TODO: wait for event?!
  375. return nil
  376. }
  377. // May block on send if queue is full.
  378. func (cs *ConsensusState) SetProposalAndBlock(proposal *types.Proposal, block *types.Block, parts *types.PartSet, peerKey string) error {
  379. cs.SetProposal(proposal, peerKey)
  380. for i := 0; i < parts.Total(); i++ {
  381. part := parts.GetPart(i)
  382. cs.AddProposalBlockPart(proposal.Height, proposal.Round, part, peerKey)
  383. }
  384. return nil // TODO errors
  385. }
  386. //------------------------------------------------------------
  387. // internal functions for managing the state
  388. func (cs *ConsensusState) updateHeight(height int) {
  389. cs.Height = height
  390. }
  391. func (cs *ConsensusState) updateRoundStep(round int, step RoundStepType) {
  392. cs.Round = round
  393. cs.Step = step
  394. }
  395. // enterNewRound(height, 0) at cs.StartTime.
  396. func (cs *ConsensusState) scheduleRound0(rs *RoundState) {
  397. //log.Info("scheduleRound0", "now", time.Now(), "startTime", cs.StartTime)
  398. sleepDuration := rs.StartTime.Sub(time.Now())
  399. if sleepDuration < time.Duration(0) {
  400. sleepDuration = time.Duration(0)
  401. }
  402. cs.scheduleTimeout(sleepDuration, rs.Height, 0, RoundStepNewHeight)
  403. }
  404. // Attempt to schedule a timeout (by sending timeoutInfo on the tickChan)
  405. func (cs *ConsensusState) scheduleTimeout(duration time.Duration, height, round int, step RoundStepType) {
  406. cs.timeoutTicker.ScheduleTimeout(timeoutInfo{duration, height, round, step})
  407. }
  408. // send a msg into the receiveRoutine regarding our own proposal, block part, or vote
  409. func (cs *ConsensusState) sendInternalMessage(mi msgInfo) {
  410. select {
  411. case cs.internalMsgQueue <- mi:
  412. default:
  413. // NOTE: using the go-routine means our votes can
  414. // be processed out of order.
  415. // TODO: use CList here for strict determinism and
  416. // attempt push to internalMsgQueue in receiveRoutine
  417. log.Warn("Internal msg queue is full. Using a go-routine")
  418. go func() { cs.internalMsgQueue <- mi }()
  419. }
  420. }
  421. // Reconstruct LastCommit from SeenCommit, which we saved along with the block,
  422. // (which happens even before saving the state)
  423. func (cs *ConsensusState) reconstructLastCommit(state *sm.State) {
  424. if state.LastBlockHeight == 0 {
  425. return
  426. }
  427. seenCommit := cs.blockStore.LoadSeenCommit(state.LastBlockHeight)
  428. lastPrecommits := types.NewVoteSet(cs.config.GetString("chain_id"), state.LastBlockHeight, seenCommit.Round(), types.VoteTypePrecommit, state.LastValidators)
  429. for _, precommit := range seenCommit.Precommits {
  430. if precommit == nil {
  431. continue
  432. }
  433. added, err := lastPrecommits.AddVote(precommit)
  434. if !added || err != nil {
  435. PanicCrisis(Fmt("Failed to reconstruct LastCommit: %v", err))
  436. }
  437. }
  438. if !lastPrecommits.HasTwoThirdsMajority() {
  439. PanicSanity("Failed to reconstruct LastCommit: Does not have +2/3 maj")
  440. }
  441. cs.LastCommit = lastPrecommits
  442. }
  443. // Updates ConsensusState and increments height to match that of state.
  444. // The round becomes 0 and cs.Step becomes RoundStepNewHeight.
  445. func (cs *ConsensusState) updateToState(state *sm.State) {
  446. if cs.CommitRound > -1 && 0 < cs.Height && cs.Height != state.LastBlockHeight {
  447. PanicSanity(Fmt("updateToState() expected state height of %v but found %v",
  448. cs.Height, state.LastBlockHeight))
  449. }
  450. if cs.state != nil && cs.state.LastBlockHeight+1 != cs.Height {
  451. // This might happen when someone else is mutating cs.state.
  452. // Someone forgot to pass in state.Copy() somewhere?!
  453. PanicSanity(Fmt("Inconsistent cs.state.LastBlockHeight+1 %v vs cs.Height %v",
  454. cs.state.LastBlockHeight+1, cs.Height))
  455. }
  456. // If state isn't further out than cs.state, just ignore.
  457. // This happens when SwitchToConsensus() is called in the reactor.
  458. // We don't want to reset e.g. the Votes.
  459. if cs.state != nil && (state.LastBlockHeight <= cs.state.LastBlockHeight) {
  460. log.Notice("Ignoring updateToState()", "newHeight", state.LastBlockHeight+1, "oldHeight", cs.state.LastBlockHeight+1)
  461. return
  462. }
  463. // Reset fields based on state.
  464. validators := state.Validators
  465. height := state.LastBlockHeight + 1 // Next desired block height
  466. lastPrecommits := (*types.VoteSet)(nil)
  467. if cs.CommitRound > -1 && cs.Votes != nil {
  468. if !cs.Votes.Precommits(cs.CommitRound).HasTwoThirdsMajority() {
  469. PanicSanity("updateToState(state) called but last Precommit round didn't have +2/3")
  470. }
  471. lastPrecommits = cs.Votes.Precommits(cs.CommitRound)
  472. }
  473. // RoundState fields
  474. cs.updateHeight(height)
  475. cs.updateRoundStep(0, RoundStepNewHeight)
  476. if cs.CommitTime.IsZero() {
  477. // "Now" makes it easier to sync up dev nodes.
  478. // We add timeoutCommit to allow transactions
  479. // to be gathered for the first block.
  480. // And alternative solution that relies on clocks:
  481. // cs.StartTime = state.LastBlockTime.Add(timeoutCommit)
  482. cs.StartTime = cs.timeoutParams.Commit(time.Now())
  483. } else {
  484. cs.StartTime = cs.timeoutParams.Commit(cs.CommitTime)
  485. }
  486. cs.Validators = validators
  487. cs.Proposal = nil
  488. cs.ProposalBlock = nil
  489. cs.ProposalBlockParts = nil
  490. cs.LockedRound = 0
  491. cs.LockedBlock = nil
  492. cs.LockedBlockParts = nil
  493. cs.Votes = NewHeightVoteSet(cs.config.GetString("chain_id"), height, validators)
  494. cs.CommitRound = -1
  495. cs.LastCommit = lastPrecommits
  496. cs.LastValidators = state.LastValidators
  497. cs.state = state
  498. // Finally, broadcast RoundState
  499. cs.newStep()
  500. }
  501. func (cs *ConsensusState) newStep() {
  502. rs := cs.RoundStateEvent()
  503. cs.wal.Save(rs)
  504. cs.nSteps += 1
  505. // newStep is called by updateToStep in NewConsensusState before the evsw is set!
  506. if cs.evsw != nil {
  507. types.FireEventNewRoundStep(cs.evsw, rs)
  508. }
  509. }
  510. //-----------------------------------------
  511. // the main go routines
  512. // a nice idea but probably more trouble than its worth
  513. func (cs *ConsensusState) stopTimer() {
  514. cs.timeoutTicker.Stop()
  515. }
  516. // receiveRoutine handles messages which may cause state transitions.
  517. // it's argument (n) is the number of messages to process before exiting - use 0 to run forever
  518. // It keeps the RoundState and is the only thing that updates it.
  519. // Updates (state transitions) happen on timeouts, complete proposals, and 2/3 majorities
  520. func (cs *ConsensusState) receiveRoutine(maxSteps int) {
  521. for {
  522. if maxSteps > 0 {
  523. if cs.nSteps >= maxSteps {
  524. log.Warn("reached max steps. exiting receive routine")
  525. cs.nSteps = 0
  526. return
  527. }
  528. }
  529. rs := cs.RoundState
  530. var mi msgInfo
  531. select {
  532. case mi = <-cs.peerMsgQueue:
  533. cs.wal.Save(mi)
  534. // handles proposals, block parts, votes
  535. // may generate internal events (votes, complete proposals, 2/3 majorities)
  536. cs.handleMsg(mi, rs)
  537. case mi = <-cs.internalMsgQueue:
  538. cs.wal.Save(mi)
  539. // handles proposals, block parts, votes
  540. cs.handleMsg(mi, rs)
  541. case ti := <-cs.timeoutTicker.Chan(): // tockChan:
  542. cs.wal.Save(ti)
  543. // if the timeout is relevant to the rs
  544. // go to the next step
  545. cs.handleTimeout(ti, rs)
  546. case <-cs.Quit:
  547. // NOTE: the internalMsgQueue may have signed messages from our
  548. // priv_val that haven't hit the WAL, but its ok because
  549. // priv_val tracks LastSig
  550. // close wal now that we're done writing to it
  551. if cs.wal != nil {
  552. cs.wal.Stop()
  553. }
  554. return
  555. }
  556. }
  557. }
  558. // state transitions on complete-proposal, 2/3-any, 2/3-one
  559. func (cs *ConsensusState) handleMsg(mi msgInfo, rs RoundState) {
  560. cs.mtx.Lock()
  561. defer cs.mtx.Unlock()
  562. var err error
  563. msg, peerKey := mi.Msg, mi.PeerKey
  564. switch msg := msg.(type) {
  565. case *ProposalMessage:
  566. // will not cause transition.
  567. // once proposal is set, we can receive block parts
  568. err = cs.setProposal(msg.Proposal)
  569. case *BlockPartMessage:
  570. // if the proposal is complete, we'll enterPrevote or tryFinalizeCommit
  571. _, err = cs.addProposalBlockPart(msg.Height, msg.Part, peerKey != "")
  572. if err != nil && msg.Round != cs.Round {
  573. err = nil
  574. }
  575. case *VoteMessage:
  576. // attempt to add the vote and dupeout the validator if its a duplicate signature
  577. // if the vote gives us a 2/3-any or 2/3-one, we transition
  578. err := cs.tryAddVote(msg.Vote, peerKey)
  579. if err == ErrAddingVote {
  580. // TODO: punish peer
  581. }
  582. // NOTE: the vote is broadcast to peers by the reactor listening
  583. // for vote events
  584. // TODO: If rs.Height == vote.Height && rs.Round < vote.Round,
  585. // the peer is sending us CatchupCommit precommits.
  586. // We could make note of this and help filter in broadcastHasVoteMessage().
  587. default:
  588. log.Warn("Unknown msg type", reflect.TypeOf(msg))
  589. }
  590. if err != nil {
  591. log.Error("Error with msg", "type", reflect.TypeOf(msg), "peer", peerKey, "error", err, "msg", msg)
  592. }
  593. }
  594. func (cs *ConsensusState) handleTimeout(ti timeoutInfo, rs RoundState) {
  595. log.Debug("Received tock", "timeout", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step)
  596. // timeouts must be for current height, round, step
  597. if ti.Height != rs.Height || ti.Round < rs.Round || (ti.Round == rs.Round && ti.Step < rs.Step) {
  598. log.Debug("Ignoring tock because we're ahead", "height", rs.Height, "round", rs.Round, "step", rs.Step)
  599. return
  600. }
  601. // the timeout will now cause a state transition
  602. cs.mtx.Lock()
  603. defer cs.mtx.Unlock()
  604. switch ti.Step {
  605. case RoundStepNewHeight:
  606. // NewRound event fired from enterNewRound.
  607. // XXX: should we fire timeout here (for timeout commit)?
  608. cs.enterNewRound(ti.Height, 0)
  609. case RoundStepPropose:
  610. types.FireEventTimeoutPropose(cs.evsw, cs.RoundStateEvent())
  611. cs.enterPrevote(ti.Height, ti.Round)
  612. case RoundStepPrevoteWait:
  613. types.FireEventTimeoutWait(cs.evsw, cs.RoundStateEvent())
  614. cs.enterPrecommit(ti.Height, ti.Round)
  615. case RoundStepPrecommitWait:
  616. types.FireEventTimeoutWait(cs.evsw, cs.RoundStateEvent())
  617. cs.enterNewRound(ti.Height, ti.Round+1)
  618. default:
  619. panic(Fmt("Invalid timeout step: %v", ti.Step))
  620. }
  621. }
  622. //-----------------------------------------------------------------------------
  623. // State functions
  624. // Used internally by handleTimeout and handleMsg to make state transitions
  625. // Enter: +2/3 precommits for nil at (height,round-1)
  626. // Enter: `timeoutPrecommits` after any +2/3 precommits from (height,round-1)
  627. // Enter: `startTime = commitTime+timeoutCommit` from NewHeight(height)
  628. // NOTE: cs.StartTime was already set for height.
  629. func (cs *ConsensusState) enterNewRound(height int, round int) {
  630. if cs.Height != height || round < cs.Round || (cs.Round == round && cs.Step != RoundStepNewHeight) {
  631. log.Debug(Fmt("enterNewRound(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  632. return
  633. }
  634. if now := time.Now(); cs.StartTime.After(now) {
  635. log.Warn("Need to set a buffer and log.Warn() here for sanity.", "startTime", cs.StartTime, "now", now)
  636. }
  637. // cs.stopTimer()
  638. log.Notice(Fmt("enterNewRound(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  639. // Increment validators if necessary
  640. validators := cs.Validators
  641. if cs.Round < round {
  642. validators = validators.Copy()
  643. validators.IncrementAccum(round - cs.Round)
  644. }
  645. // Setup new round
  646. // we don't fire newStep for this step,
  647. // but we fire an event, so update the round step first
  648. cs.updateRoundStep(round, RoundStepNewRound)
  649. cs.Validators = validators
  650. if round == 0 {
  651. // We've already reset these upon new height,
  652. // and meanwhile we might have received a proposal
  653. // for round 0.
  654. } else {
  655. cs.Proposal = nil
  656. cs.ProposalBlock = nil
  657. cs.ProposalBlockParts = nil
  658. }
  659. cs.Votes.SetRound(round + 1) // also track next round (round+1) to allow round-skipping
  660. types.FireEventNewRound(cs.evsw, cs.RoundStateEvent())
  661. // Immediately go to enterPropose.
  662. cs.enterPropose(height, round)
  663. }
  664. // Enter: from NewRound(height,round).
  665. func (cs *ConsensusState) enterPropose(height int, round int) {
  666. if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPropose <= cs.Step) {
  667. log.Debug(Fmt("enterPropose(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  668. return
  669. }
  670. log.Info(Fmt("enterPropose(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  671. defer func() {
  672. // Done enterPropose:
  673. cs.updateRoundStep(round, RoundStepPropose)
  674. cs.newStep()
  675. // If we have the whole proposal + POL, then goto Prevote now.
  676. // else, we'll enterPrevote when the rest of the proposal is received (in AddProposalBlockPart),
  677. // or else after timeoutPropose
  678. if cs.isProposalComplete() {
  679. cs.enterPrevote(height, cs.Round)
  680. }
  681. }()
  682. // If we don't get the proposal and all block parts quick enough, enterPrevote
  683. cs.scheduleTimeout(cs.timeoutParams.Propose(round), height, round, RoundStepPropose)
  684. // Nothing more to do if we're not a validator
  685. if cs.privValidator == nil {
  686. return
  687. }
  688. if !bytes.Equal(cs.Validators.Proposer().Address, cs.privValidator.GetAddress()) {
  689. log.Info("enterPropose: Not our turn to propose", "proposer", cs.Validators.Proposer().Address, "privValidator", cs.privValidator)
  690. } else {
  691. log.Info("enterPropose: Our turn to propose", "proposer", cs.Validators.Proposer().Address, "privValidator", cs.privValidator)
  692. cs.decideProposal(height, round)
  693. }
  694. }
  695. func (cs *ConsensusState) defaultDecideProposal(height, round int) {
  696. var block *types.Block
  697. var blockParts *types.PartSet
  698. // Decide on block
  699. if cs.LockedBlock != nil {
  700. // If we're locked onto a block, just choose that.
  701. block, blockParts = cs.LockedBlock, cs.LockedBlockParts
  702. } else {
  703. // Create a new proposal block from state/txs from the mempool.
  704. block, blockParts = cs.createProposalBlock()
  705. if block == nil { // on error
  706. return
  707. }
  708. }
  709. // Make proposal
  710. polRound, polBlockID := cs.Votes.POLInfo()
  711. proposal := types.NewProposal(height, round, blockParts.Header(), polRound, polBlockID)
  712. err := cs.privValidator.SignProposal(cs.state.ChainID, proposal)
  713. if err == nil {
  714. // Set fields
  715. /* fields set by setProposal and addBlockPart
  716. cs.Proposal = proposal
  717. cs.ProposalBlock = block
  718. cs.ProposalBlockParts = blockParts
  719. */
  720. // send proposal and block parts on internal msg queue
  721. cs.sendInternalMessage(msgInfo{&ProposalMessage{proposal}, ""})
  722. for i := 0; i < blockParts.Total(); i++ {
  723. part := blockParts.GetPart(i)
  724. cs.sendInternalMessage(msgInfo{&BlockPartMessage{cs.Height, cs.Round, part}, ""})
  725. }
  726. log.Info("Signed proposal", "height", height, "round", round, "proposal", proposal)
  727. log.Debug(Fmt("Signed proposal block: %v", block))
  728. } else {
  729. if !cs.replayMode {
  730. log.Warn("enterPropose: Error signing proposal", "height", height, "round", round, "error", err)
  731. }
  732. }
  733. }
  734. // Returns true if the proposal block is complete &&
  735. // (if POLRound was proposed, we have +2/3 prevotes from there).
  736. func (cs *ConsensusState) isProposalComplete() bool {
  737. if cs.Proposal == nil || cs.ProposalBlock == nil {
  738. return false
  739. }
  740. // we have the proposal. if there's a POLRound,
  741. // make sure we have the prevotes from it too
  742. if cs.Proposal.POLRound < 0 {
  743. return true
  744. } else {
  745. // if this is false the proposer is lying or we haven't received the POL yet
  746. return cs.Votes.Prevotes(cs.Proposal.POLRound).HasTwoThirdsMajority()
  747. }
  748. }
  749. // Create the next block to propose and return it.
  750. // Returns nil block upon error.
  751. // NOTE: keep it side-effect free for clarity.
  752. func (cs *ConsensusState) createProposalBlock() (block *types.Block, blockParts *types.PartSet) {
  753. var commit *types.Commit
  754. if cs.Height == 1 {
  755. // We're creating a proposal for the first block.
  756. // The commit is empty, but not nil.
  757. commit = &types.Commit{}
  758. } else if cs.LastCommit.HasTwoThirdsMajority() {
  759. // Make the commit from LastCommit
  760. commit = cs.LastCommit.MakeCommit()
  761. } else {
  762. // This shouldn't happen.
  763. log.Error("enterPropose: Cannot propose anything: No commit for the previous block.")
  764. return
  765. }
  766. // Mempool validated transactions
  767. txs := cs.mempool.Reap(cs.config.GetInt("block_size"))
  768. return types.MakeBlock(cs.Height, cs.state.ChainID, txs, commit,
  769. cs.state.LastBlockID, cs.state.Validators.Hash(), cs.state.AppHash, cs.config.GetInt("block_part_size"))
  770. }
  771. // Enter: `timeoutPropose` after entering Propose.
  772. // Enter: proposal block and POL is ready.
  773. // Enter: any +2/3 prevotes for future round.
  774. // Prevote for LockedBlock if we're locked, or ProposalBlock if valid.
  775. // Otherwise vote nil.
  776. func (cs *ConsensusState) enterPrevote(height int, round int) {
  777. if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPrevote <= cs.Step) {
  778. log.Debug(Fmt("enterPrevote(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  779. return
  780. }
  781. defer func() {
  782. // Done enterPrevote:
  783. cs.updateRoundStep(round, RoundStepPrevote)
  784. cs.newStep()
  785. }()
  786. // fire event for how we got here
  787. if cs.isProposalComplete() {
  788. types.FireEventCompleteProposal(cs.evsw, cs.RoundStateEvent())
  789. } else {
  790. // we received +2/3 prevotes for a future round
  791. // TODO: catchup event?
  792. }
  793. // cs.stopTimer()
  794. log.Info(Fmt("enterPrevote(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  795. // Sign and broadcast vote as necessary
  796. cs.doPrevote(height, round)
  797. // Once `addVote` hits any +2/3 prevotes, we will go to PrevoteWait
  798. // (so we have more time to try and collect +2/3 prevotes for a single block)
  799. }
  800. func (cs *ConsensusState) defaultDoPrevote(height int, round int) {
  801. // If a block is locked, prevote that.
  802. if cs.LockedBlock != nil {
  803. log.Notice("enterPrevote: Block was locked")
  804. cs.signAddVote(types.VoteTypePrevote, cs.LockedBlock.Hash(), cs.LockedBlockParts.Header())
  805. return
  806. }
  807. // If ProposalBlock is nil, prevote nil.
  808. if cs.ProposalBlock == nil {
  809. log.Warn("enterPrevote: ProposalBlock is nil")
  810. cs.signAddVote(types.VoteTypePrevote, nil, types.PartSetHeader{})
  811. return
  812. }
  813. // Valdiate proposal block
  814. err := cs.state.ValidateBlock(cs.ProposalBlock)
  815. if err != nil {
  816. // ProposalBlock is invalid, prevote nil.
  817. log.Warn("enterPrevote: ProposalBlock is invalid", "error", err)
  818. cs.signAddVote(types.VoteTypePrevote, nil, types.PartSetHeader{})
  819. return
  820. }
  821. // Prevote cs.ProposalBlock
  822. // NOTE: the proposal signature is validated when it is received,
  823. // and the proposal block parts are validated as they are received (against the merkle hash in the proposal)
  824. cs.signAddVote(types.VoteTypePrevote, cs.ProposalBlock.Hash(), cs.ProposalBlockParts.Header())
  825. return
  826. }
  827. // Enter: any +2/3 prevotes at next round.
  828. func (cs *ConsensusState) enterPrevoteWait(height int, round int) {
  829. if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPrevoteWait <= cs.Step) {
  830. log.Debug(Fmt("enterPrevoteWait(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  831. return
  832. }
  833. if !cs.Votes.Prevotes(round).HasTwoThirdsAny() {
  834. PanicSanity(Fmt("enterPrevoteWait(%v/%v), but Prevotes does not have any +2/3 votes", height, round))
  835. }
  836. log.Info(Fmt("enterPrevoteWait(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  837. defer func() {
  838. // Done enterPrevoteWait:
  839. cs.updateRoundStep(round, RoundStepPrevoteWait)
  840. cs.newStep()
  841. }()
  842. // Wait for some more prevotes; enterPrecommit
  843. cs.scheduleTimeout(cs.timeoutParams.Prevote(round), height, round, RoundStepPrevoteWait)
  844. }
  845. // Enter: +2/3 precomits for block or nil.
  846. // Enter: `timeoutPrevote` after any +2/3 prevotes.
  847. // Enter: any +2/3 precommits for next round.
  848. // Lock & precommit the ProposalBlock if we have enough prevotes for it (a POL in this round)
  849. // else, unlock an existing lock and precommit nil if +2/3 of prevotes were nil,
  850. // else, precommit nil otherwise.
  851. func (cs *ConsensusState) enterPrecommit(height int, round int) {
  852. if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPrecommit <= cs.Step) {
  853. log.Debug(Fmt("enterPrecommit(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  854. return
  855. }
  856. // cs.stopTimer()
  857. log.Info(Fmt("enterPrecommit(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  858. defer func() {
  859. // Done enterPrecommit:
  860. cs.updateRoundStep(round, RoundStepPrecommit)
  861. cs.newStep()
  862. }()
  863. blockID, ok := cs.Votes.Prevotes(round).TwoThirdsMajority()
  864. // If we don't have a polka, we must precommit nil
  865. if !ok {
  866. if cs.LockedBlock != nil {
  867. log.Notice("enterPrecommit: No +2/3 prevotes during enterPrecommit while we're locked. Precommitting nil")
  868. } else {
  869. log.Notice("enterPrecommit: No +2/3 prevotes during enterPrecommit. Precommitting nil.")
  870. }
  871. cs.signAddVote(types.VoteTypePrecommit, nil, types.PartSetHeader{})
  872. return
  873. }
  874. // At this point +2/3 prevoted for a particular block or nil
  875. types.FireEventPolka(cs.evsw, cs.RoundStateEvent())
  876. // the latest POLRound should be this round
  877. polRound, _ := cs.Votes.POLInfo()
  878. if polRound < round {
  879. PanicSanity(Fmt("This POLRound should be %v but got %", round, polRound))
  880. }
  881. // +2/3 prevoted nil. Unlock and precommit nil.
  882. if len(blockID.Hash) == 0 {
  883. if cs.LockedBlock == nil {
  884. log.Notice("enterPrecommit: +2/3 prevoted for nil.")
  885. } else {
  886. log.Notice("enterPrecommit: +2/3 prevoted for nil. Unlocking")
  887. cs.LockedRound = 0
  888. cs.LockedBlock = nil
  889. cs.LockedBlockParts = nil
  890. types.FireEventUnlock(cs.evsw, cs.RoundStateEvent())
  891. }
  892. cs.signAddVote(types.VoteTypePrecommit, nil, types.PartSetHeader{})
  893. return
  894. }
  895. // At this point, +2/3 prevoted for a particular block.
  896. // If we're already locked on that block, precommit it, and update the LockedRound
  897. if cs.LockedBlock.HashesTo(blockID.Hash) {
  898. log.Notice("enterPrecommit: +2/3 prevoted locked block. Relocking")
  899. cs.LockedRound = round
  900. types.FireEventRelock(cs.evsw, cs.RoundStateEvent())
  901. cs.signAddVote(types.VoteTypePrecommit, blockID.Hash, blockID.PartsHeader)
  902. return
  903. }
  904. // If +2/3 prevoted for proposal block, stage and precommit it
  905. if cs.ProposalBlock.HashesTo(blockID.Hash) {
  906. log.Notice("enterPrecommit: +2/3 prevoted proposal block. Locking", "hash", blockID.Hash)
  907. // Validate the block.
  908. if err := cs.state.ValidateBlock(cs.ProposalBlock); err != nil {
  909. PanicConsensus(Fmt("enterPrecommit: +2/3 prevoted for an invalid block: %v", err))
  910. }
  911. cs.LockedRound = round
  912. cs.LockedBlock = cs.ProposalBlock
  913. cs.LockedBlockParts = cs.ProposalBlockParts
  914. types.FireEventLock(cs.evsw, cs.RoundStateEvent())
  915. cs.signAddVote(types.VoteTypePrecommit, blockID.Hash, blockID.PartsHeader)
  916. return
  917. }
  918. // There was a polka in this round for a block we don't have.
  919. // Fetch that block, unlock, and precommit nil.
  920. // The +2/3 prevotes for this round is the POL for our unlock.
  921. // TODO: In the future save the POL prevotes for justification.
  922. cs.LockedRound = 0
  923. cs.LockedBlock = nil
  924. cs.LockedBlockParts = nil
  925. if !cs.ProposalBlockParts.HasHeader(blockID.PartsHeader) {
  926. cs.ProposalBlock = nil
  927. cs.ProposalBlockParts = types.NewPartSetFromHeader(blockID.PartsHeader)
  928. }
  929. types.FireEventUnlock(cs.evsw, cs.RoundStateEvent())
  930. cs.signAddVote(types.VoteTypePrecommit, nil, types.PartSetHeader{})
  931. return
  932. }
  933. // Enter: any +2/3 precommits for next round.
  934. func (cs *ConsensusState) enterPrecommitWait(height int, round int) {
  935. if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPrecommitWait <= cs.Step) {
  936. log.Debug(Fmt("enterPrecommitWait(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  937. return
  938. }
  939. if !cs.Votes.Precommits(round).HasTwoThirdsAny() {
  940. PanicSanity(Fmt("enterPrecommitWait(%v/%v), but Precommits does not have any +2/3 votes", height, round))
  941. }
  942. log.Info(Fmt("enterPrecommitWait(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  943. defer func() {
  944. // Done enterPrecommitWait:
  945. cs.updateRoundStep(round, RoundStepPrecommitWait)
  946. cs.newStep()
  947. }()
  948. // Wait for some more precommits; enterNewRound
  949. cs.scheduleTimeout(cs.timeoutParams.Precommit(round), height, round, RoundStepPrecommitWait)
  950. }
  951. // Enter: +2/3 precommits for block
  952. func (cs *ConsensusState) enterCommit(height int, commitRound int) {
  953. if cs.Height != height || RoundStepCommit <= cs.Step {
  954. log.Debug(Fmt("enterCommit(%v/%v): Invalid args. Current step: %v/%v/%v", height, commitRound, cs.Height, cs.Round, cs.Step))
  955. return
  956. }
  957. log.Info(Fmt("enterCommit(%v/%v). Current: %v/%v/%v", height, commitRound, cs.Height, cs.Round, cs.Step))
  958. defer func() {
  959. // Done enterCommit:
  960. // keep cs.Round the same, commitRound points to the right Precommits set.
  961. cs.updateRoundStep(cs.Round, RoundStepCommit)
  962. cs.CommitRound = commitRound
  963. cs.CommitTime = time.Now()
  964. cs.newStep()
  965. // Maybe finalize immediately.
  966. cs.tryFinalizeCommit(height)
  967. }()
  968. blockID, ok := cs.Votes.Precommits(commitRound).TwoThirdsMajority()
  969. if !ok {
  970. PanicSanity("RunActionCommit() expects +2/3 precommits")
  971. }
  972. // The Locked* fields no longer matter.
  973. // Move them over to ProposalBlock if they match the commit hash,
  974. // otherwise they'll be cleared in updateToState.
  975. if cs.LockedBlock.HashesTo(blockID.Hash) {
  976. cs.ProposalBlock = cs.LockedBlock
  977. cs.ProposalBlockParts = cs.LockedBlockParts
  978. }
  979. // If we don't have the block being committed, set up to get it.
  980. if !cs.ProposalBlock.HashesTo(blockID.Hash) {
  981. if !cs.ProposalBlockParts.HasHeader(blockID.PartsHeader) {
  982. // We're getting the wrong block.
  983. // Set up ProposalBlockParts and keep waiting.
  984. cs.ProposalBlock = nil
  985. cs.ProposalBlockParts = types.NewPartSetFromHeader(blockID.PartsHeader)
  986. } else {
  987. // We just need to keep waiting.
  988. }
  989. }
  990. }
  991. // If we have the block AND +2/3 commits for it, finalize.
  992. func (cs *ConsensusState) tryFinalizeCommit(height int) {
  993. if cs.Height != height {
  994. PanicSanity(Fmt("tryFinalizeCommit() cs.Height: %v vs height: %v", cs.Height, height))
  995. }
  996. blockID, ok := cs.Votes.Precommits(cs.CommitRound).TwoThirdsMajority()
  997. if !ok || len(blockID.Hash) == 0 {
  998. log.Warn("Attempt to finalize failed. There was no +2/3 majority, or +2/3 was for <nil>.")
  999. return
  1000. }
  1001. if !cs.ProposalBlock.HashesTo(blockID.Hash) {
  1002. // TODO: this happens every time if we're not a validator (ugly logs)
  1003. // TODO: ^^ wait, why does it matter that we're a validator?
  1004. log.Warn("Attempt to finalize failed. We don't have the commit block.")
  1005. return
  1006. }
  1007. // go
  1008. cs.finalizeCommit(height)
  1009. }
  1010. // Increment height and goto RoundStepNewHeight
  1011. func (cs *ConsensusState) finalizeCommit(height int) {
  1012. if cs.Height != height || cs.Step != RoundStepCommit {
  1013. log.Debug(Fmt("finalizeCommit(%v): Invalid args. Current step: %v/%v/%v", height, cs.Height, cs.Round, cs.Step))
  1014. return
  1015. }
  1016. blockID, ok := cs.Votes.Precommits(cs.CommitRound).TwoThirdsMajority()
  1017. block, blockParts := cs.ProposalBlock, cs.ProposalBlockParts
  1018. if !ok {
  1019. PanicSanity(Fmt("Cannot finalizeCommit, commit does not have two thirds majority"))
  1020. }
  1021. if !blockParts.HasHeader(blockID.PartsHeader) {
  1022. PanicSanity(Fmt("Expected ProposalBlockParts header to be commit header"))
  1023. }
  1024. if !block.HashesTo(blockID.Hash) {
  1025. PanicSanity(Fmt("Cannot finalizeCommit, ProposalBlock does not hash to commit hash"))
  1026. }
  1027. if err := cs.state.ValidateBlock(block); err != nil {
  1028. PanicConsensus(Fmt("+2/3 committed an invalid block: %v", err))
  1029. }
  1030. log.Notice(Fmt("Finalizing commit of block with %d txs", block.NumTxs),
  1031. "height", block.Height, "hash", block.Hash(), "root", block.AppHash)
  1032. log.Info(Fmt("%v", block))
  1033. fail.Fail() // XXX
  1034. // Save to blockStore.
  1035. if cs.blockStore.Height() < block.Height {
  1036. // NOTE: the seenCommit is local justification to commit this block,
  1037. // but may differ from the LastCommit included in the next block
  1038. precommits := cs.Votes.Precommits(cs.CommitRound)
  1039. seenCommit := precommits.MakeCommit()
  1040. cs.blockStore.SaveBlock(block, blockParts, seenCommit)
  1041. } else {
  1042. log.Warn("Why are we finalizeCommitting a block height we already have?", "height", block.Height)
  1043. }
  1044. fail.Fail() // XXX
  1045. // Create a copy of the state for staging
  1046. // and an event cache for txs
  1047. stateCopy := cs.state.Copy()
  1048. eventCache := types.NewEventCache(cs.evsw)
  1049. // Execute and commit the block, and update the mempool.
  1050. // All calls to the proxyAppConn should come here.
  1051. // NOTE: the block.AppHash wont reflect these txs until the next block
  1052. err := stateCopy.ApplyBlock(eventCache, cs.proxyAppConn, block, blockParts.Header(), cs.mempool)
  1053. if err != nil {
  1054. // TODO!
  1055. }
  1056. fail.Fail() // XXX
  1057. // Fire off event for new block.
  1058. // TODO: Handle app failure. See #177
  1059. types.FireEventNewBlock(cs.evsw, types.EventDataNewBlock{block})
  1060. types.FireEventNewBlockHeader(cs.evsw, types.EventDataNewBlockHeader{block.Header})
  1061. eventCache.Flush()
  1062. // Save the state.
  1063. stateCopy.Save()
  1064. fail.Fail() // XXX
  1065. // NewHeightStep!
  1066. cs.updateToState(stateCopy)
  1067. // cs.StartTime is already set.
  1068. // Schedule Round0 to start soon.
  1069. cs.scheduleRound0(&cs.RoundState)
  1070. // By here,
  1071. // * cs.Height has been increment to height+1
  1072. // * cs.Step is now RoundStepNewHeight
  1073. // * cs.StartTime is set to when we will start round0.
  1074. return
  1075. }
  1076. //-----------------------------------------------------------------------------
  1077. func (cs *ConsensusState) defaultSetProposal(proposal *types.Proposal) error {
  1078. // Already have one
  1079. // TODO: possibly catch double proposals
  1080. if cs.Proposal != nil {
  1081. return nil
  1082. }
  1083. // Does not apply
  1084. if proposal.Height != cs.Height || proposal.Round != cs.Round {
  1085. return nil
  1086. }
  1087. // We don't care about the proposal if we're already in RoundStepCommit.
  1088. if RoundStepCommit <= cs.Step {
  1089. return nil
  1090. }
  1091. // Verify POLRound, which must be -1 or between 0 and proposal.Round exclusive.
  1092. if proposal.POLRound != -1 &&
  1093. (proposal.POLRound < 0 || proposal.Round <= proposal.POLRound) {
  1094. return ErrInvalidProposalPOLRound
  1095. }
  1096. // Verify signature
  1097. if !cs.Validators.Proposer().PubKey.VerifyBytes(types.SignBytes(cs.state.ChainID, proposal), proposal.Signature) {
  1098. return ErrInvalidProposalSignature
  1099. }
  1100. cs.Proposal = proposal
  1101. cs.ProposalBlockParts = types.NewPartSetFromHeader(proposal.BlockPartsHeader)
  1102. return nil
  1103. }
  1104. // NOTE: block is not necessarily valid.
  1105. // Asynchronously triggers either enterPrevote (before we timeout of propose) or tryFinalizeCommit, once we have the full block.
  1106. func (cs *ConsensusState) addProposalBlockPart(height int, part *types.Part, verify bool) (added bool, err error) {
  1107. // Blocks might be reused, so round mismatch is OK
  1108. if cs.Height != height {
  1109. return false, nil
  1110. }
  1111. // We're not expecting a block part.
  1112. if cs.ProposalBlockParts == nil {
  1113. return false, nil // TODO: bad peer? Return error?
  1114. }
  1115. added, err = cs.ProposalBlockParts.AddPart(part, verify)
  1116. if err != nil {
  1117. return added, err
  1118. }
  1119. if added && cs.ProposalBlockParts.IsComplete() {
  1120. // Added and completed!
  1121. var n int
  1122. var err error
  1123. cs.ProposalBlock = wire.ReadBinary(&types.Block{}, cs.ProposalBlockParts.GetReader(), types.MaxBlockSize, &n, &err).(*types.Block)
  1124. // NOTE: it's possible to receive complete proposal blocks for future rounds without having the proposal
  1125. log.Info("Received complete proposal block", "height", cs.ProposalBlock.Height, "hash", cs.ProposalBlock.Hash())
  1126. if cs.Step == RoundStepPropose && cs.isProposalComplete() {
  1127. // Move onto the next step
  1128. cs.enterPrevote(height, cs.Round)
  1129. } else if cs.Step == RoundStepCommit {
  1130. // If we're waiting on the proposal block...
  1131. cs.tryFinalizeCommit(height)
  1132. }
  1133. return true, err
  1134. }
  1135. return added, nil
  1136. }
  1137. // Attempt to add the vote. if its a duplicate signature, dupeout the validator
  1138. func (cs *ConsensusState) tryAddVote(vote *types.Vote, peerKey string) error {
  1139. _, err := cs.addVote(vote, peerKey)
  1140. if err != nil {
  1141. // If the vote height is off, we'll just ignore it,
  1142. // But if it's a conflicting sig, broadcast evidence tx for slashing.
  1143. // If it's otherwise invalid, punish peer.
  1144. if err == ErrVoteHeightMismatch {
  1145. return err
  1146. } else if _, ok := err.(*types.ErrVoteConflictingVotes); ok {
  1147. if peerKey == "" {
  1148. log.Warn("Found conflicting vote from ourselves. Did you unsafe_reset a validator?", "height", vote.Height, "round", vote.Round, "type", vote.Type)
  1149. return err
  1150. }
  1151. log.Warn("Found conflicting vote. Publish evidence (TODO)")
  1152. /* TODO
  1153. evidenceTx := &types.DupeoutTx{
  1154. Address: address,
  1155. VoteA: *errDupe.VoteA,
  1156. VoteB: *errDupe.VoteB,
  1157. }
  1158. cs.mempool.BroadcastTx(struct{???}{evidenceTx}) // shouldn't need to check returned err
  1159. */
  1160. return err
  1161. } else {
  1162. // Probably an invalid signature. Bad peer.
  1163. log.Warn("Error attempting to add vote", "error", err)
  1164. return ErrAddingVote
  1165. }
  1166. }
  1167. return nil
  1168. }
  1169. //-----------------------------------------------------------------------------
  1170. func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool, err error) {
  1171. log.Debug("addVote", "voteHeight", vote.Height, "voteType", vote.Type, "csHeight", cs.Height)
  1172. // A precommit for the previous height?
  1173. // These come in while we wait timeoutCommit
  1174. if vote.Height+1 == cs.Height {
  1175. if !(cs.Step == RoundStepNewHeight && vote.Type == types.VoteTypePrecommit) {
  1176. // TODO: give the reason ..
  1177. // fmt.Errorf("tryAddVote: Wrong height, not a LastCommit straggler commit.")
  1178. return added, ErrVoteHeightMismatch
  1179. }
  1180. added, err = cs.LastCommit.AddVote(vote)
  1181. if added {
  1182. log.Info(Fmt("Added to lastPrecommits: %v", cs.LastCommit.StringShort()))
  1183. types.FireEventVote(cs.evsw, types.EventDataVote{vote})
  1184. if cs.LastCommit.HasAll() {
  1185. // if we have all the votes now,
  1186. // schedule the timeoutCommit to happen right away
  1187. // NOTE: this won't apply if only one validator
  1188. // cs.scheduleTimeout(time.Duration(0), cs.Height, 0, RoundStepNewHeight)
  1189. cs.enterNewRound(cs.Height, 0)
  1190. }
  1191. }
  1192. return
  1193. }
  1194. // A prevote/precommit for this height?
  1195. if vote.Height == cs.Height {
  1196. height := cs.Height
  1197. added, err = cs.Votes.AddVote(vote, peerKey)
  1198. if added {
  1199. types.FireEventVote(cs.evsw, types.EventDataVote{vote})
  1200. switch vote.Type {
  1201. case types.VoteTypePrevote:
  1202. prevotes := cs.Votes.Prevotes(vote.Round)
  1203. log.Info("Added to prevote", "vote", vote, "prevotes", prevotes.StringShort())
  1204. // First, unlock if prevotes is a valid POL.
  1205. // >> lockRound < POLRound <= unlockOrChangeLockRound (see spec)
  1206. // NOTE: If (lockRound < POLRound) but !(POLRound <= unlockOrChangeLockRound),
  1207. // we'll still enterNewRound(H,vote.R) and enterPrecommit(H,vote.R) to process it
  1208. // there.
  1209. if (cs.LockedBlock != nil) && (cs.LockedRound < vote.Round) && (vote.Round <= cs.Round) {
  1210. blockID, ok := prevotes.TwoThirdsMajority()
  1211. if ok && !cs.LockedBlock.HashesTo(blockID.Hash) {
  1212. log.Notice("Unlocking because of POL.", "lockedRound", cs.LockedRound, "POLRound", vote.Round)
  1213. cs.LockedRound = 0
  1214. cs.LockedBlock = nil
  1215. cs.LockedBlockParts = nil
  1216. types.FireEventUnlock(cs.evsw, cs.RoundStateEvent())
  1217. }
  1218. }
  1219. if cs.Round <= vote.Round && prevotes.HasTwoThirdsAny() {
  1220. // Round-skip over to PrevoteWait or goto Precommit.
  1221. cs.enterNewRound(height, vote.Round) // if the vote is ahead of us
  1222. if prevotes.HasTwoThirdsMajority() {
  1223. cs.enterPrecommit(height, vote.Round)
  1224. } else {
  1225. cs.enterPrevote(height, vote.Round) // if the vote is ahead of us
  1226. cs.enterPrevoteWait(height, vote.Round)
  1227. }
  1228. } else if cs.Proposal != nil && 0 <= cs.Proposal.POLRound && cs.Proposal.POLRound == vote.Round {
  1229. // If the proposal is now complete, enter prevote of cs.Round.
  1230. if cs.isProposalComplete() {
  1231. cs.enterPrevote(height, cs.Round)
  1232. }
  1233. }
  1234. case types.VoteTypePrecommit:
  1235. precommits := cs.Votes.Precommits(vote.Round)
  1236. log.Info("Added to precommit", "vote", vote, "precommits", precommits.StringShort())
  1237. blockID, ok := precommits.TwoThirdsMajority()
  1238. if ok {
  1239. if len(blockID.Hash) == 0 {
  1240. cs.enterNewRound(height, vote.Round+1)
  1241. } else {
  1242. cs.enterNewRound(height, vote.Round)
  1243. cs.enterPrecommit(height, vote.Round)
  1244. cs.enterCommit(height, vote.Round)
  1245. }
  1246. } else if cs.Round <= vote.Round && precommits.HasTwoThirdsAny() {
  1247. cs.enterNewRound(height, vote.Round)
  1248. cs.enterPrecommit(height, vote.Round)
  1249. cs.enterPrecommitWait(height, vote.Round)
  1250. }
  1251. default:
  1252. PanicSanity(Fmt("Unexpected vote type %X", vote.Type)) // Should not happen.
  1253. }
  1254. }
  1255. // Either duplicate, or error upon cs.Votes.AddByIndex()
  1256. return
  1257. } else {
  1258. err = ErrVoteHeightMismatch
  1259. }
  1260. // Height mismatch, bad peer?
  1261. log.Info("Vote ignored and not added", "voteHeight", vote.Height, "csHeight", cs.Height, "err", err)
  1262. return
  1263. }
  1264. func (cs *ConsensusState) signVote(type_ byte, hash []byte, header types.PartSetHeader) (*types.Vote, error) {
  1265. addr := cs.privValidator.GetAddress()
  1266. valIndex, _ := cs.Validators.GetByAddress(addr)
  1267. vote := &types.Vote{
  1268. ValidatorAddress: addr,
  1269. ValidatorIndex: valIndex,
  1270. Height: cs.Height,
  1271. Round: cs.Round,
  1272. Type: type_,
  1273. BlockID: types.BlockID{hash, header},
  1274. }
  1275. err := cs.privValidator.SignVote(cs.state.ChainID, vote)
  1276. return vote, err
  1277. }
  1278. // sign the vote and publish on internalMsgQueue
  1279. func (cs *ConsensusState) signAddVote(type_ byte, hash []byte, header types.PartSetHeader) *types.Vote {
  1280. // if we don't have a key or we're not in the validator set, do nothing
  1281. if cs.privValidator == nil || !cs.Validators.HasAddress(cs.privValidator.GetAddress()) {
  1282. return nil
  1283. }
  1284. vote, err := cs.signVote(type_, hash, header)
  1285. if err == nil {
  1286. cs.sendInternalMessage(msgInfo{&VoteMessage{vote}, ""})
  1287. log.Info("Signed and pushed vote", "height", cs.Height, "round", cs.Round, "vote", vote, "error", err)
  1288. return vote
  1289. } else {
  1290. //if !cs.replayMode {
  1291. log.Warn("Error signing vote", "height", cs.Height, "round", cs.Round, "vote", vote, "error", err)
  1292. //}
  1293. return nil
  1294. }
  1295. }
  1296. //---------------------------------------------------------
  1297. func CompareHRS(h1, r1 int, s1 RoundStepType, h2, r2 int, s2 RoundStepType) int {
  1298. if h1 < h2 {
  1299. return -1
  1300. } else if h1 > h2 {
  1301. return 1
  1302. }
  1303. if r1 < r2 {
  1304. return -1
  1305. } else if r1 > r2 {
  1306. return 1
  1307. }
  1308. if s1 < s2 {
  1309. return -1
  1310. } else if s1 > s2 {
  1311. return 1
  1312. }
  1313. return 0
  1314. }