You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

533 lines
17 KiB

  1. package consensus
  2. import (
  3. "bytes"
  4. "fmt"
  5. "hash/crc32"
  6. "io"
  7. "reflect"
  8. "time"
  9. abci "github.com/tendermint/tendermint/abci/types"
  10. "github.com/tendermint/tendermint/crypto/merkle"
  11. "github.com/tendermint/tendermint/libs/log"
  12. "github.com/tendermint/tendermint/proxy"
  13. sm "github.com/tendermint/tendermint/state"
  14. "github.com/tendermint/tendermint/types"
  15. )
  16. var crc32c = crc32.MakeTable(crc32.Castagnoli)
  17. // Functionality to replay blocks and messages on recovery from a crash.
  18. // There are two general failure scenarios:
  19. //
  20. // 1. failure during consensus
  21. // 2. failure while applying the block
  22. //
  23. // The former is handled by the WAL, the latter by the proxyApp Handshake on
  24. // restart, which ultimately hands off the work to the WAL.
  25. //-----------------------------------------
  26. // 1. Recover from failure during consensus
  27. // (by replaying messages from the WAL)
  28. //-----------------------------------------
  29. // Unmarshal and apply a single message to the consensus state as if it were
  30. // received in receiveRoutine. Lines that start with "#" are ignored.
  31. // NOTE: receiveRoutine should not be running.
  32. func (cs *State) readReplayMessage(msg *TimedWALMessage, newStepSub types.Subscription) error {
  33. // Skip meta messages which exist for demarcating boundaries.
  34. if _, ok := msg.Msg.(EndHeightMessage); ok {
  35. return nil
  36. }
  37. // for logging
  38. switch m := msg.Msg.(type) {
  39. case types.EventDataRoundState:
  40. cs.Logger.Info("Replay: New Step", "height", m.Height, "round", m.Round, "step", m.Step)
  41. // these are playback checks
  42. ticker := time.After(time.Second * 2)
  43. if newStepSub != nil {
  44. select {
  45. case stepMsg := <-newStepSub.Out():
  46. m2 := stepMsg.Data().(types.EventDataRoundState)
  47. if m.Height != m2.Height || m.Round != m2.Round || m.Step != m2.Step {
  48. return fmt.Errorf("roundState mismatch. Got %v; Expected %v", m2, m)
  49. }
  50. case <-newStepSub.Cancelled():
  51. return fmt.Errorf("failed to read off newStepSub.Out(). newStepSub was cancelled")
  52. case <-ticker:
  53. return fmt.Errorf("failed to read off newStepSub.Out()")
  54. }
  55. }
  56. case msgInfo:
  57. peerID := m.PeerID
  58. if peerID == "" {
  59. peerID = "local"
  60. }
  61. switch msg := m.Msg.(type) {
  62. case *ProposalMessage:
  63. p := msg.Proposal
  64. cs.Logger.Info("Replay: Proposal", "height", p.Height, "round", p.Round, "header",
  65. p.BlockID.PartSetHeader, "pol", p.POLRound, "peer", peerID)
  66. case *BlockPartMessage:
  67. cs.Logger.Info("Replay: BlockPart", "height", msg.Height, "round", msg.Round, "peer", peerID)
  68. case *VoteMessage:
  69. v := msg.Vote
  70. cs.Logger.Info("Replay: Vote", "height", v.Height, "round", v.Round, "type", v.Type,
  71. "blockID", v.BlockID, "peer", peerID)
  72. }
  73. cs.handleMsg(m)
  74. case timeoutInfo:
  75. cs.Logger.Info("Replay: Timeout", "height", m.Height, "round", m.Round, "step", m.Step, "dur", m.Duration)
  76. cs.handleTimeout(m, cs.RoundState)
  77. default:
  78. return fmt.Errorf("replay: Unknown TimedWALMessage type: %v", reflect.TypeOf(msg.Msg))
  79. }
  80. return nil
  81. }
  82. // Replay only those messages since the last block. `timeoutRoutine` should
  83. // run concurrently to read off tickChan.
  84. func (cs *State) catchupReplay(csHeight int64) error {
  85. // Set replayMode to true so we don't log signing errors.
  86. cs.replayMode = true
  87. defer func() { cs.replayMode = false }()
  88. // Ensure that #ENDHEIGHT for this height doesn't exist.
  89. // NOTE: This is just a sanity check. As far as we know things work fine
  90. // without it, and Handshake could reuse State if it weren't for
  91. // this check (since we can crash after writing #ENDHEIGHT).
  92. //
  93. // Ignore data corruption errors since this is a sanity check.
  94. gr, found, err := cs.wal.SearchForEndHeight(csHeight, &WALSearchOptions{IgnoreDataCorruptionErrors: true})
  95. if err != nil {
  96. return err
  97. }
  98. if gr != nil {
  99. if err := gr.Close(); err != nil {
  100. return err
  101. }
  102. }
  103. if found {
  104. return fmt.Errorf("wal should not contain #ENDHEIGHT %d", csHeight)
  105. }
  106. // Search for last height marker.
  107. //
  108. // Ignore data corruption errors in previous heights because we only care about last height
  109. if csHeight < cs.state.InitialHeight {
  110. return fmt.Errorf("cannot replay height %v, below initial height %v", csHeight, cs.state.InitialHeight)
  111. }
  112. endHeight := csHeight - 1
  113. if csHeight == cs.state.InitialHeight {
  114. endHeight = 0
  115. }
  116. gr, found, err = cs.wal.SearchForEndHeight(endHeight, &WALSearchOptions{IgnoreDataCorruptionErrors: true})
  117. if err == io.EOF {
  118. cs.Logger.Error("Replay: wal.group.Search returned EOF", "#ENDHEIGHT", endHeight)
  119. } else if err != nil {
  120. return err
  121. }
  122. if !found {
  123. return fmt.Errorf("cannot replay height %d. WAL does not contain #ENDHEIGHT for %d", csHeight, endHeight)
  124. }
  125. defer gr.Close()
  126. cs.Logger.Info("Catchup by replaying consensus messages", "height", csHeight)
  127. var msg *TimedWALMessage
  128. dec := WALDecoder{gr}
  129. LOOP:
  130. for {
  131. msg, err = dec.Decode()
  132. switch {
  133. case err == io.EOF:
  134. break LOOP
  135. case IsDataCorruptionError(err):
  136. cs.Logger.Error("data has been corrupted in last height of consensus WAL", "err", err, "height", csHeight)
  137. return err
  138. case err != nil:
  139. return err
  140. }
  141. // NOTE: since the priv key is set when the msgs are received
  142. // it will attempt to eg double sign but we can just ignore it
  143. // since the votes will be replayed and we'll get to the next step
  144. if err := cs.readReplayMessage(msg, nil); err != nil {
  145. return err
  146. }
  147. }
  148. cs.Logger.Info("Replay: Done")
  149. return nil
  150. }
  151. //--------------------------------------------------------------------------------
  152. // Parses marker lines of the form:
  153. // #ENDHEIGHT: 12345
  154. /*
  155. func makeHeightSearchFunc(height int64) auto.SearchFunc {
  156. return func(line string) (int, error) {
  157. line = strings.TrimRight(line, "\n")
  158. parts := strings.Split(line, " ")
  159. if len(parts) != 2 {
  160. return -1, errors.New("line did not have 2 parts")
  161. }
  162. i, err := strconv.Atoi(parts[1])
  163. if err != nil {
  164. return -1, errors.New("failed to parse INFO: " + err.Error())
  165. }
  166. if height < i {
  167. return 1, nil
  168. } else if height == i {
  169. return 0, nil
  170. } else {
  171. return -1, nil
  172. }
  173. }
  174. }*/
  175. //---------------------------------------------------
  176. // 2. Recover from failure while applying the block.
  177. // (by handshaking with the app to figure out where
  178. // we were last, and using the WAL to recover there.)
  179. //---------------------------------------------------
  180. type Handshaker struct {
  181. stateStore sm.Store
  182. initialState sm.State
  183. store sm.BlockStore
  184. eventBus types.BlockEventPublisher
  185. genDoc *types.GenesisDoc
  186. logger log.Logger
  187. nBlocks int // number of blocks applied to the state
  188. }
  189. func NewHandshaker(stateStore sm.Store, state sm.State,
  190. store sm.BlockStore, genDoc *types.GenesisDoc) *Handshaker {
  191. return &Handshaker{
  192. stateStore: stateStore,
  193. initialState: state,
  194. store: store,
  195. eventBus: types.NopEventBus{},
  196. genDoc: genDoc,
  197. logger: log.NewNopLogger(),
  198. nBlocks: 0,
  199. }
  200. }
  201. func (h *Handshaker) SetLogger(l log.Logger) {
  202. h.logger = l
  203. }
  204. // SetEventBus - sets the event bus for publishing block related events.
  205. // If not called, it defaults to types.NopEventBus.
  206. func (h *Handshaker) SetEventBus(eventBus types.BlockEventPublisher) {
  207. h.eventBus = eventBus
  208. }
  209. // NBlocks returns the number of blocks applied to the state.
  210. func (h *Handshaker) NBlocks() int {
  211. return h.nBlocks
  212. }
  213. // TODO: retry the handshake/replay if it fails ?
  214. func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error {
  215. // Handshake is done via ABCI Info on the query conn.
  216. res, err := proxyApp.Query().InfoSync(proxy.RequestInfo)
  217. if err != nil {
  218. return fmt.Errorf("error calling Info: %v", err)
  219. }
  220. blockHeight := res.LastBlockHeight
  221. if blockHeight < 0 {
  222. return fmt.Errorf("got a negative last block height (%d) from the app", blockHeight)
  223. }
  224. appHash := res.LastBlockAppHash
  225. h.logger.Info("ABCI Handshake App Info",
  226. "height", blockHeight,
  227. "hash", fmt.Sprintf("%X", appHash),
  228. "software-version", res.Version,
  229. "protocol-version", res.AppVersion,
  230. )
  231. // Only set the version if there is no existing state.
  232. if h.initialState.LastBlockHeight == 0 {
  233. h.initialState.Version.Consensus.App = res.AppVersion
  234. }
  235. // Replay blocks up to the latest in the blockstore.
  236. _, err = h.ReplayBlocks(h.initialState, appHash, blockHeight, proxyApp)
  237. if err != nil {
  238. return fmt.Errorf("error on replay: %v", err)
  239. }
  240. h.logger.Info("Completed ABCI Handshake - Tendermint and App are synced",
  241. "appHeight", blockHeight, "appHash", fmt.Sprintf("%X", appHash))
  242. // TODO: (on restart) replay mempool
  243. return nil
  244. }
  245. // ReplayBlocks replays all blocks since appBlockHeight and ensures the result
  246. // matches the current state.
  247. // Returns the final AppHash or an error.
  248. func (h *Handshaker) ReplayBlocks(
  249. state sm.State,
  250. appHash []byte,
  251. appBlockHeight int64,
  252. proxyApp proxy.AppConns,
  253. ) ([]byte, error) {
  254. storeBlockBase := h.store.Base()
  255. storeBlockHeight := h.store.Height()
  256. stateBlockHeight := state.LastBlockHeight
  257. h.logger.Info(
  258. "ABCI Replay Blocks",
  259. "appHeight",
  260. appBlockHeight,
  261. "storeHeight",
  262. storeBlockHeight,
  263. "stateHeight",
  264. stateBlockHeight)
  265. // If appBlockHeight == 0 it means that we are at genesis and hence should send InitChain.
  266. if appBlockHeight == 0 {
  267. validators := make([]*types.Validator, len(h.genDoc.Validators))
  268. for i, val := range h.genDoc.Validators {
  269. validators[i] = types.NewValidator(val.PubKey, val.Power)
  270. }
  271. validatorSet := types.NewValidatorSet(validators)
  272. nextVals := types.TM2PB.ValidatorUpdates(validatorSet)
  273. csParams := types.TM2PB.ConsensusParams(h.genDoc.ConsensusParams)
  274. req := abci.RequestInitChain{
  275. Time: h.genDoc.GenesisTime,
  276. ChainId: h.genDoc.ChainID,
  277. InitialHeight: h.genDoc.InitialHeight,
  278. ConsensusParams: csParams,
  279. Validators: nextVals,
  280. AppStateBytes: h.genDoc.AppState,
  281. }
  282. res, err := proxyApp.Consensus().InitChainSync(req)
  283. if err != nil {
  284. return nil, err
  285. }
  286. appHash = res.AppHash
  287. if stateBlockHeight == 0 { // we only update state when we are in initial state
  288. // If the app did not return an app hash, we keep the one set from the genesis doc in
  289. // the state. We don't set appHash since we don't want the genesis doc app hash
  290. // recorded in the genesis block. We should probably just remove GenesisDoc.AppHash.
  291. if len(res.AppHash) > 0 {
  292. state.AppHash = res.AppHash
  293. }
  294. // If the app returned validators or consensus params, update the state.
  295. if len(res.Validators) > 0 {
  296. vals, err := types.PB2TM.ValidatorUpdates(res.Validators)
  297. if err != nil {
  298. return nil, err
  299. }
  300. state.Validators = types.NewValidatorSet(vals)
  301. state.NextValidators = types.NewValidatorSet(vals).CopyIncrementProposerPriority(1)
  302. } else if len(h.genDoc.Validators) == 0 {
  303. // If validator set is not set in genesis and still empty after InitChain, exit.
  304. return nil, fmt.Errorf("validator set is nil in genesis and still empty after InitChain")
  305. }
  306. if res.ConsensusParams != nil {
  307. state.ConsensusParams = types.UpdateConsensusParams(state.ConsensusParams, res.ConsensusParams)
  308. state.Version.Consensus.App = state.ConsensusParams.Version.AppVersion
  309. }
  310. // We update the last results hash with the empty hash, to conform with RFC-6962.
  311. state.LastResultsHash = merkle.HashFromByteSlices(nil)
  312. if err := h.stateStore.Save(state); err != nil {
  313. return nil, err
  314. }
  315. }
  316. }
  317. // First handle edge cases and constraints on the storeBlockHeight and storeBlockBase.
  318. switch {
  319. case storeBlockHeight == 0:
  320. assertAppHashEqualsOneFromState(appHash, state)
  321. return appHash, nil
  322. case appBlockHeight == 0 && state.InitialHeight < storeBlockBase:
  323. // the app has no state, and the block store is truncated above the initial height
  324. return appHash, sm.ErrAppBlockHeightTooLow{AppHeight: appBlockHeight, StoreBase: storeBlockBase}
  325. case appBlockHeight > 0 && appBlockHeight < storeBlockBase-1:
  326. // the app is too far behind truncated store (can be 1 behind since we replay the next)
  327. return appHash, sm.ErrAppBlockHeightTooLow{AppHeight: appBlockHeight, StoreBase: storeBlockBase}
  328. case storeBlockHeight < appBlockHeight:
  329. // the app should never be ahead of the store (but this is under app's control)
  330. return appHash, sm.ErrAppBlockHeightTooHigh{CoreHeight: storeBlockHeight, AppHeight: appBlockHeight}
  331. case storeBlockHeight < stateBlockHeight:
  332. // the state should never be ahead of the store (this is under tendermint's control)
  333. panic(fmt.Sprintf("StateBlockHeight (%d) > StoreBlockHeight (%d)", stateBlockHeight, storeBlockHeight))
  334. case storeBlockHeight > stateBlockHeight+1:
  335. // store should be at most one ahead of the state (this is under tendermint's control)
  336. panic(fmt.Sprintf("StoreBlockHeight (%d) > StateBlockHeight + 1 (%d)", storeBlockHeight, stateBlockHeight+1))
  337. }
  338. var err error
  339. // Now either store is equal to state, or one ahead.
  340. // For each, consider all cases of where the app could be, given app <= store
  341. if storeBlockHeight == stateBlockHeight {
  342. // Tendermint ran Commit and saved the state.
  343. // Either the app is asking for replay, or we're all synced up.
  344. if appBlockHeight < storeBlockHeight {
  345. // the app is behind, so replay blocks, but no need to go through WAL (state is already synced to store)
  346. return h.replayBlocks(state, proxyApp, appBlockHeight, storeBlockHeight, false)
  347. } else if appBlockHeight == storeBlockHeight {
  348. // We're good!
  349. assertAppHashEqualsOneFromState(appHash, state)
  350. return appHash, nil
  351. }
  352. } else if storeBlockHeight == stateBlockHeight+1 {
  353. // We saved the block in the store but haven't updated the state,
  354. // so we'll need to replay a block using the WAL.
  355. switch {
  356. case appBlockHeight < stateBlockHeight:
  357. // the app is further behind than it should be, so replay blocks
  358. // but leave the last block to go through the WAL
  359. return h.replayBlocks(state, proxyApp, appBlockHeight, storeBlockHeight, true)
  360. case appBlockHeight == stateBlockHeight:
  361. // We haven't run Commit (both the state and app are one block behind),
  362. // so replayBlock with the real app.
  363. // NOTE: We could instead use the cs.WAL on cs.Start,
  364. // but we'd have to allow the WAL to replay a block that wrote it's #ENDHEIGHT
  365. h.logger.Info("Replay last block using real app")
  366. state, err = h.replayBlock(state, storeBlockHeight, proxyApp.Consensus())
  367. return state.AppHash, err
  368. case appBlockHeight == storeBlockHeight:
  369. // We ran Commit, but didn't save the state, so replayBlock with mock app.
  370. abciResponses, err := h.stateStore.LoadABCIResponses(storeBlockHeight)
  371. if err != nil {
  372. return nil, err
  373. }
  374. mockApp := newMockProxyApp(appHash, abciResponses)
  375. h.logger.Info("Replay last block using mock app")
  376. state, err = h.replayBlock(state, storeBlockHeight, mockApp)
  377. return state.AppHash, err
  378. }
  379. }
  380. panic(fmt.Sprintf("uncovered case! appHeight: %d, storeHeight: %d, stateHeight: %d",
  381. appBlockHeight, storeBlockHeight, stateBlockHeight))
  382. }
  383. func (h *Handshaker) replayBlocks(
  384. state sm.State,
  385. proxyApp proxy.AppConns,
  386. appBlockHeight,
  387. storeBlockHeight int64,
  388. mutateState bool) ([]byte, error) {
  389. // App is further behind than it should be, so we need to replay blocks.
  390. // We replay all blocks from appBlockHeight+1.
  391. //
  392. // Note that we don't have an old version of the state,
  393. // so we by-pass state validation/mutation using sm.ExecCommitBlock.
  394. // This also means we won't be saving validator sets if they change during this period.
  395. // TODO: Load the historical information to fix this and just use state.ApplyBlock
  396. //
  397. // If mutateState == true, the final block is replayed with h.replayBlock()
  398. var appHash []byte
  399. var err error
  400. finalBlock := storeBlockHeight
  401. if mutateState {
  402. finalBlock--
  403. }
  404. firstBlock := appBlockHeight + 1
  405. if firstBlock == 1 {
  406. firstBlock = state.InitialHeight
  407. }
  408. for i := firstBlock; i <= finalBlock; i++ {
  409. h.logger.Info("Applying block", "height", i)
  410. block := h.store.LoadBlock(i)
  411. // Extra check to ensure the app was not changed in a way it shouldn't have.
  412. if len(appHash) > 0 {
  413. assertAppHashEqualsOneFromBlock(appHash, block)
  414. }
  415. appHash, err = sm.ExecCommitBlock(proxyApp.Consensus(), block, h.logger, h.stateStore, h.genDoc.InitialHeight)
  416. if err != nil {
  417. return nil, err
  418. }
  419. h.nBlocks++
  420. }
  421. if mutateState {
  422. // sync the final block
  423. state, err = h.replayBlock(state, storeBlockHeight, proxyApp.Consensus())
  424. if err != nil {
  425. return nil, err
  426. }
  427. appHash = state.AppHash
  428. }
  429. assertAppHashEqualsOneFromState(appHash, state)
  430. return appHash, nil
  431. }
  432. // ApplyBlock on the proxyApp with the last block.
  433. func (h *Handshaker) replayBlock(state sm.State, height int64, proxyApp proxy.AppConnConsensus) (sm.State, error) {
  434. block := h.store.LoadBlock(height)
  435. meta := h.store.LoadBlockMeta(height)
  436. // Use stubs for both mempool and evidence pool since no transactions nor
  437. // evidence are needed here - block already exists.
  438. blockExec := sm.NewBlockExecutor(h.stateStore, h.logger, proxyApp, emptyMempool{}, sm.EmptyEvidencePool{})
  439. blockExec.SetEventBus(h.eventBus)
  440. var err error
  441. state, _, err = blockExec.ApplyBlock(state, meta.BlockID, block)
  442. if err != nil {
  443. return sm.State{}, err
  444. }
  445. h.nBlocks++
  446. return state, nil
  447. }
  448. func assertAppHashEqualsOneFromBlock(appHash []byte, block *types.Block) {
  449. if !bytes.Equal(appHash, block.AppHash) {
  450. panic(fmt.Sprintf(`block.AppHash does not match AppHash after replay. Got %X, expected %X.
  451. Block: %v
  452. `,
  453. appHash, block.AppHash, block))
  454. }
  455. }
  456. func assertAppHashEqualsOneFromState(appHash []byte, state sm.State) {
  457. if !bytes.Equal(appHash, state.AppHash) {
  458. panic(fmt.Sprintf(`state.AppHash does not match AppHash after replay. Got
  459. %X, expected %X.
  460. State: %v
  461. Did you reset Tendermint without resetting your application's data?`,
  462. appHash, state.AppHash, state))
  463. }
  464. }