You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

832 lines
23 KiB

  1. package statesync
  2. import (
  3. "bytes"
  4. "context"
  5. "errors"
  6. "fmt"
  7. "reflect"
  8. "runtime/debug"
  9. "sort"
  10. "time"
  11. abci "github.com/tendermint/tendermint/abci/types"
  12. "github.com/tendermint/tendermint/config"
  13. tmsync "github.com/tendermint/tendermint/internal/libs/sync"
  14. "github.com/tendermint/tendermint/internal/p2p"
  15. "github.com/tendermint/tendermint/libs/log"
  16. "github.com/tendermint/tendermint/libs/service"
  17. ssproto "github.com/tendermint/tendermint/proto/tendermint/statesync"
  18. "github.com/tendermint/tendermint/proxy"
  19. sm "github.com/tendermint/tendermint/state"
  20. "github.com/tendermint/tendermint/store"
  21. "github.com/tendermint/tendermint/types"
  22. )
  23. var (
  24. _ service.Service = (*Reactor)(nil)
  25. _ p2p.Wrapper = (*ssproto.Message)(nil)
  26. // ChannelShims contains a map of ChannelDescriptorShim objects, where each
  27. // object wraps a reference to a legacy p2p ChannelDescriptor and the corresponding
  28. // p2p proto.Message the new p2p Channel is responsible for handling.
  29. //
  30. //
  31. // TODO: Remove once p2p refactor is complete.
  32. // ref: https://github.com/tendermint/tendermint/issues/5670
  33. ChannelShims = map[p2p.ChannelID]*p2p.ChannelDescriptorShim{
  34. SnapshotChannel: {
  35. MsgType: new(ssproto.Message),
  36. Descriptor: &p2p.ChannelDescriptor{
  37. ID: byte(SnapshotChannel),
  38. Priority: 6,
  39. SendQueueCapacity: 10,
  40. RecvMessageCapacity: snapshotMsgSize,
  41. RecvBufferCapacity: 128,
  42. MaxSendBytes: 400,
  43. },
  44. },
  45. ChunkChannel: {
  46. MsgType: new(ssproto.Message),
  47. Descriptor: &p2p.ChannelDescriptor{
  48. ID: byte(ChunkChannel),
  49. Priority: 3,
  50. SendQueueCapacity: 4,
  51. RecvMessageCapacity: chunkMsgSize,
  52. RecvBufferCapacity: 128,
  53. MaxSendBytes: 400,
  54. },
  55. },
  56. LightBlockChannel: {
  57. MsgType: new(ssproto.Message),
  58. Descriptor: &p2p.ChannelDescriptor{
  59. ID: byte(LightBlockChannel),
  60. Priority: 2,
  61. SendQueueCapacity: 10,
  62. RecvMessageCapacity: lightBlockMsgSize,
  63. RecvBufferCapacity: 128,
  64. MaxSendBytes: 400,
  65. },
  66. },
  67. }
  68. )
  69. const (
  70. // SnapshotChannel exchanges snapshot metadata
  71. SnapshotChannel = p2p.ChannelID(0x60)
  72. // ChunkChannel exchanges chunk contents
  73. ChunkChannel = p2p.ChannelID(0x61)
  74. // LightBlockChannel exchanges light blocks
  75. LightBlockChannel = p2p.ChannelID(0x62)
  76. // recentSnapshots is the number of recent snapshots to send and receive per peer.
  77. recentSnapshots = 10
  78. // snapshotMsgSize is the maximum size of a snapshotResponseMessage
  79. snapshotMsgSize = int(4e6) // ~4MB
  80. // chunkMsgSize is the maximum size of a chunkResponseMessage
  81. chunkMsgSize = int(16e6) // ~16MB
  82. // lightBlockMsgSize is the maximum size of a lightBlockResponseMessage
  83. lightBlockMsgSize = int(1e7) // ~10MB
  84. // lightBlockResponseTimeout is how long the dispatcher waits for a peer to
  85. // return a light block
  86. lightBlockResponseTimeout = 30 * time.Second
  87. // maxLightBlockRequestRetries is the amount of retries acceptable before
  88. // the backfill process aborts
  89. maxLightBlockRequestRetries = 20
  90. )
  91. // Reactor handles state sync, both restoring snapshots for the local node and
  92. // serving snapshots for other nodes.
  93. type Reactor struct {
  94. service.BaseService
  95. cfg config.StateSyncConfig
  96. stateStore sm.Store
  97. blockStore *store.BlockStore
  98. conn proxy.AppConnSnapshot
  99. connQuery proxy.AppConnQuery
  100. tempDir string
  101. snapshotCh *p2p.Channel
  102. chunkCh *p2p.Channel
  103. blockCh *p2p.Channel
  104. peerUpdates *p2p.PeerUpdates
  105. closeCh chan struct{}
  106. dispatcher *dispatcher
  107. // This will only be set when a state sync is in progress. It is used to feed
  108. // received snapshots and chunks into the sync.
  109. mtx tmsync.RWMutex
  110. syncer *syncer
  111. }
  112. // NewReactor returns a reference to a new state sync reactor, which implements
  113. // the service.Service interface. It accepts a logger, connections for snapshots
  114. // and querying, references to p2p Channels and a channel to listen for peer
  115. // updates on. Note, the reactor will close all p2p Channels when stopping.
  116. func NewReactor(
  117. cfg config.StateSyncConfig,
  118. logger log.Logger,
  119. conn proxy.AppConnSnapshot,
  120. connQuery proxy.AppConnQuery,
  121. snapshotCh, chunkCh, blockCh *p2p.Channel,
  122. peerUpdates *p2p.PeerUpdates,
  123. stateStore sm.Store,
  124. blockStore *store.BlockStore,
  125. tempDir string,
  126. ) *Reactor {
  127. r := &Reactor{
  128. cfg: cfg,
  129. conn: conn,
  130. connQuery: connQuery,
  131. snapshotCh: snapshotCh,
  132. chunkCh: chunkCh,
  133. blockCh: blockCh,
  134. peerUpdates: peerUpdates,
  135. closeCh: make(chan struct{}),
  136. tempDir: tempDir,
  137. dispatcher: newDispatcher(blockCh.Out, lightBlockResponseTimeout),
  138. stateStore: stateStore,
  139. blockStore: blockStore,
  140. }
  141. r.BaseService = *service.NewBaseService(logger, "StateSync", r)
  142. return r
  143. }
  144. // OnStart starts separate go routines for each p2p Channel and listens for
  145. // envelopes on each. In addition, it also listens for peer updates and handles
  146. // messages on that p2p channel accordingly. The caller must be sure to execute
  147. // OnStop to ensure the outbound p2p Channels are closed. No error is returned.
  148. func (r *Reactor) OnStart() error {
  149. // Listen for envelopes on the snapshot p2p Channel in a separate go-routine
  150. // as to not block or cause IO contention with the chunk p2p Channel. Note,
  151. // we do not launch a go-routine to handle individual envelopes as to not
  152. // have to deal with bounding workers or pools.
  153. go r.processSnapshotCh()
  154. // Listen for envelopes on the chunk p2p Channel in a separate go-routine
  155. // as to not block or cause IO contention with the snapshot p2p Channel. Note,
  156. // we do not launch a go-routine to handle individual envelopes as to not
  157. // have to deal with bounding workers or pools.
  158. go r.processChunkCh()
  159. go r.processBlockCh()
  160. go r.processPeerUpdates()
  161. r.dispatcher.start()
  162. return nil
  163. }
  164. // OnStop stops the reactor by signaling to all spawned goroutines to exit and
  165. // blocking until they all exit.
  166. func (r *Reactor) OnStop() {
  167. // tell the dispatcher to stop sending any more requests
  168. r.dispatcher.stop()
  169. // Close closeCh to signal to all spawned goroutines to gracefully exit. All
  170. // p2p Channels should execute Close().
  171. close(r.closeCh)
  172. // Wait for all p2p Channels to be closed before returning. This ensures we
  173. // can easily reason about synchronization of all p2p Channels and ensure no
  174. // panics will occur.
  175. <-r.snapshotCh.Done()
  176. <-r.chunkCh.Done()
  177. <-r.blockCh.Done()
  178. <-r.peerUpdates.Done()
  179. }
  180. // Sync runs a state sync, fetching snapshots and providing chunks to the
  181. // application. It also saves tendermint state and runs a backfill process to
  182. // retrieve the necessary amount of headers, commits and validators sets to be
  183. // able to process evidence and participate in consensus.
  184. func (r *Reactor) Sync(
  185. ctx context.Context,
  186. stateProvider StateProvider,
  187. discoveryTime time.Duration,
  188. ) (sm.State, error) {
  189. r.mtx.Lock()
  190. if r.syncer != nil {
  191. r.mtx.Unlock()
  192. return sm.State{}, errors.New("a state sync is already in progress")
  193. }
  194. r.syncer = newSyncer(
  195. r.cfg,
  196. r.Logger,
  197. r.conn,
  198. r.connQuery,
  199. stateProvider,
  200. r.snapshotCh.Out,
  201. r.chunkCh.Out,
  202. r.tempDir,
  203. )
  204. r.mtx.Unlock()
  205. requestSnapshotsHook := func() {
  206. // request snapshots from all currently connected peers
  207. r.snapshotCh.Out <- p2p.Envelope{
  208. Broadcast: true,
  209. Message: &ssproto.SnapshotsRequest{},
  210. }
  211. }
  212. state, commit, err := r.syncer.SyncAny(ctx, discoveryTime, requestSnapshotsHook)
  213. if err != nil {
  214. return sm.State{}, err
  215. }
  216. r.mtx.Lock()
  217. r.syncer = nil
  218. r.mtx.Unlock()
  219. err = r.stateStore.Bootstrap(state)
  220. if err != nil {
  221. return sm.State{}, fmt.Errorf("failed to bootstrap node with new state: %w", err)
  222. }
  223. err = r.blockStore.SaveSeenCommit(state.LastBlockHeight, commit)
  224. if err != nil {
  225. return sm.State{}, fmt.Errorf("failed to store last seen commit: %w", err)
  226. }
  227. return state, nil
  228. }
  229. // Backfill sequentially fetches, verifies and stores light blocks in reverse
  230. // order. It does not stop verifying blocks until reaching a block with a height
  231. // and time that is less or equal to the stopHeight and stopTime. The
  232. // trustedBlockID should be of the header at startHeight.
  233. func (r *Reactor) Backfill(state sm.State) error {
  234. params := state.ConsensusParams.Evidence
  235. stopHeight := state.LastBlockHeight - params.MaxAgeNumBlocks
  236. stopTime := state.LastBlockTime.Add(-params.MaxAgeDuration)
  237. // ensure that stop height doesn't go below the initial height
  238. if stopHeight < state.InitialHeight {
  239. stopHeight = state.InitialHeight
  240. // this essentially makes stop time a void criteria for termination
  241. stopTime = state.LastBlockTime
  242. }
  243. return r.backfill(
  244. context.Background(),
  245. state.ChainID,
  246. state.LastBlockHeight,
  247. stopHeight,
  248. state.InitialHeight,
  249. state.LastBlockID,
  250. stopTime,
  251. )
  252. }
  253. func (r *Reactor) backfill(
  254. ctx context.Context,
  255. chainID string,
  256. startHeight, stopHeight, initialHeight int64,
  257. trustedBlockID types.BlockID,
  258. stopTime time.Time,
  259. ) error {
  260. r.Logger.Info("starting backfill process...", "startHeight", startHeight,
  261. "stopHeight", stopHeight, "trustedBlockID", trustedBlockID)
  262. const sleepTime = 1 * time.Second
  263. var (
  264. lastValidatorSet *types.ValidatorSet
  265. lastChangeHeight int64 = startHeight
  266. )
  267. queue := newBlockQueue(startHeight, stopHeight, initialHeight, stopTime, maxLightBlockRequestRetries)
  268. // fetch light blocks across four workers. The aim with deploying concurrent
  269. // workers is to equate the network messaging time with the verification
  270. // time. Ideally we want the verification process to never have to be
  271. // waiting on blocks. If it takes 4s to retrieve a block and 1s to verify
  272. // it, then steady state involves four workers.
  273. for i := 0; i < int(r.cfg.Fetchers); i++ {
  274. ctxWithCancel, cancel := context.WithCancel(ctx)
  275. defer cancel()
  276. go func() {
  277. for {
  278. select {
  279. case height := <-queue.nextHeight():
  280. r.Logger.Debug("fetching next block", "height", height)
  281. lb, peer, err := r.dispatcher.LightBlock(ctxWithCancel, height)
  282. if errors.Is(err, context.Canceled) {
  283. return
  284. }
  285. if err != nil {
  286. queue.retry(height)
  287. if errors.Is(err, errNoConnectedPeers) {
  288. r.Logger.Info("backfill: no connected peers to fetch light blocks from; sleeping...",
  289. "sleepTime", sleepTime)
  290. time.Sleep(sleepTime)
  291. } else {
  292. // we don't punish the peer as it might just have not responded in time
  293. r.Logger.Info("backfill: error with fetching light block",
  294. "height", height, "err", err)
  295. }
  296. continue
  297. }
  298. if lb == nil {
  299. r.Logger.Info("backfill: peer didn't have block, fetching from another peer", "height", height)
  300. queue.retry(height)
  301. // As we are fetching blocks backwards, if this node doesn't have the block it likely doesn't
  302. // have any prior ones, thus we remove it from the peer list.
  303. r.dispatcher.removePeer(peer)
  304. continue
  305. }
  306. // run a validate basic. This checks the validator set and commit
  307. // hashes line up
  308. err = lb.ValidateBasic(chainID)
  309. if err != nil || lb.Height != height {
  310. r.Logger.Info("backfill: fetched light block failed validate basic, removing peer...",
  311. "err", err, "height", height)
  312. queue.retry(height)
  313. r.blockCh.Error <- p2p.PeerError{
  314. NodeID: peer,
  315. Err: fmt.Errorf("received invalid light block: %w", err),
  316. }
  317. continue
  318. }
  319. // add block to queue to be verified
  320. queue.add(lightBlockResponse{
  321. block: lb,
  322. peer: peer,
  323. })
  324. r.Logger.Debug("backfill: added light block to processing queue", "height", height)
  325. case <-queue.done():
  326. return
  327. }
  328. }
  329. }()
  330. }
  331. // verify all light blocks
  332. for {
  333. select {
  334. case <-r.closeCh:
  335. queue.close()
  336. return nil
  337. case <-ctx.Done():
  338. queue.close()
  339. return nil
  340. case resp := <-queue.verifyNext():
  341. // validate the header hash. We take the last block id of the
  342. // previous header (i.e. one height above) as the trusted hash which
  343. // we equate to. ValidatorsHash and CommitHash have already been
  344. // checked in the `ValidateBasic`
  345. if w, g := trustedBlockID.Hash, resp.block.Hash(); !bytes.Equal(w, g) {
  346. r.Logger.Info("received invalid light block. header hash doesn't match trusted LastBlockID",
  347. "trustedHash", w, "receivedHash", g, "height", resp.block.Height)
  348. r.blockCh.Error <- p2p.PeerError{
  349. NodeID: resp.peer,
  350. Err: fmt.Errorf("received invalid light block. Expected hash %v, got: %v", w, g),
  351. }
  352. queue.retry(resp.block.Height)
  353. continue
  354. }
  355. // save the signed headers
  356. err := r.blockStore.SaveSignedHeader(resp.block.SignedHeader, trustedBlockID)
  357. if err != nil {
  358. return err
  359. }
  360. // check if there has been a change in the validator set
  361. if lastValidatorSet != nil && !bytes.Equal(resp.block.Header.ValidatorsHash, resp.block.Header.NextValidatorsHash) {
  362. // save all the heights that the last validator set was the same
  363. err = r.stateStore.SaveValidatorSets(resp.block.Height+1, lastChangeHeight, lastValidatorSet)
  364. if err != nil {
  365. return err
  366. }
  367. // update the lastChangeHeight
  368. lastChangeHeight = resp.block.Height
  369. }
  370. trustedBlockID = resp.block.LastBlockID
  371. queue.success(resp.block.Height)
  372. r.Logger.Info("backfill: verified and stored light block", "height", resp.block.Height)
  373. lastValidatorSet = resp.block.ValidatorSet
  374. case <-queue.done():
  375. if err := queue.error(); err != nil {
  376. return err
  377. }
  378. // save the final batch of validators
  379. if err := r.stateStore.SaveValidatorSets(queue.terminal.Height, lastChangeHeight, lastValidatorSet); err != nil {
  380. return err
  381. }
  382. r.Logger.Info("successfully completed backfill process", "endHeight", queue.terminal.Height)
  383. return nil
  384. }
  385. }
  386. }
  387. // Dispatcher exposes the dispatcher so that a state provider can use it for
  388. // light client verification
  389. func (r *Reactor) Dispatcher() *dispatcher { //nolint:golint
  390. return r.dispatcher
  391. }
  392. // handleSnapshotMessage handles envelopes sent from peers on the
  393. // SnapshotChannel. It returns an error only if the Envelope.Message is unknown
  394. // for this channel. This should never be called outside of handleMessage.
  395. func (r *Reactor) handleSnapshotMessage(envelope p2p.Envelope) error {
  396. logger := r.Logger.With("peer", envelope.From)
  397. switch msg := envelope.Message.(type) {
  398. case *ssproto.SnapshotsRequest:
  399. snapshots, err := r.recentSnapshots(recentSnapshots)
  400. if err != nil {
  401. logger.Error("failed to fetch snapshots", "err", err)
  402. return nil
  403. }
  404. for _, snapshot := range snapshots {
  405. logger.Info(
  406. "advertising snapshot",
  407. "height", snapshot.Height,
  408. "format", snapshot.Format,
  409. "peer", envelope.From,
  410. )
  411. r.snapshotCh.Out <- p2p.Envelope{
  412. To: envelope.From,
  413. Message: &ssproto.SnapshotsResponse{
  414. Height: snapshot.Height,
  415. Format: snapshot.Format,
  416. Chunks: snapshot.Chunks,
  417. Hash: snapshot.Hash,
  418. Metadata: snapshot.Metadata,
  419. },
  420. }
  421. }
  422. case *ssproto.SnapshotsResponse:
  423. r.mtx.RLock()
  424. defer r.mtx.RUnlock()
  425. if r.syncer == nil {
  426. logger.Debug("received unexpected snapshot; no state sync in progress")
  427. return nil
  428. }
  429. logger.Debug("received snapshot", "height", msg.Height, "format", msg.Format)
  430. _, err := r.syncer.AddSnapshot(envelope.From, &snapshot{
  431. Height: msg.Height,
  432. Format: msg.Format,
  433. Chunks: msg.Chunks,
  434. Hash: msg.Hash,
  435. Metadata: msg.Metadata,
  436. })
  437. if err != nil {
  438. logger.Error(
  439. "failed to add snapshot",
  440. "height", msg.Height,
  441. "format", msg.Format,
  442. "err", err,
  443. "channel", r.snapshotCh.ID,
  444. )
  445. return nil
  446. }
  447. default:
  448. return fmt.Errorf("received unknown message: %T", msg)
  449. }
  450. return nil
  451. }
  452. // handleChunkMessage handles envelopes sent from peers on the ChunkChannel.
  453. // It returns an error only if the Envelope.Message is unknown for this channel.
  454. // This should never be called outside of handleMessage.
  455. func (r *Reactor) handleChunkMessage(envelope p2p.Envelope) error {
  456. switch msg := envelope.Message.(type) {
  457. case *ssproto.ChunkRequest:
  458. r.Logger.Debug(
  459. "received chunk request",
  460. "height", msg.Height,
  461. "format", msg.Format,
  462. "chunk", msg.Index,
  463. "peer", envelope.From,
  464. )
  465. resp, err := r.conn.LoadSnapshotChunkSync(context.Background(), abci.RequestLoadSnapshotChunk{
  466. Height: msg.Height,
  467. Format: msg.Format,
  468. Chunk: msg.Index,
  469. })
  470. if err != nil {
  471. r.Logger.Error(
  472. "failed to load chunk",
  473. "height", msg.Height,
  474. "format", msg.Format,
  475. "chunk", msg.Index,
  476. "err", err,
  477. "peer", envelope.From,
  478. )
  479. return nil
  480. }
  481. r.Logger.Debug(
  482. "sending chunk",
  483. "height", msg.Height,
  484. "format", msg.Format,
  485. "chunk", msg.Index,
  486. "peer", envelope.From,
  487. )
  488. r.chunkCh.Out <- p2p.Envelope{
  489. To: envelope.From,
  490. Message: &ssproto.ChunkResponse{
  491. Height: msg.Height,
  492. Format: msg.Format,
  493. Index: msg.Index,
  494. Chunk: resp.Chunk,
  495. Missing: resp.Chunk == nil,
  496. },
  497. }
  498. case *ssproto.ChunkResponse:
  499. r.mtx.RLock()
  500. defer r.mtx.RUnlock()
  501. if r.syncer == nil {
  502. r.Logger.Debug("received unexpected chunk; no state sync in progress", "peer", envelope.From)
  503. return nil
  504. }
  505. r.Logger.Debug(
  506. "received chunk; adding to sync",
  507. "height", msg.Height,
  508. "format", msg.Format,
  509. "chunk", msg.Index,
  510. "peer", envelope.From,
  511. )
  512. _, err := r.syncer.AddChunk(&chunk{
  513. Height: msg.Height,
  514. Format: msg.Format,
  515. Index: msg.Index,
  516. Chunk: msg.Chunk,
  517. Sender: envelope.From,
  518. })
  519. if err != nil {
  520. r.Logger.Error(
  521. "failed to add chunk",
  522. "height", msg.Height,
  523. "format", msg.Format,
  524. "chunk", msg.Index,
  525. "err", err,
  526. "peer", envelope.From,
  527. )
  528. return nil
  529. }
  530. default:
  531. return fmt.Errorf("received unknown message: %T", msg)
  532. }
  533. return nil
  534. }
  535. func (r *Reactor) handleLightBlockMessage(envelope p2p.Envelope) error {
  536. switch msg := envelope.Message.(type) {
  537. case *ssproto.LightBlockRequest:
  538. r.Logger.Info("received light block request", "height", msg.Height)
  539. lb, err := r.fetchLightBlock(msg.Height)
  540. if err != nil {
  541. r.Logger.Error("failed to retrieve light block", "err", err, "height", msg.Height)
  542. return err
  543. }
  544. lbproto, err := lb.ToProto()
  545. if err != nil {
  546. r.Logger.Error("marshaling light block to proto", "err", err)
  547. return nil
  548. }
  549. // NOTE: If we don't have the light block we will send a nil light block
  550. // back to the requested node, indicating that we don't have it.
  551. r.blockCh.Out <- p2p.Envelope{
  552. To: envelope.From,
  553. Message: &ssproto.LightBlockResponse{
  554. LightBlock: lbproto,
  555. },
  556. }
  557. case *ssproto.LightBlockResponse:
  558. if err := r.dispatcher.respond(msg.LightBlock, envelope.From); err != nil {
  559. r.Logger.Error("error processing light block response", "err", err)
  560. }
  561. default:
  562. return fmt.Errorf("received unknown message: %T", msg)
  563. }
  564. return nil
  565. }
  566. // handleMessage handles an Envelope sent from a peer on a specific p2p Channel.
  567. // It will handle errors and any possible panics gracefully. A caller can handle
  568. // any error returned by sending a PeerError on the respective channel.
  569. func (r *Reactor) handleMessage(chID p2p.ChannelID, envelope p2p.Envelope) (err error) {
  570. defer func() {
  571. if e := recover(); e != nil {
  572. err = fmt.Errorf("panic in processing message: %v", e)
  573. r.Logger.Error(
  574. "recovering from processing message panic",
  575. "err", err,
  576. "stack", string(debug.Stack()),
  577. )
  578. }
  579. }()
  580. r.Logger.Debug("received message", "message", reflect.TypeOf(envelope.Message), "peer", envelope.From)
  581. switch chID {
  582. case SnapshotChannel:
  583. err = r.handleSnapshotMessage(envelope)
  584. case ChunkChannel:
  585. err = r.handleChunkMessage(envelope)
  586. case LightBlockChannel:
  587. err = r.handleLightBlockMessage(envelope)
  588. default:
  589. err = fmt.Errorf("unknown channel ID (%d) for envelope (%v)", chID, envelope)
  590. }
  591. return err
  592. }
  593. // processSnapshotCh initiates a blocking process where we listen for and handle
  594. // envelopes on the SnapshotChannel.
  595. func (r *Reactor) processSnapshotCh() {
  596. r.processCh(r.snapshotCh, "snapshot")
  597. }
  598. // processChunkCh initiates a blocking process where we listen for and handle
  599. // envelopes on the ChunkChannel.
  600. func (r *Reactor) processChunkCh() {
  601. r.processCh(r.chunkCh, "chunk")
  602. }
  603. // processBlockCh initiates a blocking process where we listen for and handle
  604. // envelopes on the LightBlockChannel.
  605. func (r *Reactor) processBlockCh() {
  606. r.processCh(r.blockCh, "light block")
  607. }
  608. // processCh routes state sync messages to their respective handlers. Any error
  609. // encountered during message execution will result in a PeerError being sent on
  610. // the respective channel. When the reactor is stopped, we will catch the signal
  611. // and close the p2p Channel gracefully.
  612. func (r *Reactor) processCh(ch *p2p.Channel, chName string) {
  613. defer ch.Close()
  614. for {
  615. select {
  616. case envelope := <-ch.In:
  617. if err := r.handleMessage(ch.ID, envelope); err != nil {
  618. r.Logger.Error(fmt.Sprintf("failed to process %s message", chName),
  619. "ch_id", ch.ID, "envelope", envelope, "err", err)
  620. ch.Error <- p2p.PeerError{
  621. NodeID: envelope.From,
  622. Err: err,
  623. }
  624. }
  625. case <-r.closeCh:
  626. r.Logger.Debug(fmt.Sprintf("stopped listening on %s channel; closing...", chName))
  627. return
  628. }
  629. }
  630. }
  631. // processPeerUpdate processes a PeerUpdate, returning an error upon failing to
  632. // handle the PeerUpdate or if a panic is recovered.
  633. func (r *Reactor) processPeerUpdate(peerUpdate p2p.PeerUpdate) {
  634. r.Logger.Debug("received peer update", "peer", peerUpdate.NodeID, "status", peerUpdate.Status)
  635. r.mtx.RLock()
  636. defer r.mtx.RUnlock()
  637. switch peerUpdate.Status {
  638. case p2p.PeerStatusUp:
  639. if r.syncer != nil {
  640. r.syncer.AddPeer(peerUpdate.NodeID)
  641. }
  642. r.dispatcher.addPeer(peerUpdate.NodeID)
  643. case p2p.PeerStatusDown:
  644. if r.syncer != nil {
  645. r.syncer.RemovePeer(peerUpdate.NodeID)
  646. }
  647. r.dispatcher.removePeer(peerUpdate.NodeID)
  648. }
  649. }
  650. // processPeerUpdates initiates a blocking process where we listen for and handle
  651. // PeerUpdate messages. When the reactor is stopped, we will catch the signal and
  652. // close the p2p PeerUpdatesCh gracefully.
  653. func (r *Reactor) processPeerUpdates() {
  654. defer r.peerUpdates.Close()
  655. for {
  656. select {
  657. case peerUpdate := <-r.peerUpdates.Updates():
  658. r.processPeerUpdate(peerUpdate)
  659. case <-r.closeCh:
  660. r.Logger.Debug("stopped listening on peer updates channel; closing...")
  661. return
  662. }
  663. }
  664. }
  665. // recentSnapshots fetches the n most recent snapshots from the app
  666. func (r *Reactor) recentSnapshots(n uint32) ([]*snapshot, error) {
  667. resp, err := r.conn.ListSnapshotsSync(context.Background(), abci.RequestListSnapshots{})
  668. if err != nil {
  669. return nil, err
  670. }
  671. sort.Slice(resp.Snapshots, func(i, j int) bool {
  672. a := resp.Snapshots[i]
  673. b := resp.Snapshots[j]
  674. switch {
  675. case a.Height > b.Height:
  676. return true
  677. case a.Height == b.Height && a.Format > b.Format:
  678. return true
  679. default:
  680. return false
  681. }
  682. })
  683. snapshots := make([]*snapshot, 0, n)
  684. for i, s := range resp.Snapshots {
  685. if i >= recentSnapshots {
  686. break
  687. }
  688. snapshots = append(snapshots, &snapshot{
  689. Height: s.Height,
  690. Format: s.Format,
  691. Chunks: s.Chunks,
  692. Hash: s.Hash,
  693. Metadata: s.Metadata,
  694. })
  695. }
  696. return snapshots, nil
  697. }
  698. // fetchLightBlock works out whether the node has a light block at a particular
  699. // height and if so returns it so it can be gossiped to peers
  700. func (r *Reactor) fetchLightBlock(height uint64) (*types.LightBlock, error) {
  701. h := int64(height)
  702. blockMeta := r.blockStore.LoadBlockMeta(h)
  703. if blockMeta == nil {
  704. return nil, nil
  705. }
  706. commit := r.blockStore.LoadBlockCommit(h)
  707. if commit == nil {
  708. return nil, nil
  709. }
  710. vals, err := r.stateStore.LoadValidators(h)
  711. if err != nil {
  712. return nil, err
  713. }
  714. if vals == nil {
  715. return nil, nil
  716. }
  717. return &types.LightBlock{
  718. SignedHeader: &types.SignedHeader{
  719. Header: &blockMeta.Header,
  720. Commit: commit,
  721. },
  722. ValidatorSet: vals,
  723. }, nil
  724. }