You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

442 lines
15 KiB

  1. package statesync
  2. import (
  3. "bytes"
  4. "context"
  5. "errors"
  6. "fmt"
  7. "sync"
  8. "time"
  9. abci "github.com/tendermint/tendermint/abci/types"
  10. "github.com/tendermint/tendermint/libs/log"
  11. "github.com/tendermint/tendermint/p2p"
  12. ssproto "github.com/tendermint/tendermint/proto/statesync"
  13. "github.com/tendermint/tendermint/proxy"
  14. sm "github.com/tendermint/tendermint/state"
  15. "github.com/tendermint/tendermint/types"
  16. )
  17. const (
  18. // defaultDiscoveryTime is the time to spend discovering snapshots.
  19. defaultDiscoveryTime = 20 * time.Second
  20. // chunkFetchers is the number of concurrent chunk fetchers to run.
  21. chunkFetchers = 4
  22. // chunkTimeout is the timeout while waiting for the next chunk from the chunk queue.
  23. chunkTimeout = 2 * time.Minute
  24. // requestTimeout is the timeout before rerequesting a chunk, possibly from a different peer.
  25. chunkRequestTimeout = 10 * time.Second
  26. )
  27. var (
  28. // errAbort is returned by Sync() when snapshot restoration is aborted.
  29. errAbort = errors.New("state sync aborted")
  30. // errRetrySnapshot is returned by Sync() when the snapshot should be retried.
  31. errRetrySnapshot = errors.New("retry snapshot")
  32. // errRejectSnapshot is returned by Sync() when the snapshot is rejected.
  33. errRejectSnapshot = errors.New("snapshot was rejected")
  34. // errRejectFormat is returned by Sync() when the snapshot format is rejected.
  35. errRejectFormat = errors.New("snapshot format was rejected")
  36. // errRejectSender is returned by Sync() when the snapshot sender is rejected.
  37. errRejectSender = errors.New("snapshot sender was rejected")
  38. // errVerifyFailed is returned by Sync() when app hash or last height verification fails.
  39. errVerifyFailed = errors.New("verification failed")
  40. // errTimeout is returned by Sync() when we've waited too long to receive a chunk.
  41. errTimeout = errors.New("timed out waiting for chunk")
  42. // errNoSnapshots is returned by SyncAny() if no snapshots are found and discovery is disabled.
  43. errNoSnapshots = errors.New("no suitable snapshots found")
  44. )
  45. // syncer runs a state sync against an ABCI app. Use either SyncAny() to automatically attempt to
  46. // sync all snapshots in the pool (pausing to discover new ones), or Sync() to sync a specific
  47. // snapshot. Snapshots and chunks are fed via AddSnapshot() and AddChunk() as appropriate.
  48. type syncer struct {
  49. logger log.Logger
  50. stateProvider StateProvider
  51. conn proxy.AppConnSnapshot
  52. connQuery proxy.AppConnQuery
  53. snapshots *snapshotPool
  54. tempDir string
  55. mtx sync.RWMutex
  56. chunks *chunkQueue
  57. }
  58. // newSyncer creates a new syncer.
  59. func newSyncer(logger log.Logger, conn proxy.AppConnSnapshot, connQuery proxy.AppConnQuery,
  60. stateProvider StateProvider, tempDir string) *syncer {
  61. return &syncer{
  62. logger: logger,
  63. stateProvider: stateProvider,
  64. conn: conn,
  65. connQuery: connQuery,
  66. snapshots: newSnapshotPool(stateProvider),
  67. tempDir: tempDir,
  68. }
  69. }
  70. // AddChunk adds a chunk to the chunk queue, if any. It returns false if the chunk has already
  71. // been added to the queue, or an error if there's no sync in progress.
  72. func (s *syncer) AddChunk(chunk *chunk) (bool, error) {
  73. s.mtx.RLock()
  74. defer s.mtx.RUnlock()
  75. if s.chunks == nil {
  76. return false, errors.New("no state sync in progress")
  77. }
  78. added, err := s.chunks.Add(chunk)
  79. if err != nil {
  80. return false, err
  81. }
  82. if added {
  83. s.logger.Debug("Added chunk to queue", "height", chunk.Height, "format", chunk.Format,
  84. "chunk", chunk.Index)
  85. } else {
  86. s.logger.Debug("Ignoring duplicate chunk in queue", "height", chunk.Height, "format", chunk.Format,
  87. "chunk", chunk.Index)
  88. }
  89. return added, nil
  90. }
  91. // AddSnapshot adds a snapshot to the snapshot pool. It returns true if a new, previously unseen
  92. // snapshot was accepted and added.
  93. func (s *syncer) AddSnapshot(peer p2p.Peer, snapshot *snapshot) (bool, error) {
  94. added, err := s.snapshots.Add(peer, snapshot)
  95. if err != nil {
  96. return false, err
  97. }
  98. if added {
  99. s.logger.Info("Discovered new snapshot", "height", snapshot.Height, "format", snapshot.Format,
  100. "hash", fmt.Sprintf("%X", snapshot.Hash))
  101. }
  102. return added, nil
  103. }
  104. // AddPeer adds a peer to the pool. For now we just keep it simple and send a single request
  105. // to discover snapshots, later we may want to do retries and stuff.
  106. func (s *syncer) AddPeer(peer p2p.Peer) {
  107. s.logger.Debug("Requesting snapshots from peer", "peer", peer.ID())
  108. peer.Send(SnapshotChannel, mustEncodeMsg(&ssproto.SnapshotsRequest{}))
  109. }
  110. // RemovePeer removes a peer from the pool.
  111. func (s *syncer) RemovePeer(peer p2p.Peer) {
  112. s.logger.Debug("Removing peer from sync", "peer", peer.ID())
  113. s.snapshots.RemovePeer(peer.ID())
  114. }
  115. // SyncAny tries to sync any of the snapshots in the snapshot pool, waiting to discover further
  116. // snapshots if none were found and discoveryTime > 0. It returns the latest state and block commit
  117. // which the caller must use to bootstrap the node.
  118. func (s *syncer) SyncAny(discoveryTime time.Duration) (sm.State, *types.Commit, error) {
  119. if discoveryTime > 0 {
  120. s.logger.Info(fmt.Sprintf("Discovering snapshots for %v", discoveryTime))
  121. time.Sleep(discoveryTime)
  122. }
  123. // The app may ask us to retry a snapshot restoration, in which case we need to reuse
  124. // the snapshot and chunk queue from the previous loop iteration.
  125. var (
  126. snapshot *snapshot
  127. chunks *chunkQueue
  128. err error
  129. )
  130. for {
  131. // If not nil, we're going to retry restoration of the same snapshot.
  132. if snapshot == nil {
  133. snapshot = s.snapshots.Best()
  134. chunks = nil
  135. }
  136. if snapshot == nil {
  137. if discoveryTime == 0 {
  138. return sm.State{}, nil, errNoSnapshots
  139. }
  140. s.logger.Info(fmt.Sprintf("Discovering snapshots for %v", discoveryTime))
  141. time.Sleep(discoveryTime)
  142. continue
  143. }
  144. if chunks == nil {
  145. chunks, err = newChunkQueue(snapshot, s.tempDir)
  146. if err != nil {
  147. return sm.State{}, nil, fmt.Errorf("failed to create chunk queue: %w", err)
  148. }
  149. defer chunks.Close() // in case we forget to close it elsewhere
  150. }
  151. newState, commit, err := s.Sync(snapshot, chunks)
  152. switch {
  153. case err == nil:
  154. return newState, commit, nil
  155. case errors.Is(err, errAbort):
  156. return sm.State{}, nil, err
  157. case errors.Is(err, errRetrySnapshot):
  158. chunks.RetryAll()
  159. s.logger.Info("Retrying snapshot", "height", snapshot.Height, "format", snapshot.Format,
  160. "hash", fmt.Sprintf("%X", snapshot.Hash))
  161. continue
  162. case errors.Is(err, errTimeout):
  163. s.snapshots.Reject(snapshot)
  164. s.logger.Error("Timed out waiting for snapshot chunks, rejected snapshot",
  165. "height", snapshot.Height, "format", snapshot.Format, "hash", fmt.Sprintf("%X", snapshot.Hash))
  166. case errors.Is(err, errRejectSnapshot):
  167. s.snapshots.Reject(snapshot)
  168. s.logger.Info("Snapshot rejected", "height", snapshot.Height, "format", snapshot.Format,
  169. "hash", fmt.Sprintf("%X", snapshot.Hash))
  170. case errors.Is(err, errRejectFormat):
  171. s.snapshots.RejectFormat(snapshot.Format)
  172. s.logger.Info("Snapshot format rejected", "format", snapshot.Format)
  173. case errors.Is(err, errRejectSender):
  174. s.logger.Info("Snapshot senders rejected", "height", snapshot.Height, "format", snapshot.Format,
  175. "hash", fmt.Sprintf("%X", snapshot.Hash))
  176. for _, peer := range s.snapshots.GetPeers(snapshot) {
  177. s.snapshots.RejectPeer(peer.ID())
  178. s.logger.Info("Snapshot sender rejected", "peer", peer.ID())
  179. }
  180. default:
  181. return sm.State{}, nil, fmt.Errorf("snapshot restoration failed: %w", err)
  182. }
  183. // Discard snapshot and chunks for next iteration
  184. err = chunks.Close()
  185. if err != nil {
  186. s.logger.Error("Failed to clean up chunk queue", "err", err)
  187. }
  188. snapshot = nil
  189. chunks = nil
  190. }
  191. }
  192. // Sync executes a sync for a specific snapshot, returning the latest state and block commit which
  193. // the caller must use to bootstrap the node.
  194. func (s *syncer) Sync(snapshot *snapshot, chunks *chunkQueue) (sm.State, *types.Commit, error) {
  195. s.mtx.Lock()
  196. if s.chunks != nil {
  197. s.mtx.Unlock()
  198. return sm.State{}, nil, errors.New("a state sync is already in progress")
  199. }
  200. s.chunks = chunks
  201. s.mtx.Unlock()
  202. defer func() {
  203. s.mtx.Lock()
  204. s.chunks = nil
  205. s.mtx.Unlock()
  206. }()
  207. // Offer snapshot to ABCI app.
  208. err := s.offerSnapshot(snapshot)
  209. if err != nil {
  210. return sm.State{}, nil, err
  211. }
  212. // Spawn chunk fetchers. They will terminate when the chunk queue is closed or context cancelled.
  213. ctx, cancel := context.WithCancel(context.Background())
  214. defer cancel()
  215. for i := int32(0); i < chunkFetchers; i++ {
  216. go s.fetchChunks(ctx, snapshot, chunks)
  217. }
  218. // Optimistically build new state, so we don't discover any light client failures at the end.
  219. state, err := s.stateProvider.State(snapshot.Height)
  220. if err != nil {
  221. return sm.State{}, nil, fmt.Errorf("failed to build new state: %w", err)
  222. }
  223. commit, err := s.stateProvider.Commit(snapshot.Height)
  224. if err != nil {
  225. return sm.State{}, nil, fmt.Errorf("failed to fetch commit: %w", err)
  226. }
  227. // Restore snapshot
  228. err = s.applyChunks(chunks)
  229. if err != nil {
  230. return sm.State{}, nil, err
  231. }
  232. // Verify app and update app version
  233. appVersion, err := s.verifyApp(snapshot)
  234. if err != nil {
  235. return sm.State{}, nil, err
  236. }
  237. state.Version.Consensus.App = appVersion
  238. // Done! 🎉
  239. s.logger.Info("Snapshot restored", "height", snapshot.Height, "format", snapshot.Format,
  240. "hash", fmt.Sprintf("%X", snapshot.Hash))
  241. return state, commit, nil
  242. }
  243. // offerSnapshot offers a snapshot to the app. It returns various errors depending on the app's
  244. // response, or nil if the snapshot was accepted.
  245. func (s *syncer) offerSnapshot(snapshot *snapshot) error {
  246. s.logger.Info("Offering snapshot to ABCI app", "height", snapshot.Height,
  247. "format", snapshot.Format, "hash", fmt.Sprintf("%X", snapshot.Hash))
  248. resp, err := s.conn.OfferSnapshotSync(abci.RequestOfferSnapshot{
  249. Snapshot: &abci.Snapshot{
  250. Height: snapshot.Height,
  251. Format: snapshot.Format,
  252. Chunks: snapshot.Chunks,
  253. Hash: snapshot.Hash,
  254. Metadata: snapshot.Metadata,
  255. },
  256. AppHash: snapshot.trustedAppHash,
  257. })
  258. if err != nil {
  259. return fmt.Errorf("failed to offer snapshot: %w", err)
  260. }
  261. switch resp.Result {
  262. case abci.ResponseOfferSnapshot_ACCEPT:
  263. s.logger.Info("Snapshot accepted, restoring", "height", snapshot.Height,
  264. "format", snapshot.Format, "hash", fmt.Sprintf("%X", snapshot.Hash))
  265. return nil
  266. case abci.ResponseOfferSnapshot_ABORT:
  267. return errAbort
  268. case abci.ResponseOfferSnapshot_REJECT:
  269. return errRejectSnapshot
  270. case abci.ResponseOfferSnapshot_REJECT_FORMAT:
  271. return errRejectFormat
  272. case abci.ResponseOfferSnapshot_REJECT_SENDER:
  273. return errRejectSender
  274. default:
  275. return fmt.Errorf("invalid ResponseOfferSnapshot result %v", resp.Result)
  276. }
  277. }
  278. // applyChunks applies chunks to the app. It returns various errors depending on the app's
  279. // response, or nil once the snapshot is fully restored.
  280. func (s *syncer) applyChunks(chunks *chunkQueue) error {
  281. for {
  282. chunk, err := chunks.Next()
  283. if err == errDone {
  284. return nil
  285. } else if err != nil {
  286. return fmt.Errorf("failed to fetch chunk: %w", err)
  287. }
  288. resp, err := s.conn.ApplySnapshotChunkSync(abci.RequestApplySnapshotChunk{
  289. Index: chunk.Index,
  290. Chunk: chunk.Chunk,
  291. Sender: string(chunk.Sender),
  292. })
  293. if err != nil {
  294. return fmt.Errorf("failed to apply chunk %v: %w", chunk.Index, err)
  295. }
  296. s.logger.Info("Applied snapshot chunk to ABCI app", "height", chunk.Height,
  297. "format", chunk.Format, "chunk", chunk.Index, "total", chunks.Size())
  298. // Discard and refetch any chunks as requested by the app
  299. for _, index := range resp.RefetchChunks {
  300. err := chunks.Discard(index)
  301. if err != nil {
  302. return fmt.Errorf("failed to discard chunk %v: %w", index, err)
  303. }
  304. }
  305. // Reject any senders as requested by the app
  306. for _, sender := range resp.RejectSenders {
  307. if sender != "" {
  308. s.snapshots.RejectPeer(p2p.ID(sender))
  309. err := chunks.DiscardSender(p2p.ID(sender))
  310. if err != nil {
  311. return fmt.Errorf("failed to reject sender: %w", err)
  312. }
  313. }
  314. }
  315. switch resp.Result {
  316. case abci.ResponseApplySnapshotChunk_ACCEPT:
  317. case abci.ResponseApplySnapshotChunk_ABORT:
  318. return errAbort
  319. case abci.ResponseApplySnapshotChunk_RETRY:
  320. chunks.Retry(chunk.Index)
  321. case abci.ResponseApplySnapshotChunk_RETRY_SNAPSHOT:
  322. return errRetrySnapshot
  323. case abci.ResponseApplySnapshotChunk_REJECT_SNAPSHOT:
  324. return errRejectSnapshot
  325. default:
  326. return fmt.Errorf("unknown ResponseApplySnapshotChunk result %v", resp.Result)
  327. }
  328. }
  329. }
  330. // fetchChunks requests chunks from peers, receiving allocations from the chunk queue. Chunks
  331. // will be received from the reactor via syncer.AddChunks() to chunkQueue.Add().
  332. func (s *syncer) fetchChunks(ctx context.Context, snapshot *snapshot, chunks *chunkQueue) {
  333. for {
  334. index, err := chunks.Allocate()
  335. if err == errDone {
  336. // Keep checking until the context is cancelled (restore is done), in case any
  337. // chunks need to be refetched.
  338. select {
  339. case <-ctx.Done():
  340. return
  341. default:
  342. }
  343. time.Sleep(2 * time.Second)
  344. continue
  345. }
  346. if err != nil {
  347. s.logger.Error("Failed to allocate chunk from queue", "err", err)
  348. return
  349. }
  350. s.logger.Info("Fetching snapshot chunk", "height", snapshot.Height,
  351. "format", snapshot.Format, "chunk", index, "total", chunks.Size())
  352. ticker := time.NewTicker(chunkRequestTimeout)
  353. defer ticker.Stop()
  354. s.requestChunk(snapshot, index)
  355. select {
  356. case <-chunks.WaitFor(index):
  357. case <-ticker.C:
  358. s.requestChunk(snapshot, index)
  359. case <-ctx.Done():
  360. return
  361. }
  362. ticker.Stop()
  363. }
  364. }
  365. // requestChunk requests a chunk from a peer.
  366. func (s *syncer) requestChunk(snapshot *snapshot, chunk uint32) {
  367. peer := s.snapshots.GetPeer(snapshot)
  368. if peer == nil {
  369. s.logger.Error("No valid peers found for snapshot", "height", snapshot.Height,
  370. "format", snapshot.Format, "hash", snapshot.Hash)
  371. return
  372. }
  373. s.logger.Debug("Requesting snapshot chunk", "height", snapshot.Height,
  374. "format", snapshot.Format, "chunk", chunk, "peer", peer.ID())
  375. peer.Send(ChunkChannel, mustEncodeMsg(&ssproto.ChunkRequest{
  376. Height: snapshot.Height,
  377. Format: snapshot.Format,
  378. Index: chunk,
  379. }))
  380. }
  381. // verifyApp verifies the sync, checking the app hash and last block height. It returns the
  382. // app version, which should be returned as part of the initial state.
  383. func (s *syncer) verifyApp(snapshot *snapshot) (uint64, error) {
  384. resp, err := s.connQuery.InfoSync(proxy.RequestInfo)
  385. if err != nil {
  386. return 0, fmt.Errorf("failed to query ABCI app for appHash: %w", err)
  387. }
  388. if !bytes.Equal(snapshot.trustedAppHash, resp.LastBlockAppHash) {
  389. s.logger.Error("appHash verification failed",
  390. "expected", fmt.Sprintf("%X", snapshot.trustedAppHash),
  391. "actual", fmt.Sprintf("%X", resp.LastBlockAppHash))
  392. return 0, errVerifyFailed
  393. }
  394. if uint64(resp.LastBlockHeight) != snapshot.Height {
  395. s.logger.Error("ABCI app reported unexpected last block height",
  396. "expected", snapshot.Height, "actual", resp.LastBlockHeight)
  397. return 0, errVerifyFailed
  398. }
  399. s.logger.Info("Verified ABCI app", "height", snapshot.Height,
  400. "appHash", fmt.Sprintf("%X", snapshot.trustedAppHash))
  401. return resp.AppVersion, nil
  402. }