You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

923 lines
26 KiB

  1. package statesync
  2. import (
  3. "context"
  4. "fmt"
  5. "strings"
  6. "sync"
  7. "testing"
  8. "time"
  9. "github.com/fortytw2/leaktest"
  10. "github.com/stretchr/testify/mock"
  11. "github.com/stretchr/testify/require"
  12. dbm "github.com/tendermint/tm-db"
  13. abci "github.com/tendermint/tendermint/abci/types"
  14. "github.com/tendermint/tendermint/config"
  15. "github.com/tendermint/tendermint/internal/p2p"
  16. "github.com/tendermint/tendermint/internal/proxy"
  17. proxymocks "github.com/tendermint/tendermint/internal/proxy/mocks"
  18. smmocks "github.com/tendermint/tendermint/internal/state/mocks"
  19. "github.com/tendermint/tendermint/internal/statesync/mocks"
  20. "github.com/tendermint/tendermint/internal/store"
  21. "github.com/tendermint/tendermint/internal/test/factory"
  22. "github.com/tendermint/tendermint/libs/log"
  23. "github.com/tendermint/tendermint/light/provider"
  24. ssproto "github.com/tendermint/tendermint/proto/tendermint/statesync"
  25. tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
  26. "github.com/tendermint/tendermint/types"
  27. )
  28. var m = PrometheusMetrics(config.TestConfig().Instrumentation.Namespace)
  29. const testAppVersion = 9
  30. type reactorTestSuite struct {
  31. reactor *Reactor
  32. syncer *syncer
  33. conn *proxymocks.AppConnSnapshot
  34. connQuery *proxymocks.AppConnQuery
  35. stateProvider *mocks.StateProvider
  36. snapshotChannel *p2p.Channel
  37. snapshotInCh chan p2p.Envelope
  38. snapshotOutCh chan p2p.Envelope
  39. snapshotPeerErrCh chan p2p.PeerError
  40. chunkChannel *p2p.Channel
  41. chunkInCh chan p2p.Envelope
  42. chunkOutCh chan p2p.Envelope
  43. chunkPeerErrCh chan p2p.PeerError
  44. blockChannel *p2p.Channel
  45. blockInCh chan p2p.Envelope
  46. blockOutCh chan p2p.Envelope
  47. blockPeerErrCh chan p2p.PeerError
  48. paramsChannel *p2p.Channel
  49. paramsInCh chan p2p.Envelope
  50. paramsOutCh chan p2p.Envelope
  51. paramsPeerErrCh chan p2p.PeerError
  52. peerUpdateCh chan p2p.PeerUpdate
  53. peerUpdates *p2p.PeerUpdates
  54. stateStore *smmocks.Store
  55. blockStore *store.BlockStore
  56. }
  57. func setup(
  58. ctx context.Context,
  59. t *testing.T,
  60. conn *proxymocks.AppConnSnapshot,
  61. connQuery *proxymocks.AppConnQuery,
  62. stateProvider *mocks.StateProvider,
  63. chBuf uint,
  64. ) *reactorTestSuite {
  65. t.Helper()
  66. if conn == nil {
  67. conn = &proxymocks.AppConnSnapshot{}
  68. }
  69. if connQuery == nil {
  70. connQuery = &proxymocks.AppConnQuery{}
  71. }
  72. if stateProvider == nil {
  73. stateProvider = &mocks.StateProvider{}
  74. }
  75. rts := &reactorTestSuite{
  76. snapshotInCh: make(chan p2p.Envelope, chBuf),
  77. snapshotOutCh: make(chan p2p.Envelope, chBuf),
  78. snapshotPeerErrCh: make(chan p2p.PeerError, chBuf),
  79. chunkInCh: make(chan p2p.Envelope, chBuf),
  80. chunkOutCh: make(chan p2p.Envelope, chBuf),
  81. chunkPeerErrCh: make(chan p2p.PeerError, chBuf),
  82. blockInCh: make(chan p2p.Envelope, chBuf),
  83. blockOutCh: make(chan p2p.Envelope, chBuf),
  84. blockPeerErrCh: make(chan p2p.PeerError, chBuf),
  85. paramsInCh: make(chan p2p.Envelope, chBuf),
  86. paramsOutCh: make(chan p2p.Envelope, chBuf),
  87. paramsPeerErrCh: make(chan p2p.PeerError, chBuf),
  88. conn: conn,
  89. connQuery: connQuery,
  90. stateProvider: stateProvider,
  91. }
  92. rts.peerUpdateCh = make(chan p2p.PeerUpdate, chBuf)
  93. rts.peerUpdates = p2p.NewPeerUpdates(rts.peerUpdateCh, int(chBuf))
  94. rts.snapshotChannel = p2p.NewChannel(
  95. SnapshotChannel,
  96. new(ssproto.Message),
  97. rts.snapshotInCh,
  98. rts.snapshotOutCh,
  99. rts.snapshotPeerErrCh,
  100. )
  101. rts.chunkChannel = p2p.NewChannel(
  102. ChunkChannel,
  103. new(ssproto.Message),
  104. rts.chunkInCh,
  105. rts.chunkOutCh,
  106. rts.chunkPeerErrCh,
  107. )
  108. rts.blockChannel = p2p.NewChannel(
  109. LightBlockChannel,
  110. new(ssproto.Message),
  111. rts.blockInCh,
  112. rts.blockOutCh,
  113. rts.blockPeerErrCh,
  114. )
  115. rts.paramsChannel = p2p.NewChannel(
  116. ParamsChannel,
  117. new(ssproto.Message),
  118. rts.paramsInCh,
  119. rts.paramsOutCh,
  120. rts.paramsPeerErrCh,
  121. )
  122. rts.stateStore = &smmocks.Store{}
  123. rts.blockStore = store.NewBlockStore(dbm.NewMemDB())
  124. cfg := config.DefaultStateSyncConfig()
  125. chCreator := func(ctx context.Context, desc *p2p.ChannelDescriptor) (*p2p.Channel, error) {
  126. switch desc.ID {
  127. case SnapshotChannel:
  128. return rts.snapshotChannel, nil
  129. case ChunkChannel:
  130. return rts.chunkChannel, nil
  131. case LightBlockChannel:
  132. return rts.blockChannel, nil
  133. case ParamsChannel:
  134. return rts.paramsChannel, nil
  135. default:
  136. return nil, fmt.Errorf("invalid channel; %v", desc.ID)
  137. }
  138. }
  139. logger := log.NewNopLogger()
  140. var err error
  141. rts.reactor, err = NewReactor(
  142. ctx,
  143. factory.DefaultTestChainID,
  144. 1,
  145. *cfg,
  146. logger.With("component", "reactor"),
  147. conn,
  148. connQuery,
  149. chCreator,
  150. rts.peerUpdates,
  151. rts.stateStore,
  152. rts.blockStore,
  153. "",
  154. m,
  155. nil, // eventbus can be nil
  156. )
  157. require.NoError(t, err)
  158. rts.syncer = newSyncer(
  159. *cfg,
  160. logger.With("component", "syncer"),
  161. conn,
  162. connQuery,
  163. stateProvider,
  164. rts.snapshotChannel,
  165. rts.chunkChannel,
  166. "",
  167. rts.reactor.metrics,
  168. )
  169. ctx, cancel := context.WithCancel(ctx)
  170. require.NoError(t, rts.reactor.Start(ctx))
  171. require.True(t, rts.reactor.IsRunning())
  172. t.Cleanup(cancel)
  173. t.Cleanup(rts.reactor.Wait)
  174. t.Cleanup(leaktest.Check(t))
  175. return rts
  176. }
  177. func TestReactor_Sync(t *testing.T) {
  178. ctx, cancel := context.WithCancel(context.Background())
  179. defer cancel()
  180. const snapshotHeight = 7
  181. rts := setup(ctx, t, nil, nil, nil, 2)
  182. chain := buildLightBlockChain(ctx, t, 1, 10, time.Now())
  183. // app accepts any snapshot
  184. rts.conn.On("OfferSnapshot", ctx, mock.AnythingOfType("types.RequestOfferSnapshot")).
  185. Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_ACCEPT}, nil)
  186. // app accepts every chunk
  187. rts.conn.On("ApplySnapshotChunk", ctx, mock.AnythingOfType("types.RequestApplySnapshotChunk")).
  188. Return(&abci.ResponseApplySnapshotChunk{Result: abci.ResponseApplySnapshotChunk_ACCEPT}, nil)
  189. // app query returns valid state app hash
  190. rts.connQuery.On("Info", mock.Anything, proxy.RequestInfo).Return(&abci.ResponseInfo{
  191. AppVersion: testAppVersion,
  192. LastBlockHeight: snapshotHeight,
  193. LastBlockAppHash: chain[snapshotHeight+1].AppHash,
  194. }, nil)
  195. // store accepts state and validator sets
  196. rts.stateStore.On("Bootstrap", mock.AnythingOfType("state.State")).Return(nil)
  197. rts.stateStore.On("SaveValidatorSets", mock.AnythingOfType("int64"), mock.AnythingOfType("int64"),
  198. mock.AnythingOfType("*types.ValidatorSet")).Return(nil)
  199. closeCh := make(chan struct{})
  200. defer close(closeCh)
  201. go handleLightBlockRequests(ctx, t, chain, rts.blockOutCh,
  202. rts.blockInCh, closeCh, 0)
  203. go graduallyAddPeers(t, rts.peerUpdateCh, closeCh, 1*time.Second)
  204. go handleSnapshotRequests(t, rts.snapshotOutCh, rts.snapshotInCh, closeCh, []snapshot{
  205. {
  206. Height: uint64(snapshotHeight),
  207. Format: 1,
  208. Chunks: 1,
  209. },
  210. })
  211. go handleChunkRequests(t, rts.chunkOutCh, rts.chunkInCh, closeCh, []byte("abc"))
  212. go handleConsensusParamsRequest(ctx, t, rts.paramsOutCh, rts.paramsInCh, closeCh)
  213. // update the config to use the p2p provider
  214. rts.reactor.cfg.UseP2P = true
  215. rts.reactor.cfg.TrustHeight = 1
  216. rts.reactor.cfg.TrustHash = fmt.Sprintf("%X", chain[1].Hash())
  217. rts.reactor.cfg.DiscoveryTime = 1 * time.Second
  218. // Run state sync
  219. _, err := rts.reactor.Sync(ctx)
  220. require.NoError(t, err)
  221. }
  222. func TestReactor_ChunkRequest_InvalidRequest(t *testing.T) {
  223. ctx, cancel := context.WithCancel(context.Background())
  224. defer cancel()
  225. rts := setup(ctx, t, nil, nil, nil, 2)
  226. rts.chunkInCh <- p2p.Envelope{
  227. From: types.NodeID("aa"),
  228. Message: &ssproto.SnapshotsRequest{},
  229. }
  230. response := <-rts.chunkPeerErrCh
  231. require.Error(t, response.Err)
  232. require.Empty(t, rts.chunkOutCh)
  233. require.Contains(t, response.Err.Error(), "received unknown message")
  234. require.Equal(t, types.NodeID("aa"), response.NodeID)
  235. }
  236. func TestReactor_ChunkRequest(t *testing.T) {
  237. testcases := map[string]struct {
  238. request *ssproto.ChunkRequest
  239. chunk []byte
  240. expectResponse *ssproto.ChunkResponse
  241. }{
  242. "chunk is returned": {
  243. &ssproto.ChunkRequest{Height: 1, Format: 1, Index: 1},
  244. []byte{1, 2, 3},
  245. &ssproto.ChunkResponse{Height: 1, Format: 1, Index: 1, Chunk: []byte{1, 2, 3}},
  246. },
  247. "empty chunk is returned, as empty": {
  248. &ssproto.ChunkRequest{Height: 1, Format: 1, Index: 1},
  249. []byte{},
  250. &ssproto.ChunkResponse{Height: 1, Format: 1, Index: 1, Chunk: []byte{}},
  251. },
  252. "nil (missing) chunk is returned as missing": {
  253. &ssproto.ChunkRequest{Height: 1, Format: 1, Index: 1},
  254. nil,
  255. &ssproto.ChunkResponse{Height: 1, Format: 1, Index: 1, Missing: true},
  256. },
  257. "invalid request": {
  258. &ssproto.ChunkRequest{Height: 1, Format: 1, Index: 1},
  259. nil,
  260. &ssproto.ChunkResponse{Height: 1, Format: 1, Index: 1, Missing: true},
  261. },
  262. }
  263. bctx, bcancel := context.WithCancel(context.Background())
  264. defer bcancel()
  265. for name, tc := range testcases {
  266. t.Run(name, func(t *testing.T) {
  267. ctx, cancel := context.WithCancel(bctx)
  268. defer cancel()
  269. // mock ABCI connection to return local snapshots
  270. conn := &proxymocks.AppConnSnapshot{}
  271. conn.On("LoadSnapshotChunk", mock.Anything, abci.RequestLoadSnapshotChunk{
  272. Height: tc.request.Height,
  273. Format: tc.request.Format,
  274. Chunk: tc.request.Index,
  275. }).Return(&abci.ResponseLoadSnapshotChunk{Chunk: tc.chunk}, nil)
  276. rts := setup(ctx, t, conn, nil, nil, 2)
  277. rts.chunkInCh <- p2p.Envelope{
  278. From: types.NodeID("aa"),
  279. Message: tc.request,
  280. }
  281. response := <-rts.chunkOutCh
  282. require.Equal(t, tc.expectResponse, response.Message)
  283. require.Empty(t, rts.chunkOutCh)
  284. conn.AssertExpectations(t)
  285. })
  286. }
  287. }
  288. func TestReactor_SnapshotsRequest_InvalidRequest(t *testing.T) {
  289. ctx, cancel := context.WithCancel(context.Background())
  290. defer cancel()
  291. rts := setup(ctx, t, nil, nil, nil, 2)
  292. rts.snapshotInCh <- p2p.Envelope{
  293. From: types.NodeID("aa"),
  294. Message: &ssproto.ChunkRequest{},
  295. }
  296. response := <-rts.snapshotPeerErrCh
  297. require.Error(t, response.Err)
  298. require.Empty(t, rts.snapshotOutCh)
  299. require.Contains(t, response.Err.Error(), "received unknown message")
  300. require.Equal(t, types.NodeID("aa"), response.NodeID)
  301. }
  302. func TestReactor_SnapshotsRequest(t *testing.T) {
  303. testcases := map[string]struct {
  304. snapshots []*abci.Snapshot
  305. expectResponses []*ssproto.SnapshotsResponse
  306. }{
  307. "no snapshots": {nil, []*ssproto.SnapshotsResponse{}},
  308. ">10 unordered snapshots": {
  309. []*abci.Snapshot{
  310. {Height: 1, Format: 2, Chunks: 7, Hash: []byte{1, 2}, Metadata: []byte{1}},
  311. {Height: 2, Format: 2, Chunks: 7, Hash: []byte{2, 2}, Metadata: []byte{2}},
  312. {Height: 3, Format: 2, Chunks: 7, Hash: []byte{3, 2}, Metadata: []byte{3}},
  313. {Height: 1, Format: 1, Chunks: 7, Hash: []byte{1, 1}, Metadata: []byte{4}},
  314. {Height: 2, Format: 1, Chunks: 7, Hash: []byte{2, 1}, Metadata: []byte{5}},
  315. {Height: 3, Format: 1, Chunks: 7, Hash: []byte{3, 1}, Metadata: []byte{6}},
  316. {Height: 1, Format: 4, Chunks: 7, Hash: []byte{1, 4}, Metadata: []byte{7}},
  317. {Height: 2, Format: 4, Chunks: 7, Hash: []byte{2, 4}, Metadata: []byte{8}},
  318. {Height: 3, Format: 4, Chunks: 7, Hash: []byte{3, 4}, Metadata: []byte{9}},
  319. {Height: 1, Format: 3, Chunks: 7, Hash: []byte{1, 3}, Metadata: []byte{10}},
  320. {Height: 2, Format: 3, Chunks: 7, Hash: []byte{2, 3}, Metadata: []byte{11}},
  321. {Height: 3, Format: 3, Chunks: 7, Hash: []byte{3, 3}, Metadata: []byte{12}},
  322. },
  323. []*ssproto.SnapshotsResponse{
  324. {Height: 3, Format: 4, Chunks: 7, Hash: []byte{3, 4}, Metadata: []byte{9}},
  325. {Height: 3, Format: 3, Chunks: 7, Hash: []byte{3, 3}, Metadata: []byte{12}},
  326. {Height: 3, Format: 2, Chunks: 7, Hash: []byte{3, 2}, Metadata: []byte{3}},
  327. {Height: 3, Format: 1, Chunks: 7, Hash: []byte{3, 1}, Metadata: []byte{6}},
  328. {Height: 2, Format: 4, Chunks: 7, Hash: []byte{2, 4}, Metadata: []byte{8}},
  329. {Height: 2, Format: 3, Chunks: 7, Hash: []byte{2, 3}, Metadata: []byte{11}},
  330. {Height: 2, Format: 2, Chunks: 7, Hash: []byte{2, 2}, Metadata: []byte{2}},
  331. {Height: 2, Format: 1, Chunks: 7, Hash: []byte{2, 1}, Metadata: []byte{5}},
  332. {Height: 1, Format: 4, Chunks: 7, Hash: []byte{1, 4}, Metadata: []byte{7}},
  333. {Height: 1, Format: 3, Chunks: 7, Hash: []byte{1, 3}, Metadata: []byte{10}},
  334. },
  335. },
  336. }
  337. ctx, cancel := context.WithCancel(context.Background())
  338. defer cancel()
  339. for name, tc := range testcases {
  340. tc := tc
  341. t.Run(name, func(t *testing.T) {
  342. ctx, cancel := context.WithCancel(ctx)
  343. defer cancel()
  344. // mock ABCI connection to return local snapshots
  345. conn := &proxymocks.AppConnSnapshot{}
  346. conn.On("ListSnapshots", mock.Anything, abci.RequestListSnapshots{}).Return(&abci.ResponseListSnapshots{
  347. Snapshots: tc.snapshots,
  348. }, nil)
  349. rts := setup(ctx, t, conn, nil, nil, 100)
  350. rts.snapshotInCh <- p2p.Envelope{
  351. From: types.NodeID("aa"),
  352. Message: &ssproto.SnapshotsRequest{},
  353. }
  354. if len(tc.expectResponses) > 0 {
  355. retryUntil(ctx, t, func() bool { return len(rts.snapshotOutCh) == len(tc.expectResponses) }, time.Second)
  356. }
  357. responses := make([]*ssproto.SnapshotsResponse, len(tc.expectResponses))
  358. for i := 0; i < len(tc.expectResponses); i++ {
  359. e := <-rts.snapshotOutCh
  360. responses[i] = e.Message.(*ssproto.SnapshotsResponse)
  361. }
  362. require.Equal(t, tc.expectResponses, responses)
  363. require.Empty(t, rts.snapshotOutCh)
  364. })
  365. }
  366. }
  367. func TestReactor_LightBlockResponse(t *testing.T) {
  368. ctx, cancel := context.WithCancel(context.Background())
  369. defer cancel()
  370. rts := setup(ctx, t, nil, nil, nil, 2)
  371. var height int64 = 10
  372. // generates a random header
  373. h := factory.MakeHeader(t, &types.Header{})
  374. h.Height = height
  375. blockID := factory.MakeBlockIDWithHash(h.Hash())
  376. vals, pv := factory.RandValidatorSet(ctx, t, 1, 10)
  377. vote, err := factory.MakeVote(ctx, pv[0], h.ChainID, 0, h.Height, 0, 2,
  378. blockID, factory.DefaultTestTime)
  379. require.NoError(t, err)
  380. sh := &types.SignedHeader{
  381. Header: h,
  382. Commit: &types.Commit{
  383. Height: h.Height,
  384. BlockID: blockID,
  385. Signatures: []types.CommitSig{
  386. vote.CommitSig(),
  387. },
  388. },
  389. }
  390. lb := &types.LightBlock{
  391. SignedHeader: sh,
  392. ValidatorSet: vals,
  393. }
  394. require.NoError(t, rts.blockStore.SaveSignedHeader(sh, blockID))
  395. rts.stateStore.On("LoadValidators", height).Return(vals, nil)
  396. rts.blockInCh <- p2p.Envelope{
  397. From: types.NodeID("aa"),
  398. Message: &ssproto.LightBlockRequest{
  399. Height: 10,
  400. },
  401. }
  402. require.Empty(t, rts.blockPeerErrCh)
  403. select {
  404. case response := <-rts.blockOutCh:
  405. require.Equal(t, types.NodeID("aa"), response.To)
  406. res, ok := response.Message.(*ssproto.LightBlockResponse)
  407. require.True(t, ok)
  408. receivedLB, err := types.LightBlockFromProto(res.LightBlock)
  409. require.NoError(t, err)
  410. require.Equal(t, lb, receivedLB)
  411. case <-time.After(1 * time.Second):
  412. t.Fatal("expected light block response")
  413. }
  414. }
  415. func TestReactor_BlockProviders(t *testing.T) {
  416. ctx, cancel := context.WithCancel(context.Background())
  417. defer cancel()
  418. rts := setup(ctx, t, nil, nil, nil, 2)
  419. rts.peerUpdateCh <- p2p.PeerUpdate{
  420. NodeID: types.NodeID("aa"),
  421. Status: p2p.PeerStatusUp,
  422. }
  423. rts.peerUpdateCh <- p2p.PeerUpdate{
  424. NodeID: types.NodeID("bb"),
  425. Status: p2p.PeerStatusUp,
  426. }
  427. closeCh := make(chan struct{})
  428. defer close(closeCh)
  429. chain := buildLightBlockChain(ctx, t, 1, 10, time.Now())
  430. go handleLightBlockRequests(ctx, t, chain, rts.blockOutCh, rts.blockInCh, closeCh, 0)
  431. peers := rts.reactor.peers.All()
  432. require.Len(t, peers, 2)
  433. providers := make([]provider.Provider, len(peers))
  434. for idx, peer := range peers {
  435. providers[idx] = NewBlockProvider(peer, factory.DefaultTestChainID, rts.reactor.dispatcher)
  436. }
  437. wg := sync.WaitGroup{}
  438. for _, p := range providers {
  439. wg.Add(1)
  440. go func(t *testing.T, p provider.Provider) {
  441. defer wg.Done()
  442. for height := 2; height < 10; height++ {
  443. lb, err := p.LightBlock(ctx, int64(height))
  444. require.NoError(t, err)
  445. require.NotNil(t, lb)
  446. require.Equal(t, height, int(lb.Height))
  447. }
  448. }(t, p)
  449. }
  450. go func() { wg.Wait(); cancel() }()
  451. select {
  452. case <-time.After(time.Second):
  453. // not all of the requests to the dispatcher were responded to
  454. // within the timeout
  455. t.Fail()
  456. case <-ctx.Done():
  457. }
  458. }
  459. func TestReactor_StateProviderP2P(t *testing.T) {
  460. ctx, cancel := context.WithCancel(context.Background())
  461. defer cancel()
  462. rts := setup(ctx, t, nil, nil, nil, 2)
  463. // make syncer non nil else test won't think we are state syncing
  464. rts.reactor.syncer = rts.syncer
  465. peerA := types.NodeID(strings.Repeat("a", 2*types.NodeIDByteLength))
  466. peerB := types.NodeID(strings.Repeat("b", 2*types.NodeIDByteLength))
  467. rts.peerUpdateCh <- p2p.PeerUpdate{
  468. NodeID: peerA,
  469. Status: p2p.PeerStatusUp,
  470. }
  471. rts.peerUpdateCh <- p2p.PeerUpdate{
  472. NodeID: peerB,
  473. Status: p2p.PeerStatusUp,
  474. }
  475. closeCh := make(chan struct{})
  476. defer close(closeCh)
  477. chain := buildLightBlockChain(ctx, t, 1, 10, time.Now())
  478. go handleLightBlockRequests(ctx, t, chain, rts.blockOutCh, rts.blockInCh, closeCh, 0)
  479. go handleConsensusParamsRequest(ctx, t, rts.paramsOutCh, rts.paramsInCh, closeCh)
  480. rts.reactor.cfg.UseP2P = true
  481. rts.reactor.cfg.TrustHeight = 1
  482. rts.reactor.cfg.TrustHash = fmt.Sprintf("%X", chain[1].Hash())
  483. for _, p := range []types.NodeID{peerA, peerB} {
  484. if !rts.reactor.peers.Contains(p) {
  485. rts.reactor.peers.Append(p)
  486. }
  487. }
  488. require.True(t, rts.reactor.peers.Len() >= 2, "peer network not configured")
  489. ictx, cancel := context.WithTimeout(ctx, time.Second)
  490. defer cancel()
  491. rts.reactor.mtx.Lock()
  492. err := rts.reactor.initStateProvider(ictx, factory.DefaultTestChainID, 1)
  493. rts.reactor.mtx.Unlock()
  494. require.NoError(t, err)
  495. rts.reactor.syncer.stateProvider = rts.reactor.stateProvider
  496. actx, cancel := context.WithTimeout(ctx, 10*time.Second)
  497. defer cancel()
  498. appHash, err := rts.reactor.stateProvider.AppHash(actx, 5)
  499. require.NoError(t, err)
  500. require.Len(t, appHash, 32)
  501. state, err := rts.reactor.stateProvider.State(actx, 5)
  502. require.NoError(t, err)
  503. require.Equal(t, appHash, state.AppHash)
  504. require.Equal(t, types.DefaultConsensusParams(), &state.ConsensusParams)
  505. commit, err := rts.reactor.stateProvider.Commit(actx, 5)
  506. require.NoError(t, err)
  507. require.Equal(t, commit.BlockID, state.LastBlockID)
  508. added, err := rts.reactor.syncer.AddSnapshot(peerA, &snapshot{
  509. Height: 1, Format: 2, Chunks: 7, Hash: []byte{1, 2}, Metadata: []byte{1},
  510. })
  511. require.NoError(t, err)
  512. require.True(t, added)
  513. }
  514. func TestReactor_Backfill(t *testing.T) {
  515. ctx, cancel := context.WithCancel(context.Background())
  516. defer cancel()
  517. // test backfill algorithm with varying failure rates [0, 10]
  518. failureRates := []int{0, 2, 9}
  519. for _, failureRate := range failureRates {
  520. failureRate := failureRate
  521. t.Run(fmt.Sprintf("failure rate: %d", failureRate), func(t *testing.T) {
  522. ctx, cancel := context.WithCancel(ctx)
  523. defer cancel()
  524. t.Cleanup(leaktest.CheckTimeout(t, 1*time.Minute))
  525. rts := setup(ctx, t, nil, nil, nil, 21)
  526. var (
  527. startHeight int64 = 20
  528. stopHeight int64 = 10
  529. stopTime = time.Date(2020, 1, 1, 0, 100, 0, 0, time.UTC)
  530. )
  531. peers := []string{"a", "b", "c", "d"}
  532. for _, peer := range peers {
  533. rts.peerUpdateCh <- p2p.PeerUpdate{
  534. NodeID: types.NodeID(peer),
  535. Status: p2p.PeerStatusUp,
  536. }
  537. }
  538. trackingHeight := startHeight
  539. rts.stateStore.On("SaveValidatorSets", mock.AnythingOfType("int64"), mock.AnythingOfType("int64"),
  540. mock.AnythingOfType("*types.ValidatorSet")).Return(func(lh, uh int64, vals *types.ValidatorSet) error {
  541. require.Equal(t, trackingHeight, lh)
  542. require.Equal(t, lh, uh)
  543. require.GreaterOrEqual(t, lh, stopHeight)
  544. trackingHeight--
  545. return nil
  546. })
  547. chain := buildLightBlockChain(ctx, t, stopHeight-1, startHeight+1, stopTime)
  548. closeCh := make(chan struct{})
  549. defer close(closeCh)
  550. go handleLightBlockRequests(ctx, t, chain, rts.blockOutCh,
  551. rts.blockInCh, closeCh, failureRate)
  552. err := rts.reactor.backfill(
  553. ctx,
  554. factory.DefaultTestChainID,
  555. startHeight,
  556. stopHeight,
  557. 1,
  558. factory.MakeBlockIDWithHash(chain[startHeight].Header.Hash()),
  559. stopTime,
  560. )
  561. if failureRate > 3 {
  562. require.Error(t, err)
  563. require.NotEqual(t, rts.reactor.backfilledBlocks, rts.reactor.backfillBlockTotal)
  564. require.Equal(t, startHeight-stopHeight+1, rts.reactor.backfillBlockTotal)
  565. } else {
  566. require.NoError(t, err)
  567. for height := startHeight; height <= stopHeight; height++ {
  568. blockMeta := rts.blockStore.LoadBlockMeta(height)
  569. require.NotNil(t, blockMeta)
  570. }
  571. require.Nil(t, rts.blockStore.LoadBlockMeta(stopHeight-1))
  572. require.Nil(t, rts.blockStore.LoadBlockMeta(startHeight+1))
  573. require.Equal(t, startHeight-stopHeight+1, rts.reactor.backfilledBlocks)
  574. require.Equal(t, startHeight-stopHeight+1, rts.reactor.backfillBlockTotal)
  575. }
  576. require.Equal(t, rts.reactor.backfilledBlocks, rts.reactor.BackFilledBlocks())
  577. require.Equal(t, rts.reactor.backfillBlockTotal, rts.reactor.BackFillBlocksTotal())
  578. })
  579. }
  580. }
  581. // retryUntil will continue to evaluate fn and will return successfully when true
  582. // or fail when the timeout is reached.
  583. func retryUntil(ctx context.Context, t *testing.T, fn func() bool, timeout time.Duration) {
  584. ctx, cancel := context.WithTimeout(ctx, timeout)
  585. defer cancel()
  586. for {
  587. if fn() {
  588. return
  589. }
  590. require.NoError(t, ctx.Err())
  591. }
  592. }
  593. func handleLightBlockRequests(
  594. ctx context.Context,
  595. t *testing.T,
  596. chain map[int64]*types.LightBlock,
  597. receiving chan p2p.Envelope,
  598. sending chan p2p.Envelope,
  599. close chan struct{},
  600. failureRate int) {
  601. requests := 0
  602. errorCount := 0
  603. for {
  604. select {
  605. case <-ctx.Done():
  606. return
  607. case envelope := <-receiving:
  608. if msg, ok := envelope.Message.(*ssproto.LightBlockRequest); ok {
  609. if requests%10 >= failureRate {
  610. lb, err := chain[int64(msg.Height)].ToProto()
  611. require.NoError(t, err)
  612. sending <- p2p.Envelope{
  613. From: envelope.To,
  614. Message: &ssproto.LightBlockResponse{
  615. LightBlock: lb,
  616. },
  617. }
  618. } else {
  619. switch errorCount % 3 {
  620. case 0: // send a different block
  621. vals, pv := factory.RandValidatorSet(ctx, t, 3, 10)
  622. _, _, lb := mockLB(ctx, t, int64(msg.Height), factory.DefaultTestTime, factory.MakeBlockID(), vals, pv)
  623. differntLB, err := lb.ToProto()
  624. require.NoError(t, err)
  625. sending <- p2p.Envelope{
  626. From: envelope.To,
  627. Message: &ssproto.LightBlockResponse{
  628. LightBlock: differntLB,
  629. },
  630. }
  631. case 1: // send nil block i.e. pretend we don't have it
  632. sending <- p2p.Envelope{
  633. From: envelope.To,
  634. Message: &ssproto.LightBlockResponse{
  635. LightBlock: nil,
  636. },
  637. }
  638. case 2: // don't do anything
  639. }
  640. errorCount++
  641. }
  642. }
  643. case <-close:
  644. return
  645. }
  646. requests++
  647. }
  648. }
  649. func handleConsensusParamsRequest(
  650. ctx context.Context,
  651. t *testing.T,
  652. receiving, sending chan p2p.Envelope,
  653. closeCh chan struct{},
  654. ) {
  655. t.Helper()
  656. params := types.DefaultConsensusParams()
  657. paramsProto := params.ToProto()
  658. for {
  659. select {
  660. case <-ctx.Done():
  661. return
  662. case envelope := <-receiving:
  663. if ctx.Err() != nil {
  664. return
  665. }
  666. t.Log("received consensus params request")
  667. msg, ok := envelope.Message.(*ssproto.ParamsRequest)
  668. require.True(t, ok)
  669. sending <- p2p.Envelope{
  670. From: envelope.To,
  671. Message: &ssproto.ParamsResponse{
  672. Height: msg.Height,
  673. ConsensusParams: paramsProto,
  674. },
  675. }
  676. case <-closeCh:
  677. return
  678. }
  679. }
  680. }
  681. func buildLightBlockChain(ctx context.Context, t *testing.T, fromHeight, toHeight int64, startTime time.Time) map[int64]*types.LightBlock {
  682. t.Helper()
  683. chain := make(map[int64]*types.LightBlock, toHeight-fromHeight)
  684. lastBlockID := factory.MakeBlockID()
  685. blockTime := startTime.Add(time.Duration(fromHeight-toHeight) * time.Minute)
  686. vals, pv := factory.RandValidatorSet(ctx, t, 3, 10)
  687. for height := fromHeight; height < toHeight; height++ {
  688. vals, pv, chain[height] = mockLB(ctx, t, height, blockTime, lastBlockID, vals, pv)
  689. lastBlockID = factory.MakeBlockIDWithHash(chain[height].Header.Hash())
  690. blockTime = blockTime.Add(1 * time.Minute)
  691. }
  692. return chain
  693. }
  694. func mockLB(ctx context.Context, t *testing.T, height int64, time time.Time, lastBlockID types.BlockID,
  695. currentVals *types.ValidatorSet, currentPrivVals []types.PrivValidator,
  696. ) (*types.ValidatorSet, []types.PrivValidator, *types.LightBlock) {
  697. t.Helper()
  698. header := factory.MakeHeader(t, &types.Header{
  699. Height: height,
  700. LastBlockID: lastBlockID,
  701. Time: time,
  702. })
  703. header.Version.App = testAppVersion
  704. nextVals, nextPrivVals := factory.RandValidatorSet(ctx, t, 3, 10)
  705. header.ValidatorsHash = currentVals.Hash()
  706. header.NextValidatorsHash = nextVals.Hash()
  707. header.ConsensusHash = types.DefaultConsensusParams().HashConsensusParams()
  708. lastBlockID = factory.MakeBlockIDWithHash(header.Hash())
  709. voteSet := types.NewVoteSet(factory.DefaultTestChainID, height, 0, tmproto.PrecommitType, currentVals)
  710. commit, err := factory.MakeCommit(ctx, lastBlockID, height, 0, voteSet, currentPrivVals, time)
  711. require.NoError(t, err)
  712. return nextVals, nextPrivVals, &types.LightBlock{
  713. SignedHeader: &types.SignedHeader{
  714. Header: header,
  715. Commit: commit,
  716. },
  717. ValidatorSet: currentVals,
  718. }
  719. }
  720. // graduallyAddPeers delivers a new randomly-generated peer update on peerUpdateCh once
  721. // per interval, until closeCh is closed. Each peer update is assigned a random node ID.
  722. func graduallyAddPeers(
  723. t *testing.T,
  724. peerUpdateCh chan p2p.PeerUpdate,
  725. closeCh chan struct{},
  726. interval time.Duration,
  727. ) {
  728. ticker := time.NewTicker(interval)
  729. for {
  730. select {
  731. case <-ticker.C:
  732. peerUpdateCh <- p2p.PeerUpdate{
  733. NodeID: factory.RandomNodeID(t),
  734. Status: p2p.PeerStatusUp,
  735. }
  736. case <-closeCh:
  737. return
  738. }
  739. }
  740. }
  741. func handleSnapshotRequests(
  742. t *testing.T,
  743. receivingCh chan p2p.Envelope,
  744. sendingCh chan p2p.Envelope,
  745. closeCh chan struct{},
  746. snapshots []snapshot,
  747. ) {
  748. t.Helper()
  749. for {
  750. select {
  751. case envelope := <-receivingCh:
  752. _, ok := envelope.Message.(*ssproto.SnapshotsRequest)
  753. require.True(t, ok)
  754. for _, snapshot := range snapshots {
  755. sendingCh <- p2p.Envelope{
  756. From: envelope.To,
  757. Message: &ssproto.SnapshotsResponse{
  758. Height: snapshot.Height,
  759. Format: snapshot.Format,
  760. Chunks: snapshot.Chunks,
  761. Hash: snapshot.Hash,
  762. Metadata: snapshot.Metadata,
  763. },
  764. }
  765. }
  766. case <-closeCh:
  767. return
  768. }
  769. }
  770. }
  771. func handleChunkRequests(
  772. t *testing.T,
  773. receivingCh chan p2p.Envelope,
  774. sendingCh chan p2p.Envelope,
  775. closeCh chan struct{},
  776. chunk []byte,
  777. ) {
  778. t.Helper()
  779. for {
  780. select {
  781. case envelope := <-receivingCh:
  782. msg, ok := envelope.Message.(*ssproto.ChunkRequest)
  783. require.True(t, ok)
  784. sendingCh <- p2p.Envelope{
  785. From: envelope.To,
  786. Message: &ssproto.ChunkResponse{
  787. Height: msg.Height,
  788. Format: msg.Format,
  789. Index: msg.Index,
  790. Chunk: chunk,
  791. Missing: false,
  792. },
  793. }
  794. case <-closeCh:
  795. return
  796. }
  797. }
  798. }