You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

774 lines
27 KiB

statesync: remove deadlock on init fail (#7029) When statesync is stopped during shutdown, it has the possibility of deadlocking. A dump of goroutines reveals that this is related to the peerUpdates channel not returning anything on its `Done()` channel when `OnStop` is called. As this is occuring, `processPeerUpdate` is attempting to acquire the reactor lock. It appears that this lock can never be acquired. I looked for the places where the lock may remain locked accidentally and cleaned them up in hopes to eradicate the issue. Dumps of the relevant goroutines may be found below. Note that the line numbers below are relative to the code in the `v0.35.0-rc1` tag. ``` goroutine 36 [chan receive]: github.com/tendermint/tendermint/internal/statesync.(*Reactor).OnStop(0xc00058f200) github.com/tendermint/tendermint/internal/statesync/reactor.go:243 +0x117 github.com/tendermint/tendermint/libs/service.(*BaseService).Stop(0xc00058f200, 0x0, 0x0) github.com/tendermint/tendermint/libs/service/service.go:171 +0x323 github.com/tendermint/tendermint/node.(*nodeImpl).OnStop(0xc0001ea240) github.com/tendermint/tendermint/node/node.go:769 +0x132 github.com/tendermint/tendermint/libs/service.(*BaseService).Stop(0xc0001ea240, 0x0, 0x0) github.com/tendermint/tendermint/libs/service/service.go:171 +0x323 github.com/tendermint/tendermint/cmd/tendermint/commands.NewRunNodeCmd.func1.1() github.com/tendermint/tendermint/cmd/tendermint/commands/run_node.go:143 +0x62 github.com/tendermint/tendermint/libs/os.TrapSignal.func1(0xc000629500, 0x7fdb52f96358, 0xc0002b5030, 0xc00000daa0) github.com/tendermint/tendermint/libs/os/os.go:26 +0x102 created by github.com/tendermint/tendermint/libs/os.TrapSignal github.com/tendermint/tendermint/libs/os/os.go:22 +0xe6 goroutine 188 [semacquire]: sync.runtime_SemacquireMutex(0xc00026b1cc, 0x0, 0x1) runtime/sema.go:71 +0x47 sync.(*Mutex).lockSlow(0xc00026b1c8) sync/mutex.go:138 +0x105 sync.(*Mutex).Lock(...) sync/mutex.go:81 sync.(*RWMutex).Lock(0xc00026b1c8) sync/rwmutex.go:111 +0x90 github.com/tendermint/tendermint/internal/statesync.(*Reactor).processPeerUpdate(0xc00026b080, 0xc000650008, 0x28, 0x124de90, 0x4) github.com/tendermint/tendermint/internal/statesync/reactor.go:849 +0x1a5 github.com/tendermint/tendermint/internal/statesync.(*Reactor).processPeerUpdates(0xc00026b080) github.com/tendermint/tendermint/internal/statesync/reactor.go:883 +0xab created by github.com/tendermint/tendermint/internal/statesync.(*Reactor.OnStart github.com/tendermint/tendermint/internal/statesync/reactor.go:219 +0xcd) ```
3 years ago
statesync: remove deadlock on init fail (#7029) When statesync is stopped during shutdown, it has the possibility of deadlocking. A dump of goroutines reveals that this is related to the peerUpdates channel not returning anything on its `Done()` channel when `OnStop` is called. As this is occuring, `processPeerUpdate` is attempting to acquire the reactor lock. It appears that this lock can never be acquired. I looked for the places where the lock may remain locked accidentally and cleaned them up in hopes to eradicate the issue. Dumps of the relevant goroutines may be found below. Note that the line numbers below are relative to the code in the `v0.35.0-rc1` tag. ``` goroutine 36 [chan receive]: github.com/tendermint/tendermint/internal/statesync.(*Reactor).OnStop(0xc00058f200) github.com/tendermint/tendermint/internal/statesync/reactor.go:243 +0x117 github.com/tendermint/tendermint/libs/service.(*BaseService).Stop(0xc00058f200, 0x0, 0x0) github.com/tendermint/tendermint/libs/service/service.go:171 +0x323 github.com/tendermint/tendermint/node.(*nodeImpl).OnStop(0xc0001ea240) github.com/tendermint/tendermint/node/node.go:769 +0x132 github.com/tendermint/tendermint/libs/service.(*BaseService).Stop(0xc0001ea240, 0x0, 0x0) github.com/tendermint/tendermint/libs/service/service.go:171 +0x323 github.com/tendermint/tendermint/cmd/tendermint/commands.NewRunNodeCmd.func1.1() github.com/tendermint/tendermint/cmd/tendermint/commands/run_node.go:143 +0x62 github.com/tendermint/tendermint/libs/os.TrapSignal.func1(0xc000629500, 0x7fdb52f96358, 0xc0002b5030, 0xc00000daa0) github.com/tendermint/tendermint/libs/os/os.go:26 +0x102 created by github.com/tendermint/tendermint/libs/os.TrapSignal github.com/tendermint/tendermint/libs/os/os.go:22 +0xe6 goroutine 188 [semacquire]: sync.runtime_SemacquireMutex(0xc00026b1cc, 0x0, 0x1) runtime/sema.go:71 +0x47 sync.(*Mutex).lockSlow(0xc00026b1c8) sync/mutex.go:138 +0x105 sync.(*Mutex).Lock(...) sync/mutex.go:81 sync.(*RWMutex).Lock(0xc00026b1c8) sync/rwmutex.go:111 +0x90 github.com/tendermint/tendermint/internal/statesync.(*Reactor).processPeerUpdate(0xc00026b080, 0xc000650008, 0x28, 0x124de90, 0x4) github.com/tendermint/tendermint/internal/statesync/reactor.go:849 +0x1a5 github.com/tendermint/tendermint/internal/statesync.(*Reactor).processPeerUpdates(0xc00026b080) github.com/tendermint/tendermint/internal/statesync/reactor.go:883 +0xab created by github.com/tendermint/tendermint/internal/statesync.(*Reactor.OnStart github.com/tendermint/tendermint/internal/statesync/reactor.go:219 +0xcd) ```
3 years ago
  1. package statesync
  2. import (
  3. "context"
  4. "errors"
  5. "sync"
  6. "testing"
  7. "time"
  8. "github.com/stretchr/testify/assert"
  9. "github.com/stretchr/testify/mock"
  10. "github.com/stretchr/testify/require"
  11. abci "github.com/tendermint/tendermint/abci/types"
  12. "github.com/tendermint/tendermint/internal/proxy"
  13. proxymocks "github.com/tendermint/tendermint/internal/proxy/mocks"
  14. sm "github.com/tendermint/tendermint/internal/state"
  15. "github.com/tendermint/tendermint/internal/statesync/mocks"
  16. ssproto "github.com/tendermint/tendermint/proto/tendermint/statesync"
  17. "github.com/tendermint/tendermint/types"
  18. "github.com/tendermint/tendermint/version"
  19. )
  20. func TestSyncer_SyncAny(t *testing.T) {
  21. ctx, cancel := context.WithCancel(context.Background())
  22. defer cancel()
  23. state := sm.State{
  24. ChainID: "chain",
  25. Version: sm.Version{
  26. Consensus: version.Consensus{
  27. Block: version.BlockProtocol,
  28. App: 0,
  29. },
  30. Software: version.TMVersion,
  31. },
  32. LastBlockHeight: 1,
  33. LastBlockID: types.BlockID{Hash: []byte("blockhash")},
  34. LastBlockTime: time.Now(),
  35. LastResultsHash: []byte("last_results_hash"),
  36. AppHash: []byte("app_hash"),
  37. LastValidators: &types.ValidatorSet{Proposer: &types.Validator{Address: []byte("val1")}},
  38. Validators: &types.ValidatorSet{Proposer: &types.Validator{Address: []byte("val2")}},
  39. NextValidators: &types.ValidatorSet{Proposer: &types.Validator{Address: []byte("val3")}},
  40. ConsensusParams: *types.DefaultConsensusParams(),
  41. LastHeightConsensusParamsChanged: 1,
  42. }
  43. commit := &types.Commit{BlockID: types.BlockID{Hash: []byte("blockhash")}}
  44. chunks := []*chunk{
  45. {Height: 1, Format: 1, Index: 0, Chunk: []byte{1, 1, 0}},
  46. {Height: 1, Format: 1, Index: 1, Chunk: []byte{1, 1, 1}},
  47. {Height: 1, Format: 1, Index: 2, Chunk: []byte{1, 1, 2}},
  48. }
  49. s := &snapshot{Height: 1, Format: 1, Chunks: 3, Hash: []byte{1, 2, 3}}
  50. stateProvider := &mocks.StateProvider{}
  51. stateProvider.On("AppHash", mock.Anything, uint64(1)).Return(state.AppHash, nil)
  52. stateProvider.On("AppHash", mock.Anything, uint64(2)).Return([]byte("app_hash_2"), nil)
  53. stateProvider.On("Commit", mock.Anything, uint64(1)).Return(commit, nil)
  54. stateProvider.On("State", mock.Anything, uint64(1)).Return(state, nil)
  55. connSnapshot := &proxymocks.AppConnSnapshot{}
  56. connQuery := &proxymocks.AppConnQuery{}
  57. peerAID := types.NodeID("aa")
  58. peerBID := types.NodeID("bb")
  59. peerCID := types.NodeID("cc")
  60. rts := setup(ctx, t, connSnapshot, connQuery, stateProvider, 3)
  61. rts.reactor.syncer = rts.syncer
  62. // Adding a chunk should error when no sync is in progress
  63. _, err := rts.syncer.AddChunk(&chunk{Height: 1, Format: 1, Index: 0, Chunk: []byte{1}})
  64. require.Error(t, err)
  65. // Adding a couple of peers should trigger snapshot discovery messages
  66. err = rts.syncer.AddPeer(ctx, peerAID)
  67. require.NoError(t, err)
  68. e := <-rts.snapshotOutCh
  69. require.Equal(t, &ssproto.SnapshotsRequest{}, e.Message)
  70. require.Equal(t, peerAID, e.To)
  71. err = rts.syncer.AddPeer(ctx, peerBID)
  72. require.NoError(t, err)
  73. e = <-rts.snapshotOutCh
  74. require.Equal(t, &ssproto.SnapshotsRequest{}, e.Message)
  75. require.Equal(t, peerBID, e.To)
  76. // Both peers report back with snapshots. One of them also returns a snapshot we don't want, in
  77. // format 2, which will be rejected by the ABCI application.
  78. new, err := rts.syncer.AddSnapshot(peerAID, s)
  79. require.NoError(t, err)
  80. require.True(t, new)
  81. new, err = rts.syncer.AddSnapshot(peerBID, s)
  82. require.NoError(t, err)
  83. require.False(t, new)
  84. s2 := &snapshot{Height: 2, Format: 2, Chunks: 3, Hash: []byte{1}}
  85. new, err = rts.syncer.AddSnapshot(peerBID, s2)
  86. require.NoError(t, err)
  87. require.True(t, new)
  88. new, err = rts.syncer.AddSnapshot(peerCID, s2)
  89. require.NoError(t, err)
  90. require.False(t, new)
  91. // We start a sync, with peers sending back chunks when requested. We first reject the snapshot
  92. // with height 2 format 2, and accept the snapshot at height 1.
  93. connSnapshot.On("OfferSnapshotSync", mock.Anything, abci.RequestOfferSnapshot{
  94. Snapshot: &abci.Snapshot{
  95. Height: 2,
  96. Format: 2,
  97. Chunks: 3,
  98. Hash: []byte{1},
  99. },
  100. AppHash: []byte("app_hash_2"),
  101. }).Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_REJECT_FORMAT}, nil)
  102. connSnapshot.On("OfferSnapshotSync", mock.Anything, abci.RequestOfferSnapshot{
  103. Snapshot: &abci.Snapshot{
  104. Height: s.Height,
  105. Format: s.Format,
  106. Chunks: s.Chunks,
  107. Hash: s.Hash,
  108. Metadata: s.Metadata,
  109. },
  110. AppHash: []byte("app_hash"),
  111. }).Times(2).Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_ACCEPT}, nil)
  112. chunkRequests := make(map[uint32]int)
  113. chunkRequestsMtx := sync.Mutex{}
  114. var wg sync.WaitGroup
  115. wg.Add(4)
  116. go func() {
  117. for e := range rts.chunkOutCh {
  118. msg, ok := e.Message.(*ssproto.ChunkRequest)
  119. assert.True(t, ok)
  120. assert.EqualValues(t, 1, msg.Height)
  121. assert.EqualValues(t, 1, msg.Format)
  122. assert.LessOrEqual(t, msg.Index, uint32(len(chunks)))
  123. added, err := rts.syncer.AddChunk(chunks[msg.Index])
  124. assert.NoError(t, err)
  125. assert.True(t, added)
  126. chunkRequestsMtx.Lock()
  127. chunkRequests[msg.Index]++
  128. chunkRequestsMtx.Unlock()
  129. wg.Done()
  130. }
  131. }()
  132. // The first time we're applying chunk 2 we tell it to retry the snapshot and discard chunk 1,
  133. // which should cause it to keep the existing chunk 0 and 2, and restart restoration from
  134. // beginning. We also wait for a little while, to exercise the retry logic in fetchChunks().
  135. connSnapshot.On("ApplySnapshotChunkSync", mock.Anything, abci.RequestApplySnapshotChunk{
  136. Index: 2, Chunk: []byte{1, 1, 2},
  137. }).Once().Run(func(args mock.Arguments) { time.Sleep(2 * time.Second) }).Return(
  138. &abci.ResponseApplySnapshotChunk{
  139. Result: abci.ResponseApplySnapshotChunk_RETRY_SNAPSHOT,
  140. RefetchChunks: []uint32{1},
  141. }, nil)
  142. connSnapshot.On("ApplySnapshotChunkSync", mock.Anything, abci.RequestApplySnapshotChunk{
  143. Index: 0, Chunk: []byte{1, 1, 0},
  144. }).Times(2).Return(&abci.ResponseApplySnapshotChunk{Result: abci.ResponseApplySnapshotChunk_ACCEPT}, nil)
  145. connSnapshot.On("ApplySnapshotChunkSync", mock.Anything, abci.RequestApplySnapshotChunk{
  146. Index: 1, Chunk: []byte{1, 1, 1},
  147. }).Times(2).Return(&abci.ResponseApplySnapshotChunk{Result: abci.ResponseApplySnapshotChunk_ACCEPT}, nil)
  148. connSnapshot.On("ApplySnapshotChunkSync", mock.Anything, abci.RequestApplySnapshotChunk{
  149. Index: 2, Chunk: []byte{1, 1, 2},
  150. }).Once().Return(&abci.ResponseApplySnapshotChunk{Result: abci.ResponseApplySnapshotChunk_ACCEPT}, nil)
  151. connQuery.On("InfoSync", mock.Anything, proxy.RequestInfo).Return(&abci.ResponseInfo{
  152. AppVersion: 9,
  153. LastBlockHeight: 1,
  154. LastBlockAppHash: []byte("app_hash"),
  155. }, nil)
  156. newState, lastCommit, err := rts.syncer.SyncAny(ctx, 0, func() error { return nil })
  157. require.NoError(t, err)
  158. wg.Wait()
  159. chunkRequestsMtx.Lock()
  160. require.Equal(t, map[uint32]int{0: 1, 1: 2, 2: 1}, chunkRequests)
  161. chunkRequestsMtx.Unlock()
  162. // The syncer should have updated the state app version from the ABCI info response.
  163. expectState := state
  164. expectState.Version.Consensus.App = 9
  165. require.Equal(t, expectState, newState)
  166. require.Equal(t, commit, lastCommit)
  167. require.Equal(t, len(chunks), int(rts.syncer.processingSnapshot.Chunks))
  168. require.Equal(t, expectState.LastBlockHeight, rts.syncer.lastSyncedSnapshotHeight)
  169. require.True(t, rts.syncer.avgChunkTime > 0)
  170. require.Equal(t, int64(rts.syncer.processingSnapshot.Chunks), rts.reactor.SnapshotChunksTotal())
  171. require.Equal(t, rts.syncer.lastSyncedSnapshotHeight, rts.reactor.SnapshotHeight())
  172. require.Equal(t, time.Duration(rts.syncer.avgChunkTime), rts.reactor.ChunkProcessAvgTime())
  173. require.Equal(t, int64(len(rts.syncer.snapshots.snapshots)), rts.reactor.TotalSnapshots())
  174. require.Equal(t, int64(0), rts.reactor.SnapshotChunksCount())
  175. connSnapshot.AssertExpectations(t)
  176. connQuery.AssertExpectations(t)
  177. }
  178. func TestSyncer_SyncAny_noSnapshots(t *testing.T) {
  179. stateProvider := &mocks.StateProvider{}
  180. stateProvider.On("AppHash", mock.Anything, mock.Anything).Return([]byte("app_hash"), nil)
  181. ctx, cancel := context.WithCancel(context.Background())
  182. defer cancel()
  183. rts := setup(ctx, t, nil, nil, stateProvider, 2)
  184. _, _, err := rts.syncer.SyncAny(ctx, 0, func() error { return nil })
  185. require.Equal(t, errNoSnapshots, err)
  186. }
  187. func TestSyncer_SyncAny_abort(t *testing.T) {
  188. stateProvider := &mocks.StateProvider{}
  189. stateProvider.On("AppHash", mock.Anything, mock.Anything).Return([]byte("app_hash"), nil)
  190. ctx, cancel := context.WithCancel(context.Background())
  191. defer cancel()
  192. rts := setup(ctx, t, nil, nil, stateProvider, 2)
  193. s := &snapshot{Height: 1, Format: 1, Chunks: 3, Hash: []byte{1, 2, 3}}
  194. peerID := types.NodeID("aa")
  195. _, err := rts.syncer.AddSnapshot(peerID, s)
  196. require.NoError(t, err)
  197. rts.conn.On("OfferSnapshotSync", mock.Anything, abci.RequestOfferSnapshot{
  198. Snapshot: toABCI(s), AppHash: []byte("app_hash"),
  199. }).Once().Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_ABORT}, nil)
  200. _, _, err = rts.syncer.SyncAny(ctx, 0, func() error { return nil })
  201. require.Equal(t, errAbort, err)
  202. rts.conn.AssertExpectations(t)
  203. }
  204. func TestSyncer_SyncAny_reject(t *testing.T) {
  205. stateProvider := &mocks.StateProvider{}
  206. stateProvider.On("AppHash", mock.Anything, mock.Anything).Return([]byte("app_hash"), nil)
  207. ctx, cancel := context.WithCancel(context.Background())
  208. defer cancel()
  209. rts := setup(ctx, t, nil, nil, stateProvider, 2)
  210. // s22 is tried first, then s12, then s11, then errNoSnapshots
  211. s22 := &snapshot{Height: 2, Format: 2, Chunks: 3, Hash: []byte{1, 2, 3}}
  212. s12 := &snapshot{Height: 1, Format: 2, Chunks: 3, Hash: []byte{1, 2, 3}}
  213. s11 := &snapshot{Height: 1, Format: 1, Chunks: 3, Hash: []byte{1, 2, 3}}
  214. peerID := types.NodeID("aa")
  215. _, err := rts.syncer.AddSnapshot(peerID, s22)
  216. require.NoError(t, err)
  217. _, err = rts.syncer.AddSnapshot(peerID, s12)
  218. require.NoError(t, err)
  219. _, err = rts.syncer.AddSnapshot(peerID, s11)
  220. require.NoError(t, err)
  221. rts.conn.On("OfferSnapshotSync", mock.Anything, abci.RequestOfferSnapshot{
  222. Snapshot: toABCI(s22), AppHash: []byte("app_hash"),
  223. }).Once().Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_REJECT}, nil)
  224. rts.conn.On("OfferSnapshotSync", mock.Anything, abci.RequestOfferSnapshot{
  225. Snapshot: toABCI(s12), AppHash: []byte("app_hash"),
  226. }).Once().Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_REJECT}, nil)
  227. rts.conn.On("OfferSnapshotSync", mock.Anything, abci.RequestOfferSnapshot{
  228. Snapshot: toABCI(s11), AppHash: []byte("app_hash"),
  229. }).Once().Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_REJECT}, nil)
  230. _, _, err = rts.syncer.SyncAny(ctx, 0, func() error { return nil })
  231. require.Equal(t, errNoSnapshots, err)
  232. rts.conn.AssertExpectations(t)
  233. }
  234. func TestSyncer_SyncAny_reject_format(t *testing.T) {
  235. stateProvider := &mocks.StateProvider{}
  236. stateProvider.On("AppHash", mock.Anything, mock.Anything).Return([]byte("app_hash"), nil)
  237. ctx, cancel := context.WithCancel(context.Background())
  238. defer cancel()
  239. rts := setup(ctx, t, nil, nil, stateProvider, 2)
  240. // s22 is tried first, which reject s22 and s12, then s11 will abort.
  241. s22 := &snapshot{Height: 2, Format: 2, Chunks: 3, Hash: []byte{1, 2, 3}}
  242. s12 := &snapshot{Height: 1, Format: 2, Chunks: 3, Hash: []byte{1, 2, 3}}
  243. s11 := &snapshot{Height: 1, Format: 1, Chunks: 3, Hash: []byte{1, 2, 3}}
  244. peerID := types.NodeID("aa")
  245. _, err := rts.syncer.AddSnapshot(peerID, s22)
  246. require.NoError(t, err)
  247. _, err = rts.syncer.AddSnapshot(peerID, s12)
  248. require.NoError(t, err)
  249. _, err = rts.syncer.AddSnapshot(peerID, s11)
  250. require.NoError(t, err)
  251. rts.conn.On("OfferSnapshotSync", mock.Anything, abci.RequestOfferSnapshot{
  252. Snapshot: toABCI(s22), AppHash: []byte("app_hash"),
  253. }).Once().Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_REJECT_FORMAT}, nil)
  254. rts.conn.On("OfferSnapshotSync", mock.Anything, abci.RequestOfferSnapshot{
  255. Snapshot: toABCI(s11), AppHash: []byte("app_hash"),
  256. }).Once().Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_ABORT}, nil)
  257. _, _, err = rts.syncer.SyncAny(ctx, 0, func() error { return nil })
  258. require.Equal(t, errAbort, err)
  259. rts.conn.AssertExpectations(t)
  260. }
  261. func TestSyncer_SyncAny_reject_sender(t *testing.T) {
  262. stateProvider := &mocks.StateProvider{}
  263. stateProvider.On("AppHash", mock.Anything, mock.Anything).Return([]byte("app_hash"), nil)
  264. ctx, cancel := context.WithCancel(context.Background())
  265. defer cancel()
  266. rts := setup(ctx, t, nil, nil, stateProvider, 2)
  267. peerAID := types.NodeID("aa")
  268. peerBID := types.NodeID("bb")
  269. peerCID := types.NodeID("cc")
  270. // sbc will be offered first, which will be rejected with reject_sender, causing all snapshots
  271. // submitted by both b and c (i.e. sb, sc, sbc) to be rejected. Finally, sa will reject and
  272. // errNoSnapshots is returned.
  273. sa := &snapshot{Height: 1, Format: 1, Chunks: 3, Hash: []byte{1, 2, 3}}
  274. sb := &snapshot{Height: 2, Format: 1, Chunks: 3, Hash: []byte{1, 2, 3}}
  275. sc := &snapshot{Height: 3, Format: 1, Chunks: 3, Hash: []byte{1, 2, 3}}
  276. sbc := &snapshot{Height: 4, Format: 1, Chunks: 3, Hash: []byte{1, 2, 3}}
  277. _, err := rts.syncer.AddSnapshot(peerAID, sa)
  278. require.NoError(t, err)
  279. _, err = rts.syncer.AddSnapshot(peerBID, sb)
  280. require.NoError(t, err)
  281. _, err = rts.syncer.AddSnapshot(peerCID, sc)
  282. require.NoError(t, err)
  283. _, err = rts.syncer.AddSnapshot(peerBID, sbc)
  284. require.NoError(t, err)
  285. _, err = rts.syncer.AddSnapshot(peerCID, sbc)
  286. require.NoError(t, err)
  287. rts.conn.On("OfferSnapshotSync", mock.Anything, abci.RequestOfferSnapshot{
  288. Snapshot: toABCI(sbc), AppHash: []byte("app_hash"),
  289. }).Once().Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_REJECT_SENDER}, nil)
  290. rts.conn.On("OfferSnapshotSync", mock.Anything, abci.RequestOfferSnapshot{
  291. Snapshot: toABCI(sa), AppHash: []byte("app_hash"),
  292. }).Once().Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_REJECT}, nil)
  293. _, _, err = rts.syncer.SyncAny(ctx, 0, func() error { return nil })
  294. require.Equal(t, errNoSnapshots, err)
  295. rts.conn.AssertExpectations(t)
  296. }
  297. func TestSyncer_SyncAny_abciError(t *testing.T) {
  298. stateProvider := &mocks.StateProvider{}
  299. stateProvider.On("AppHash", mock.Anything, mock.Anything).Return([]byte("app_hash"), nil)
  300. ctx, cancel := context.WithCancel(context.Background())
  301. defer cancel()
  302. rts := setup(ctx, t, nil, nil, stateProvider, 2)
  303. errBoom := errors.New("boom")
  304. s := &snapshot{Height: 1, Format: 1, Chunks: 3, Hash: []byte{1, 2, 3}}
  305. peerID := types.NodeID("aa")
  306. _, err := rts.syncer.AddSnapshot(peerID, s)
  307. require.NoError(t, err)
  308. rts.conn.On("OfferSnapshotSync", mock.Anything, abci.RequestOfferSnapshot{
  309. Snapshot: toABCI(s), AppHash: []byte("app_hash"),
  310. }).Once().Return(nil, errBoom)
  311. _, _, err = rts.syncer.SyncAny(ctx, 0, func() error { return nil })
  312. require.True(t, errors.Is(err, errBoom))
  313. rts.conn.AssertExpectations(t)
  314. }
  315. func TestSyncer_offerSnapshot(t *testing.T) {
  316. unknownErr := errors.New("unknown error")
  317. boom := errors.New("boom")
  318. testcases := map[string]struct {
  319. result abci.ResponseOfferSnapshot_Result
  320. err error
  321. expectErr error
  322. }{
  323. "accept": {abci.ResponseOfferSnapshot_ACCEPT, nil, nil},
  324. "abort": {abci.ResponseOfferSnapshot_ABORT, nil, errAbort},
  325. "reject": {abci.ResponseOfferSnapshot_REJECT, nil, errRejectSnapshot},
  326. "reject_format": {abci.ResponseOfferSnapshot_REJECT_FORMAT, nil, errRejectFormat},
  327. "reject_sender": {abci.ResponseOfferSnapshot_REJECT_SENDER, nil, errRejectSender},
  328. "unknown": {abci.ResponseOfferSnapshot_UNKNOWN, nil, unknownErr},
  329. "error": {0, boom, boom},
  330. "unknown non-zero": {9, nil, unknownErr},
  331. }
  332. ctx, cancel := context.WithCancel(context.Background())
  333. defer cancel()
  334. for name, tc := range testcases {
  335. tc := tc
  336. t.Run(name, func(t *testing.T) {
  337. ctx, cancel := context.WithCancel(ctx)
  338. defer cancel()
  339. stateProvider := &mocks.StateProvider{}
  340. stateProvider.On("AppHash", mock.Anything, mock.Anything).Return([]byte("app_hash"), nil)
  341. rts := setup(ctx, t, nil, nil, stateProvider, 2)
  342. s := &snapshot{Height: 1, Format: 1, Chunks: 3, Hash: []byte{1, 2, 3}, trustedAppHash: []byte("app_hash")}
  343. rts.conn.On("OfferSnapshotSync", mock.Anything, abci.RequestOfferSnapshot{
  344. Snapshot: toABCI(s),
  345. AppHash: []byte("app_hash"),
  346. }).Return(&abci.ResponseOfferSnapshot{Result: tc.result}, tc.err)
  347. err := rts.syncer.offerSnapshot(ctx, s)
  348. if tc.expectErr == unknownErr {
  349. require.Error(t, err)
  350. } else {
  351. unwrapped := errors.Unwrap(err)
  352. if unwrapped != nil {
  353. err = unwrapped
  354. }
  355. require.Equal(t, tc.expectErr, err)
  356. }
  357. })
  358. }
  359. }
  360. func TestSyncer_applyChunks_Results(t *testing.T) {
  361. unknownErr := errors.New("unknown error")
  362. boom := errors.New("boom")
  363. testcases := map[string]struct {
  364. result abci.ResponseApplySnapshotChunk_Result
  365. err error
  366. expectErr error
  367. }{
  368. "accept": {abci.ResponseApplySnapshotChunk_ACCEPT, nil, nil},
  369. "abort": {abci.ResponseApplySnapshotChunk_ABORT, nil, errAbort},
  370. "retry": {abci.ResponseApplySnapshotChunk_RETRY, nil, nil},
  371. "retry_snapshot": {abci.ResponseApplySnapshotChunk_RETRY_SNAPSHOT, nil, errRetrySnapshot},
  372. "reject_snapshot": {abci.ResponseApplySnapshotChunk_REJECT_SNAPSHOT, nil, errRejectSnapshot},
  373. "unknown": {abci.ResponseApplySnapshotChunk_UNKNOWN, nil, unknownErr},
  374. "error": {0, boom, boom},
  375. "unknown non-zero": {9, nil, unknownErr},
  376. }
  377. ctx, cancel := context.WithCancel(context.Background())
  378. defer cancel()
  379. for name, tc := range testcases {
  380. tc := tc
  381. t.Run(name, func(t *testing.T) {
  382. ctx, cancel := context.WithCancel(ctx)
  383. defer cancel()
  384. stateProvider := &mocks.StateProvider{}
  385. stateProvider.On("AppHash", mock.Anything, mock.Anything).Return([]byte("app_hash"), nil)
  386. rts := setup(ctx, t, nil, nil, stateProvider, 2)
  387. body := []byte{1, 2, 3}
  388. chunks, err := newChunkQueue(&snapshot{Height: 1, Format: 1, Chunks: 1}, "")
  389. require.NoError(t, err)
  390. fetchStartTime := time.Now()
  391. _, err = chunks.Add(&chunk{Height: 1, Format: 1, Index: 0, Chunk: body})
  392. require.NoError(t, err)
  393. rts.conn.On("ApplySnapshotChunkSync", mock.Anything, abci.RequestApplySnapshotChunk{
  394. Index: 0, Chunk: body,
  395. }).Once().Return(&abci.ResponseApplySnapshotChunk{Result: tc.result}, tc.err)
  396. if tc.result == abci.ResponseApplySnapshotChunk_RETRY {
  397. rts.conn.On("ApplySnapshotChunkSync", mock.Anything, abci.RequestApplySnapshotChunk{
  398. Index: 0, Chunk: body,
  399. }).Once().Return(&abci.ResponseApplySnapshotChunk{
  400. Result: abci.ResponseApplySnapshotChunk_ACCEPT}, nil)
  401. }
  402. err = rts.syncer.applyChunks(ctx, chunks, fetchStartTime)
  403. if tc.expectErr == unknownErr {
  404. require.Error(t, err)
  405. } else {
  406. unwrapped := errors.Unwrap(err)
  407. if unwrapped != nil {
  408. err = unwrapped
  409. }
  410. require.Equal(t, tc.expectErr, err)
  411. }
  412. rts.conn.AssertExpectations(t)
  413. })
  414. }
  415. }
  416. func TestSyncer_applyChunks_RefetchChunks(t *testing.T) {
  417. // Discarding chunks via refetch_chunks should work the same for all results
  418. testcases := map[string]struct {
  419. result abci.ResponseApplySnapshotChunk_Result
  420. }{
  421. "accept": {abci.ResponseApplySnapshotChunk_ACCEPT},
  422. "abort": {abci.ResponseApplySnapshotChunk_ABORT},
  423. "retry": {abci.ResponseApplySnapshotChunk_RETRY},
  424. "retry_snapshot": {abci.ResponseApplySnapshotChunk_RETRY_SNAPSHOT},
  425. "reject_snapshot": {abci.ResponseApplySnapshotChunk_REJECT_SNAPSHOT},
  426. }
  427. ctx, cancel := context.WithCancel(context.Background())
  428. defer cancel()
  429. for name, tc := range testcases {
  430. tc := tc
  431. t.Run(name, func(t *testing.T) {
  432. ctx, cancel := context.WithCancel(ctx)
  433. defer cancel()
  434. stateProvider := &mocks.StateProvider{}
  435. stateProvider.On("AppHash", mock.Anything, mock.Anything).Return([]byte("app_hash"), nil)
  436. rts := setup(ctx, t, nil, nil, stateProvider, 2)
  437. chunks, err := newChunkQueue(&snapshot{Height: 1, Format: 1, Chunks: 3}, "")
  438. require.NoError(t, err)
  439. fetchStartTime := time.Now()
  440. added, err := chunks.Add(&chunk{Height: 1, Format: 1, Index: 0, Chunk: []byte{0}})
  441. require.True(t, added)
  442. require.NoError(t, err)
  443. added, err = chunks.Add(&chunk{Height: 1, Format: 1, Index: 1, Chunk: []byte{1}})
  444. require.True(t, added)
  445. require.NoError(t, err)
  446. added, err = chunks.Add(&chunk{Height: 1, Format: 1, Index: 2, Chunk: []byte{2}})
  447. require.True(t, added)
  448. require.NoError(t, err)
  449. // The first two chunks are accepted, before the last one asks for 1 to be refetched
  450. rts.conn.On("ApplySnapshotChunkSync", mock.Anything, abci.RequestApplySnapshotChunk{
  451. Index: 0, Chunk: []byte{0},
  452. }).Once().Return(&abci.ResponseApplySnapshotChunk{Result: abci.ResponseApplySnapshotChunk_ACCEPT}, nil)
  453. rts.conn.On("ApplySnapshotChunkSync", mock.Anything, abci.RequestApplySnapshotChunk{
  454. Index: 1, Chunk: []byte{1},
  455. }).Once().Return(&abci.ResponseApplySnapshotChunk{Result: abci.ResponseApplySnapshotChunk_ACCEPT}, nil)
  456. rts.conn.On("ApplySnapshotChunkSync", mock.Anything, abci.RequestApplySnapshotChunk{
  457. Index: 2, Chunk: []byte{2},
  458. }).Once().Return(&abci.ResponseApplySnapshotChunk{
  459. Result: tc.result,
  460. RefetchChunks: []uint32{1},
  461. }, nil)
  462. // Since removing the chunk will cause Next() to block, we spawn a goroutine, then
  463. // check the queue contents, and finally close the queue to end the goroutine.
  464. // We don't really care about the result of applyChunks, since it has separate test.
  465. go func() {
  466. rts.syncer.applyChunks(ctx, chunks, fetchStartTime) //nolint:errcheck // purposefully ignore error
  467. }()
  468. time.Sleep(50 * time.Millisecond)
  469. require.True(t, chunks.Has(0))
  470. require.False(t, chunks.Has(1))
  471. require.True(t, chunks.Has(2))
  472. require.NoError(t, chunks.Close())
  473. })
  474. }
  475. }
  476. func TestSyncer_applyChunks_RejectSenders(t *testing.T) {
  477. // Banning chunks senders via ban_chunk_senders should work the same for all results
  478. testcases := map[string]struct {
  479. result abci.ResponseApplySnapshotChunk_Result
  480. }{
  481. "accept": {abci.ResponseApplySnapshotChunk_ACCEPT},
  482. "abort": {abci.ResponseApplySnapshotChunk_ABORT},
  483. "retry": {abci.ResponseApplySnapshotChunk_RETRY},
  484. "retry_snapshot": {abci.ResponseApplySnapshotChunk_RETRY_SNAPSHOT},
  485. "reject_snapshot": {abci.ResponseApplySnapshotChunk_REJECT_SNAPSHOT},
  486. }
  487. ctx, cancel := context.WithCancel(context.Background())
  488. defer cancel()
  489. for name, tc := range testcases {
  490. tc := tc
  491. t.Run(name, func(t *testing.T) {
  492. ctx, cancel := context.WithCancel(ctx)
  493. defer cancel()
  494. stateProvider := &mocks.StateProvider{}
  495. stateProvider.On("AppHash", mock.Anything, mock.Anything).Return([]byte("app_hash"), nil)
  496. rts := setup(ctx, t, nil, nil, stateProvider, 2)
  497. // Set up three peers across two snapshots, and ask for one of them to be banned.
  498. // It should be banned from all snapshots.
  499. peerAID := types.NodeID("aa")
  500. peerBID := types.NodeID("bb")
  501. peerCID := types.NodeID("cc")
  502. s1 := &snapshot{Height: 1, Format: 1, Chunks: 3}
  503. s2 := &snapshot{Height: 2, Format: 1, Chunks: 3}
  504. _, err := rts.syncer.AddSnapshot(peerAID, s1)
  505. require.NoError(t, err)
  506. _, err = rts.syncer.AddSnapshot(peerAID, s2)
  507. require.NoError(t, err)
  508. _, err = rts.syncer.AddSnapshot(peerBID, s1)
  509. require.NoError(t, err)
  510. _, err = rts.syncer.AddSnapshot(peerBID, s2)
  511. require.NoError(t, err)
  512. _, err = rts.syncer.AddSnapshot(peerCID, s1)
  513. require.NoError(t, err)
  514. _, err = rts.syncer.AddSnapshot(peerCID, s2)
  515. require.NoError(t, err)
  516. chunks, err := newChunkQueue(s1, "")
  517. require.NoError(t, err)
  518. fetchStartTime := time.Now()
  519. added, err := chunks.Add(&chunk{Height: 1, Format: 1, Index: 0, Chunk: []byte{0}, Sender: peerAID})
  520. require.True(t, added)
  521. require.NoError(t, err)
  522. added, err = chunks.Add(&chunk{Height: 1, Format: 1, Index: 1, Chunk: []byte{1}, Sender: peerBID})
  523. require.True(t, added)
  524. require.NoError(t, err)
  525. added, err = chunks.Add(&chunk{Height: 1, Format: 1, Index: 2, Chunk: []byte{2}, Sender: peerCID})
  526. require.True(t, added)
  527. require.NoError(t, err)
  528. // The first two chunks are accepted, before the last one asks for b sender to be rejected
  529. rts.conn.On("ApplySnapshotChunkSync", mock.Anything, abci.RequestApplySnapshotChunk{
  530. Index: 0, Chunk: []byte{0}, Sender: "aa",
  531. }).Once().Return(&abci.ResponseApplySnapshotChunk{Result: abci.ResponseApplySnapshotChunk_ACCEPT}, nil)
  532. rts.conn.On("ApplySnapshotChunkSync", mock.Anything, abci.RequestApplySnapshotChunk{
  533. Index: 1, Chunk: []byte{1}, Sender: "bb",
  534. }).Once().Return(&abci.ResponseApplySnapshotChunk{Result: abci.ResponseApplySnapshotChunk_ACCEPT}, nil)
  535. rts.conn.On("ApplySnapshotChunkSync", mock.Anything, abci.RequestApplySnapshotChunk{
  536. Index: 2, Chunk: []byte{2}, Sender: "cc",
  537. }).Once().Return(&abci.ResponseApplySnapshotChunk{
  538. Result: tc.result,
  539. RejectSenders: []string{string(peerBID)},
  540. }, nil)
  541. // On retry, the last chunk will be tried again, so we just accept it then.
  542. if tc.result == abci.ResponseApplySnapshotChunk_RETRY {
  543. rts.conn.On("ApplySnapshotChunkSync", mock.Anything, abci.RequestApplySnapshotChunk{
  544. Index: 2, Chunk: []byte{2}, Sender: "cc",
  545. }).Once().Return(&abci.ResponseApplySnapshotChunk{Result: abci.ResponseApplySnapshotChunk_ACCEPT}, nil)
  546. }
  547. // We don't really care about the result of applyChunks, since it has separate test.
  548. // However, it will block on e.g. retry result, so we spawn a goroutine that will
  549. // be shut down when the chunk queue closes.
  550. go func() {
  551. rts.syncer.applyChunks(ctx, chunks, fetchStartTime) //nolint:errcheck // purposefully ignore error
  552. }()
  553. time.Sleep(50 * time.Millisecond)
  554. s1peers := rts.syncer.snapshots.GetPeers(s1)
  555. require.Len(t, s1peers, 2)
  556. require.EqualValues(t, "aa", s1peers[0])
  557. require.EqualValues(t, "cc", s1peers[1])
  558. rts.syncer.snapshots.GetPeers(s1)
  559. require.Len(t, s1peers, 2)
  560. require.EqualValues(t, "aa", s1peers[0])
  561. require.EqualValues(t, "cc", s1peers[1])
  562. require.NoError(t, chunks.Close())
  563. })
  564. }
  565. }
  566. func TestSyncer_verifyApp(t *testing.T) {
  567. boom := errors.New("boom")
  568. s := &snapshot{Height: 3, Format: 1, Chunks: 5, Hash: []byte{1, 2, 3}, trustedAppHash: []byte("app_hash")}
  569. testcases := map[string]struct {
  570. response *abci.ResponseInfo
  571. err error
  572. expectErr error
  573. }{
  574. "verified": {&abci.ResponseInfo{
  575. LastBlockHeight: 3,
  576. LastBlockAppHash: []byte("app_hash"),
  577. AppVersion: 9,
  578. }, nil, nil},
  579. "invalid height": {&abci.ResponseInfo{
  580. LastBlockHeight: 5,
  581. LastBlockAppHash: []byte("app_hash"),
  582. AppVersion: 9,
  583. }, nil, errVerifyFailed},
  584. "invalid hash": {&abci.ResponseInfo{
  585. LastBlockHeight: 3,
  586. LastBlockAppHash: []byte("xxx"),
  587. AppVersion: 9,
  588. }, nil, errVerifyFailed},
  589. "error": {nil, boom, boom},
  590. }
  591. ctx, cancel := context.WithCancel(context.Background())
  592. defer cancel()
  593. for name, tc := range testcases {
  594. tc := tc
  595. t.Run(name, func(t *testing.T) {
  596. ctx, cancel := context.WithCancel(ctx)
  597. defer cancel()
  598. rts := setup(ctx, t, nil, nil, nil, 2)
  599. rts.connQuery.On("InfoSync", mock.Anything, proxy.RequestInfo).Return(tc.response, tc.err)
  600. version, err := rts.syncer.verifyApp(ctx, s)
  601. unwrapped := errors.Unwrap(err)
  602. if unwrapped != nil {
  603. err = unwrapped
  604. }
  605. require.Equal(t, tc.expectErr, err)
  606. if err == nil {
  607. require.Equal(t, tc.response.AppVersion, version)
  608. }
  609. })
  610. }
  611. }
  612. func toABCI(s *snapshot) *abci.Snapshot {
  613. return &abci.Snapshot{
  614. Height: s.Height,
  615. Format: s.Format,
  616. Chunks: s.Chunks,
  617. Hash: s.Hash,
  618. Metadata: s.Metadata,
  619. }
  620. }