You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

554 lines
15 KiB

  1. package statesync
  2. import (
  3. "io/ioutil"
  4. "os"
  5. "testing"
  6. "github.com/stretchr/testify/assert"
  7. "github.com/stretchr/testify/require"
  8. "github.com/tendermint/tendermint/p2p"
  9. )
  10. func setupChunkQueue(t *testing.T) (*chunkQueue, func()) {
  11. snapshot := &snapshot{
  12. Height: 3,
  13. Format: 1,
  14. Chunks: 5,
  15. Hash: []byte{7},
  16. Metadata: nil,
  17. }
  18. queue, err := newChunkQueue(snapshot, "")
  19. require.NoError(t, err)
  20. teardown := func() {
  21. err := queue.Close()
  22. require.NoError(t, err)
  23. }
  24. return queue, teardown
  25. }
  26. func TestNewChunkQueue_TempDir(t *testing.T) {
  27. snapshot := &snapshot{
  28. Height: 3,
  29. Format: 1,
  30. Chunks: 5,
  31. Hash: []byte{7},
  32. Metadata: nil,
  33. }
  34. dir, err := ioutil.TempDir("", "newchunkqueue")
  35. require.NoError(t, err)
  36. defer os.RemoveAll(dir)
  37. queue, err := newChunkQueue(snapshot, dir)
  38. require.NoError(t, err)
  39. files, err := ioutil.ReadDir(dir)
  40. require.NoError(t, err)
  41. assert.Len(t, files, 1)
  42. err = queue.Close()
  43. require.NoError(t, err)
  44. files, err = ioutil.ReadDir(dir)
  45. require.NoError(t, err)
  46. assert.Len(t, files, 0)
  47. }
  48. func TestChunkQueue(t *testing.T) {
  49. queue, teardown := setupChunkQueue(t)
  50. defer teardown()
  51. // Adding the first chunk should be fine
  52. added, err := queue.Add(&chunk{Height: 3, Format: 1, Index: 0, Chunk: []byte{3, 1, 0}})
  53. require.NoError(t, err)
  54. assert.True(t, added)
  55. // Adding the last chunk should also be fine
  56. added, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 4, Chunk: []byte{3, 1, 4}})
  57. require.NoError(t, err)
  58. assert.True(t, added)
  59. // Adding the first or last chunks again should return false
  60. added, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 0, Chunk: []byte{3, 1, 0}})
  61. require.NoError(t, err)
  62. assert.False(t, added)
  63. added, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 4, Chunk: []byte{3, 1, 4}})
  64. require.NoError(t, err)
  65. assert.False(t, added)
  66. // Adding the remaining chunks in reverse should be fine
  67. added, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 3, Chunk: []byte{3, 1, 3}})
  68. require.NoError(t, err)
  69. assert.True(t, added)
  70. added, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 2, Chunk: []byte{3, 1, 2}})
  71. require.NoError(t, err)
  72. assert.True(t, added)
  73. added, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 1, Chunk: []byte{3, 1, 1}})
  74. require.NoError(t, err)
  75. assert.True(t, added)
  76. // At this point, we should be able to retrieve them all via Next
  77. for i := 0; i < 5; i++ {
  78. c, err := queue.Next()
  79. require.NoError(t, err)
  80. assert.Equal(t, &chunk{Height: 3, Format: 1, Index: uint32(i), Chunk: []byte{3, 1, byte(i)}}, c)
  81. }
  82. _, err = queue.Next()
  83. require.Error(t, err)
  84. assert.Equal(t, errDone, err)
  85. // It should still be possible to try to add chunks (which will be ignored)
  86. added, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 0, Chunk: []byte{3, 1, 0}})
  87. require.NoError(t, err)
  88. assert.False(t, added)
  89. // After closing the queue it will also return false
  90. err = queue.Close()
  91. require.NoError(t, err)
  92. added, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 0, Chunk: []byte{3, 1, 0}})
  93. require.NoError(t, err)
  94. assert.False(t, added)
  95. // Closing the queue again should also be fine
  96. err = queue.Close()
  97. require.NoError(t, err)
  98. }
  99. func TestChunkQueue_Add_ChunkErrors(t *testing.T) {
  100. testcases := map[string]struct {
  101. chunk *chunk
  102. }{
  103. "nil chunk": {nil},
  104. "nil body": {&chunk{Height: 3, Format: 1, Index: 0, Chunk: nil}},
  105. "wrong height": {&chunk{Height: 9, Format: 1, Index: 0, Chunk: []byte{3, 1, 0}}},
  106. "wrong format": {&chunk{Height: 3, Format: 9, Index: 0, Chunk: []byte{3, 1, 0}}},
  107. "invalid index": {&chunk{Height: 3, Format: 1, Index: 5, Chunk: []byte{3, 1, 0}}},
  108. }
  109. for name, tc := range testcases {
  110. tc := tc
  111. t.Run(name, func(t *testing.T) {
  112. queue, teardown := setupChunkQueue(t)
  113. defer teardown()
  114. _, err := queue.Add(tc.chunk)
  115. require.Error(t, err)
  116. })
  117. }
  118. }
  119. func TestChunkQueue_Allocate(t *testing.T) {
  120. queue, teardown := setupChunkQueue(t)
  121. defer teardown()
  122. for i := uint32(0); i < queue.Size(); i++ {
  123. index, err := queue.Allocate()
  124. require.NoError(t, err)
  125. assert.EqualValues(t, i, index)
  126. }
  127. _, err := queue.Allocate()
  128. require.Error(t, err)
  129. assert.Equal(t, errDone, err)
  130. for i := uint32(0); i < queue.Size(); i++ {
  131. _, err = queue.Add(&chunk{Height: 3, Format: 1, Index: i, Chunk: []byte{byte(i)}})
  132. require.NoError(t, err)
  133. }
  134. // After all chunks have been allocated and retrieved, discarding a chunk will reallocate it.
  135. err = queue.Discard(2)
  136. require.NoError(t, err)
  137. index, err := queue.Allocate()
  138. require.NoError(t, err)
  139. assert.EqualValues(t, 2, index)
  140. _, err = queue.Allocate()
  141. require.Error(t, err)
  142. assert.Equal(t, errDone, err)
  143. // Discarding a chunk the closing the queue will return errDone.
  144. err = queue.Discard(2)
  145. require.NoError(t, err)
  146. err = queue.Close()
  147. require.NoError(t, err)
  148. _, err = queue.Allocate()
  149. require.Error(t, err)
  150. assert.Equal(t, errDone, err)
  151. }
  152. func TestChunkQueue_Discard(t *testing.T) {
  153. queue, teardown := setupChunkQueue(t)
  154. defer teardown()
  155. // Add a few chunks to the queue and fetch a couple
  156. _, err := queue.Add(&chunk{Height: 3, Format: 1, Index: 0, Chunk: []byte{byte(0)}})
  157. require.NoError(t, err)
  158. _, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 1, Chunk: []byte{byte(1)}})
  159. require.NoError(t, err)
  160. _, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 2, Chunk: []byte{byte(2)}})
  161. require.NoError(t, err)
  162. c, err := queue.Next()
  163. require.NoError(t, err)
  164. assert.EqualValues(t, 0, c.Index)
  165. c, err = queue.Next()
  166. require.NoError(t, err)
  167. assert.EqualValues(t, 1, c.Index)
  168. // Discarding the first chunk and re-adding it should cause it to be returned
  169. // immediately by Next(), before procceeding with chunk 2
  170. err = queue.Discard(0)
  171. require.NoError(t, err)
  172. added, err := queue.Add(&chunk{Height: 3, Format: 1, Index: 0, Chunk: []byte{byte(0)}})
  173. require.NoError(t, err)
  174. assert.True(t, added)
  175. c, err = queue.Next()
  176. require.NoError(t, err)
  177. assert.EqualValues(t, 0, c.Index)
  178. c, err = queue.Next()
  179. require.NoError(t, err)
  180. assert.EqualValues(t, 2, c.Index)
  181. // Discard then allocate, add and fetch all chunks
  182. for i := uint32(0); i < queue.Size(); i++ {
  183. err := queue.Discard(i)
  184. require.NoError(t, err)
  185. }
  186. for i := uint32(0); i < queue.Size(); i++ {
  187. _, err := queue.Allocate()
  188. require.NoError(t, err)
  189. _, err = queue.Add(&chunk{Height: 3, Format: 1, Index: i, Chunk: []byte{byte(i)}})
  190. require.NoError(t, err)
  191. c, err = queue.Next()
  192. require.NoError(t, err)
  193. assert.EqualValues(t, i, c.Index)
  194. }
  195. // Discarding a non-existent chunk does nothing.
  196. err = queue.Discard(99)
  197. require.NoError(t, err)
  198. // When discard a couple of chunks, we should be able to allocate, add, and fetch them again.
  199. err = queue.Discard(3)
  200. require.NoError(t, err)
  201. err = queue.Discard(1)
  202. require.NoError(t, err)
  203. index, err := queue.Allocate()
  204. require.NoError(t, err)
  205. assert.EqualValues(t, 1, index)
  206. index, err = queue.Allocate()
  207. require.NoError(t, err)
  208. assert.EqualValues(t, 3, index)
  209. added, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 3, Chunk: []byte{3}})
  210. require.NoError(t, err)
  211. assert.True(t, added)
  212. added, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 1, Chunk: []byte{1}})
  213. require.NoError(t, err)
  214. assert.True(t, added)
  215. chunk, err := queue.Next()
  216. require.NoError(t, err)
  217. assert.EqualValues(t, 1, chunk.Index)
  218. chunk, err = queue.Next()
  219. require.NoError(t, err)
  220. assert.EqualValues(t, 3, chunk.Index)
  221. _, err = queue.Next()
  222. require.Error(t, err)
  223. assert.Equal(t, errDone, err)
  224. // After closing the queue, discarding does nothing
  225. err = queue.Close()
  226. require.NoError(t, err)
  227. err = queue.Discard(2)
  228. require.NoError(t, err)
  229. }
  230. func TestChunkQueue_DiscardSender(t *testing.T) {
  231. queue, teardown := setupChunkQueue(t)
  232. defer teardown()
  233. // Allocate and add all chunks to the queue
  234. senders := []p2p.NodeID{p2p.NodeID("a"), p2p.NodeID("b"), p2p.NodeID("c")}
  235. for i := uint32(0); i < queue.Size(); i++ {
  236. _, err := queue.Allocate()
  237. require.NoError(t, err)
  238. _, err = queue.Add(&chunk{
  239. Height: 3,
  240. Format: 1,
  241. Index: i,
  242. Chunk: []byte{byte(i)},
  243. Sender: senders[int(i)%len(senders)],
  244. })
  245. require.NoError(t, err)
  246. }
  247. // Fetch the first three chunks
  248. for i := uint32(0); i < 3; i++ {
  249. _, err := queue.Next()
  250. require.NoError(t, err)
  251. }
  252. // Discarding an unknown sender should do nothing
  253. err := queue.DiscardSender(p2p.NodeID("x"))
  254. require.NoError(t, err)
  255. _, err = queue.Allocate()
  256. assert.Equal(t, errDone, err)
  257. // Discarding sender b should discard chunk 4, but not chunk 1 which has already been
  258. // returned.
  259. err = queue.DiscardSender(p2p.NodeID("b"))
  260. require.NoError(t, err)
  261. index, err := queue.Allocate()
  262. require.NoError(t, err)
  263. assert.EqualValues(t, 4, index)
  264. _, err = queue.Allocate()
  265. assert.Equal(t, errDone, err)
  266. }
  267. func TestChunkQueue_GetSender(t *testing.T) {
  268. queue, teardown := setupChunkQueue(t)
  269. defer teardown()
  270. peerAID := p2p.NodeID("aa")
  271. peerBID := p2p.NodeID("bb")
  272. _, err := queue.Add(&chunk{Height: 3, Format: 1, Index: 0, Chunk: []byte{1}, Sender: peerAID})
  273. require.NoError(t, err)
  274. _, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 1, Chunk: []byte{2}, Sender: peerBID})
  275. require.NoError(t, err)
  276. assert.EqualValues(t, "aa", queue.GetSender(0))
  277. assert.EqualValues(t, "bb", queue.GetSender(1))
  278. assert.EqualValues(t, "", queue.GetSender(2))
  279. // After the chunk has been processed, we should still know who the sender was
  280. chunk, err := queue.Next()
  281. require.NoError(t, err)
  282. require.NotNil(t, chunk)
  283. require.EqualValues(t, 0, chunk.Index)
  284. assert.EqualValues(t, "aa", queue.GetSender(0))
  285. }
  286. func TestChunkQueue_Next(t *testing.T) {
  287. queue, teardown := setupChunkQueue(t)
  288. defer teardown()
  289. // Next should block waiting for the next chunks, even when given out of order.
  290. chNext := make(chan *chunk, 10)
  291. go func() {
  292. for {
  293. c, err := queue.Next()
  294. if err == errDone {
  295. close(chNext)
  296. break
  297. }
  298. require.NoError(t, err)
  299. chNext <- c
  300. }
  301. }()
  302. assert.Empty(t, chNext)
  303. _, err := queue.Add(&chunk{Height: 3, Format: 1, Index: 1, Chunk: []byte{3, 1, 1}, Sender: p2p.NodeID("b")})
  304. require.NoError(t, err)
  305. select {
  306. case <-chNext:
  307. assert.Fail(t, "channel should be empty")
  308. default:
  309. }
  310. _, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 0, Chunk: []byte{3, 1, 0}, Sender: p2p.NodeID("a")})
  311. require.NoError(t, err)
  312. assert.Equal(t,
  313. &chunk{Height: 3, Format: 1, Index: 0, Chunk: []byte{3, 1, 0}, Sender: p2p.NodeID("a")},
  314. <-chNext)
  315. assert.Equal(t,
  316. &chunk{Height: 3, Format: 1, Index: 1, Chunk: []byte{3, 1, 1}, Sender: p2p.NodeID("b")},
  317. <-chNext)
  318. _, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 4, Chunk: []byte{3, 1, 4}, Sender: p2p.NodeID("e")})
  319. require.NoError(t, err)
  320. select {
  321. case <-chNext:
  322. assert.Fail(t, "channel should be empty")
  323. default:
  324. }
  325. _, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 2, Chunk: []byte{3, 1, 2}, Sender: p2p.NodeID("c")})
  326. require.NoError(t, err)
  327. _, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 3, Chunk: []byte{3, 1, 3}, Sender: p2p.NodeID("d")})
  328. require.NoError(t, err)
  329. assert.Equal(t,
  330. &chunk{Height: 3, Format: 1, Index: 2, Chunk: []byte{3, 1, 2}, Sender: p2p.NodeID("c")},
  331. <-chNext)
  332. assert.Equal(t,
  333. &chunk{Height: 3, Format: 1, Index: 3, Chunk: []byte{3, 1, 3}, Sender: p2p.NodeID("d")},
  334. <-chNext)
  335. assert.Equal(t,
  336. &chunk{Height: 3, Format: 1, Index: 4, Chunk: []byte{3, 1, 4}, Sender: p2p.NodeID("e")},
  337. <-chNext)
  338. _, ok := <-chNext
  339. assert.False(t, ok, "channel should be closed")
  340. // Calling next on a finished queue should return done
  341. _, err = queue.Next()
  342. assert.Equal(t, errDone, err)
  343. }
  344. func TestChunkQueue_Next_Closed(t *testing.T) {
  345. queue, teardown := setupChunkQueue(t)
  346. defer teardown()
  347. // Calling Next on a closed queue should return done
  348. _, err := queue.Add(&chunk{Height: 3, Format: 1, Index: 1, Chunk: []byte{3, 1, 1}})
  349. require.NoError(t, err)
  350. err = queue.Close()
  351. require.NoError(t, err)
  352. _, err = queue.Next()
  353. assert.Equal(t, errDone, err)
  354. }
  355. func TestChunkQueue_Retry(t *testing.T) {
  356. queue, teardown := setupChunkQueue(t)
  357. defer teardown()
  358. // Allocate and add all chunks to the queue
  359. for i := uint32(0); i < queue.Size(); i++ {
  360. _, err := queue.Allocate()
  361. require.NoError(t, err)
  362. _, err = queue.Add(&chunk{Height: 3, Format: 1, Index: i, Chunk: []byte{byte(i)}})
  363. require.NoError(t, err)
  364. _, err = queue.Next()
  365. require.NoError(t, err)
  366. }
  367. // Retrying a couple of chunks makes Next() return them, but they are not allocatable
  368. queue.Retry(3)
  369. queue.Retry(1)
  370. _, err := queue.Allocate()
  371. assert.Equal(t, errDone, err)
  372. chunk, err := queue.Next()
  373. require.NoError(t, err)
  374. assert.EqualValues(t, 1, chunk.Index)
  375. chunk, err = queue.Next()
  376. require.NoError(t, err)
  377. assert.EqualValues(t, 3, chunk.Index)
  378. _, err = queue.Next()
  379. assert.Equal(t, errDone, err)
  380. }
  381. func TestChunkQueue_RetryAll(t *testing.T) {
  382. queue, teardown := setupChunkQueue(t)
  383. defer teardown()
  384. // Allocate and add all chunks to the queue
  385. for i := uint32(0); i < queue.Size(); i++ {
  386. _, err := queue.Allocate()
  387. require.NoError(t, err)
  388. _, err = queue.Add(&chunk{Height: 3, Format: 1, Index: i, Chunk: []byte{byte(i)}})
  389. require.NoError(t, err)
  390. _, err = queue.Next()
  391. require.NoError(t, err)
  392. }
  393. _, err := queue.Next()
  394. assert.Equal(t, errDone, err)
  395. queue.RetryAll()
  396. _, err = queue.Allocate()
  397. assert.Equal(t, errDone, err)
  398. for i := uint32(0); i < queue.Size(); i++ {
  399. chunk, err := queue.Next()
  400. require.NoError(t, err)
  401. assert.EqualValues(t, i, chunk.Index)
  402. }
  403. _, err = queue.Next()
  404. assert.Equal(t, errDone, err)
  405. }
  406. func TestChunkQueue_Size(t *testing.T) {
  407. queue, teardown := setupChunkQueue(t)
  408. defer teardown()
  409. assert.EqualValues(t, 5, queue.Size())
  410. err := queue.Close()
  411. require.NoError(t, err)
  412. assert.EqualValues(t, 0, queue.Size())
  413. }
  414. func TestChunkQueue_WaitFor(t *testing.T) {
  415. queue, teardown := setupChunkQueue(t)
  416. defer teardown()
  417. waitFor1 := queue.WaitFor(1)
  418. waitFor4 := queue.WaitFor(4)
  419. // Adding 0 and 2 should not trigger waiters
  420. _, err := queue.Add(&chunk{Height: 3, Format: 1, Index: 0, Chunk: []byte{3, 1, 0}})
  421. require.NoError(t, err)
  422. _, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 2, Chunk: []byte{3, 1, 2}})
  423. require.NoError(t, err)
  424. select {
  425. case <-waitFor1:
  426. require.Fail(t, "WaitFor(1) should not trigger on 0 or 2")
  427. case <-waitFor4:
  428. require.Fail(t, "WaitFor(4) should not trigger on 0 or 2")
  429. default:
  430. }
  431. // Adding 1 should trigger WaitFor(1), but not WaitFor(4). The channel should be closed.
  432. _, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 1, Chunk: []byte{3, 1, 1}})
  433. require.NoError(t, err)
  434. assert.EqualValues(t, 1, <-waitFor1)
  435. _, ok := <-waitFor1
  436. assert.False(t, ok)
  437. select {
  438. case <-waitFor4:
  439. require.Fail(t, "WaitFor(4) should not trigger on 0 or 2")
  440. default:
  441. }
  442. // Fetch the first chunk. At this point, waiting for either 0 (retrieved from pool) or 1
  443. // (queued in pool) should immediately return true.
  444. c, err := queue.Next()
  445. require.NoError(t, err)
  446. assert.EqualValues(t, 0, c.Index)
  447. w := queue.WaitFor(0)
  448. assert.EqualValues(t, 0, <-w)
  449. _, ok = <-w
  450. assert.False(t, ok)
  451. w = queue.WaitFor(1)
  452. assert.EqualValues(t, 1, <-w)
  453. _, ok = <-w
  454. assert.False(t, ok)
  455. // Close the queue. This should cause the waiter for 4 to close, and also cause any future
  456. // waiters to get closed channels.
  457. err = queue.Close()
  458. require.NoError(t, err)
  459. _, ok = <-waitFor4
  460. assert.False(t, ok)
  461. w = queue.WaitFor(3)
  462. _, ok = <-w
  463. assert.False(t, ok)
  464. }