You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

283 lines
6.7 KiB

  1. package statesync
  2. import (
  3. "math/rand"
  4. "sync"
  5. "testing"
  6. "time"
  7. "github.com/stretchr/testify/assert"
  8. "github.com/stretchr/testify/require"
  9. "github.com/tendermint/tendermint/internal/test/factory"
  10. "github.com/tendermint/tendermint/types"
  11. )
  12. var (
  13. startHeight int64 = 200
  14. stopHeight int64 = 100
  15. stopTime = time.Date(2019, 1, 1, 1, 0, 0, 0, time.UTC)
  16. endTime = stopTime.Add(-1 * time.Second)
  17. numWorkers = 1
  18. )
  19. func TestBlockQueueBasic(t *testing.T) {
  20. peerID, err := types.NewNodeID("0011223344556677889900112233445566778899")
  21. require.NoError(t, err)
  22. queue := newBlockQueue(startHeight, stopHeight, 1, stopTime, 1)
  23. wg := &sync.WaitGroup{}
  24. // asynchronously fetch blocks and add it to the queue
  25. for i := 0; i <= numWorkers; i++ {
  26. wg.Add(1)
  27. go func() {
  28. for {
  29. select {
  30. case height := <-queue.nextHeight():
  31. queue.add(mockLBResp(t, peerID, height, endTime))
  32. case <-queue.done():
  33. wg.Done()
  34. return
  35. }
  36. }
  37. }()
  38. }
  39. trackingHeight := startHeight
  40. wg.Add(1)
  41. loop:
  42. for {
  43. select {
  44. case <-queue.done():
  45. wg.Done()
  46. break loop
  47. case resp := <-queue.verifyNext():
  48. // assert that the queue serializes the blocks
  49. require.Equal(t, resp.block.Height, trackingHeight)
  50. trackingHeight--
  51. queue.success(resp.block.Height)
  52. }
  53. }
  54. wg.Wait()
  55. assert.Less(t, trackingHeight, stopHeight)
  56. }
  57. // Test with spurious failures and retries
  58. func TestBlockQueueWithFailures(t *testing.T) {
  59. peerID, err := types.NewNodeID("0011223344556677889900112233445566778899")
  60. require.NoError(t, err)
  61. queue := newBlockQueue(startHeight, stopHeight, 1, stopTime, 200)
  62. wg := &sync.WaitGroup{}
  63. failureRate := 4
  64. for i := 0; i <= numWorkers; i++ {
  65. wg.Add(1)
  66. go func() {
  67. for {
  68. select {
  69. case height := <-queue.nextHeight():
  70. if rand.Intn(failureRate) == 0 {
  71. queue.retry(height)
  72. } else {
  73. queue.add(mockLBResp(t, peerID, height, endTime))
  74. }
  75. case <-queue.done():
  76. wg.Done()
  77. return
  78. }
  79. }
  80. }()
  81. }
  82. trackingHeight := startHeight
  83. for {
  84. select {
  85. case resp := <-queue.verifyNext():
  86. // assert that the queue serializes the blocks
  87. assert.Equal(t, resp.block.Height, trackingHeight)
  88. if rand.Intn(failureRate) == 0 {
  89. queue.retry(resp.block.Height)
  90. } else {
  91. trackingHeight--
  92. queue.success(resp.block.Height)
  93. }
  94. case <-queue.done():
  95. wg.Wait()
  96. assert.Less(t, trackingHeight, stopHeight)
  97. return
  98. }
  99. }
  100. }
  101. // Test that when all the blocks are retrieved that the queue still holds on to
  102. // it's workers and in the event of failure can still fetch the failed block
  103. func TestBlockQueueBlocks(t *testing.T) {
  104. peerID, err := types.NewNodeID("0011223344556677889900112233445566778899")
  105. require.NoError(t, err)
  106. queue := newBlockQueue(startHeight, stopHeight, 1, stopTime, 2)
  107. expectedHeight := startHeight
  108. retryHeight := stopHeight + 2
  109. loop:
  110. for {
  111. select {
  112. case height := <-queue.nextHeight():
  113. require.Equal(t, height, expectedHeight)
  114. require.GreaterOrEqual(t, height, stopHeight)
  115. expectedHeight--
  116. queue.add(mockLBResp(t, peerID, height, endTime))
  117. case <-time.After(1 * time.Second):
  118. if expectedHeight >= stopHeight {
  119. t.Fatalf("expected next height %d", expectedHeight)
  120. }
  121. break loop
  122. }
  123. }
  124. // close any waiter channels that the previous worker left hanging
  125. for _, ch := range queue.waiters {
  126. close(ch)
  127. }
  128. queue.waiters = make([]chan int64, 0)
  129. wg := &sync.WaitGroup{}
  130. wg.Add(1)
  131. // so far so good. The worker is waiting. Now we fail a previous
  132. // block and check that the worker fetches them
  133. go func(t *testing.T) {
  134. defer wg.Done()
  135. select {
  136. case height := <-queue.nextHeight():
  137. require.Equal(t, retryHeight, height)
  138. case <-time.After(1 * time.Second):
  139. require.Fail(t, "queue didn't ask worker to fetch failed height")
  140. }
  141. }(t)
  142. queue.retry(retryHeight)
  143. wg.Wait()
  144. }
  145. func TestBlockQueueAcceptsNoMoreBlocks(t *testing.T) {
  146. peerID, err := types.NewNodeID("0011223344556677889900112233445566778899")
  147. require.NoError(t, err)
  148. queue := newBlockQueue(startHeight, stopHeight, 1, stopTime, 1)
  149. defer queue.close()
  150. loop:
  151. for {
  152. select {
  153. case height := <-queue.nextHeight():
  154. require.GreaterOrEqual(t, height, stopHeight)
  155. queue.add(mockLBResp(t, peerID, height, endTime))
  156. case <-time.After(1 * time.Second):
  157. break loop
  158. }
  159. }
  160. require.Len(t, queue.pending, int(startHeight-stopHeight)+1)
  161. queue.add(mockLBResp(t, peerID, stopHeight-1, endTime))
  162. require.Len(t, queue.pending, int(startHeight-stopHeight)+1)
  163. }
  164. // Test a scenario where more blocks are needed then just the stopheight because
  165. // we haven't found a block with a small enough time.
  166. func TestBlockQueueStopTime(t *testing.T) {
  167. peerID, err := types.NewNodeID("0011223344556677889900112233445566778899")
  168. require.NoError(t, err)
  169. queue := newBlockQueue(startHeight, stopHeight, 1, stopTime, 1)
  170. wg := &sync.WaitGroup{}
  171. baseTime := stopTime.Add(-50 * time.Second)
  172. // asynchronously fetch blocks and add it to the queue
  173. for i := 0; i <= numWorkers; i++ {
  174. wg.Add(1)
  175. go func() {
  176. for {
  177. select {
  178. case height := <-queue.nextHeight():
  179. blockTime := baseTime.Add(time.Duration(height) * time.Second)
  180. queue.add(mockLBResp(t, peerID, height, blockTime))
  181. case <-queue.done():
  182. wg.Done()
  183. return
  184. }
  185. }
  186. }()
  187. }
  188. trackingHeight := startHeight
  189. for {
  190. select {
  191. case resp := <-queue.verifyNext():
  192. // assert that the queue serializes the blocks
  193. assert.Equal(t, resp.block.Height, trackingHeight)
  194. trackingHeight--
  195. queue.success(resp.block.Height)
  196. case <-queue.done():
  197. wg.Wait()
  198. assert.Less(t, trackingHeight, stopHeight-50)
  199. return
  200. }
  201. }
  202. }
  203. func TestBlockQueueInitialHeight(t *testing.T) {
  204. peerID, err := types.NewNodeID("0011223344556677889900112233445566778899")
  205. require.NoError(t, err)
  206. const initialHeight int64 = 120
  207. queue := newBlockQueue(startHeight, stopHeight, initialHeight, stopTime, 1)
  208. wg := &sync.WaitGroup{}
  209. // asynchronously fetch blocks and add it to the queue
  210. for i := 0; i <= numWorkers; i++ {
  211. wg.Add(1)
  212. go func() {
  213. for {
  214. select {
  215. case height := <-queue.nextHeight():
  216. require.GreaterOrEqual(t, height, initialHeight)
  217. queue.add(mockLBResp(t, peerID, height, endTime))
  218. case <-queue.done():
  219. wg.Done()
  220. return
  221. }
  222. }
  223. }()
  224. }
  225. loop:
  226. for {
  227. select {
  228. case <-queue.done():
  229. wg.Wait()
  230. require.NoError(t, queue.error())
  231. break loop
  232. case resp := <-queue.verifyNext():
  233. require.GreaterOrEqual(t, resp.block.Height, initialHeight)
  234. queue.success(resp.block.Height)
  235. }
  236. }
  237. }
  238. func mockLBResp(t *testing.T, peer types.NodeID, height int64, time time.Time) lightBlockResponse {
  239. vals, pv := factory.RandValidatorSet(3, 10)
  240. _, _, lb := mockLB(t, height, time, factory.MakeBlockID(), vals, pv)
  241. return lightBlockResponse{
  242. block: lb,
  243. peer: peer,
  244. }
  245. }