You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

236 lines
5.3 KiB

add support for block pruning via ABCI Commit response (#4588) * Added BlockStore.DeleteBlock() * Added initial block pruner prototype * wip * Added BlockStore.PruneBlocks() * Added consensus setting for block pruning * Added BlockStore base * Error on replay if base does not have blocks * Handle missing blocks when sending VoteSetMaj23Message * Error message tweak * Properly update blockstore state * Error message fix again * blockchain: ignore peer missing blocks * Added FIXME * Added test for block replay with truncated history * Handle peer base in blockchain reactor * Improved replay error handling * Added tests for Store.PruneBlocks() * Fix non-RPC handling of truncated block history * Panic on missing block meta in needProofBlock() * Updated changelog * Handle truncated block history in RPC layer * Added info about earliest block in /status RPC * Reorder height and base in blockchain reactor messages * Updated changelog * Fix tests * Appease linter * Minor review fixes * Non-empty BlockStores should always have base > 0 * Update code to assume base > 0 invariant * Added blockstore tests for pruning to 0 * Make sure we don't prune below the current base * Added BlockStore.Size() * config: added retain_blocks recommendations * Update v1 blockchain reactor to handle blockstore base * Added state database pruning * Propagate errors on missing validator sets * Comment tweaks * Improved error message Co-Authored-By: Anton Kaliaev <anton.kalyaev@gmail.com> * use ABCI field ResponseCommit.retain_height instead of retain-blocks config option * remove State.RetainHeight, return value instead * fix minor issues * rename pruneHeights() to pruneBlocks() * noop to fix GitHub borkage Co-authored-by: Anton Kaliaev <anton.kalyaev@gmail.com>
5 years ago
add support for block pruning via ABCI Commit response (#4588) * Added BlockStore.DeleteBlock() * Added initial block pruner prototype * wip * Added BlockStore.PruneBlocks() * Added consensus setting for block pruning * Added BlockStore base * Error on replay if base does not have blocks * Handle missing blocks when sending VoteSetMaj23Message * Error message tweak * Properly update blockstore state * Error message fix again * blockchain: ignore peer missing blocks * Added FIXME * Added test for block replay with truncated history * Handle peer base in blockchain reactor * Improved replay error handling * Added tests for Store.PruneBlocks() * Fix non-RPC handling of truncated block history * Panic on missing block meta in needProofBlock() * Updated changelog * Handle truncated block history in RPC layer * Added info about earliest block in /status RPC * Reorder height and base in blockchain reactor messages * Updated changelog * Fix tests * Appease linter * Minor review fixes * Non-empty BlockStores should always have base > 0 * Update code to assume base > 0 invariant * Added blockstore tests for pruning to 0 * Make sure we don't prune below the current base * Added BlockStore.Size() * config: added retain_blocks recommendations * Update v1 blockchain reactor to handle blockstore base * Added state database pruning * Propagate errors on missing validator sets * Comment tweaks * Improved error message Co-Authored-By: Anton Kaliaev <anton.kalyaev@gmail.com> * use ABCI field ResponseCommit.retain_height instead of retain-blocks config option * remove State.RetainHeight, return value instead * fix minor issues * rename pruneHeights() to pruneBlocks() * noop to fix GitHub borkage Co-authored-by: Anton Kaliaev <anton.kalyaev@gmail.com>
5 years ago
add support for block pruning via ABCI Commit response (#4588) * Added BlockStore.DeleteBlock() * Added initial block pruner prototype * wip * Added BlockStore.PruneBlocks() * Added consensus setting for block pruning * Added BlockStore base * Error on replay if base does not have blocks * Handle missing blocks when sending VoteSetMaj23Message * Error message tweak * Properly update blockstore state * Error message fix again * blockchain: ignore peer missing blocks * Added FIXME * Added test for block replay with truncated history * Handle peer base in blockchain reactor * Improved replay error handling * Added tests for Store.PruneBlocks() * Fix non-RPC handling of truncated block history * Panic on missing block meta in needProofBlock() * Updated changelog * Handle truncated block history in RPC layer * Added info about earliest block in /status RPC * Reorder height and base in blockchain reactor messages * Updated changelog * Fix tests * Appease linter * Minor review fixes * Non-empty BlockStores should always have base > 0 * Update code to assume base > 0 invariant * Added blockstore tests for pruning to 0 * Make sure we don't prune below the current base * Added BlockStore.Size() * config: added retain_blocks recommendations * Update v1 blockchain reactor to handle blockstore base * Added state database pruning * Propagate errors on missing validator sets * Comment tweaks * Improved error message Co-Authored-By: Anton Kaliaev <anton.kalyaev@gmail.com> * use ABCI field ResponseCommit.retain_height instead of retain-blocks config option * remove State.RetainHeight, return value instead * fix minor issues * rename pruneHeights() to pruneBlocks() * noop to fix GitHub borkage Co-authored-by: Anton Kaliaev <anton.kalyaev@gmail.com>
5 years ago
8 years ago
10 years ago
10 years ago
10 years ago
add support for block pruning via ABCI Commit response (#4588) * Added BlockStore.DeleteBlock() * Added initial block pruner prototype * wip * Added BlockStore.PruneBlocks() * Added consensus setting for block pruning * Added BlockStore base * Error on replay if base does not have blocks * Handle missing blocks when sending VoteSetMaj23Message * Error message tweak * Properly update blockstore state * Error message fix again * blockchain: ignore peer missing blocks * Added FIXME * Added test for block replay with truncated history * Handle peer base in blockchain reactor * Improved replay error handling * Added tests for Store.PruneBlocks() * Fix non-RPC handling of truncated block history * Panic on missing block meta in needProofBlock() * Updated changelog * Handle truncated block history in RPC layer * Added info about earliest block in /status RPC * Reorder height and base in blockchain reactor messages * Updated changelog * Fix tests * Appease linter * Minor review fixes * Non-empty BlockStores should always have base > 0 * Update code to assume base > 0 invariant * Added blockstore tests for pruning to 0 * Make sure we don't prune below the current base * Added BlockStore.Size() * config: added retain_blocks recommendations * Update v1 blockchain reactor to handle blockstore base * Added state database pruning * Propagate errors on missing validator sets * Comment tweaks * Improved error message Co-Authored-By: Anton Kaliaev <anton.kalyaev@gmail.com> * use ABCI field ResponseCommit.retain_height instead of retain-blocks config option * remove State.RetainHeight, return value instead * fix minor issues * rename pruneHeights() to pruneBlocks() * noop to fix GitHub borkage Co-authored-by: Anton Kaliaev <anton.kalyaev@gmail.com>
5 years ago
10 years ago
10 years ago
10 years ago
10 years ago
add support for block pruning via ABCI Commit response (#4588) * Added BlockStore.DeleteBlock() * Added initial block pruner prototype * wip * Added BlockStore.PruneBlocks() * Added consensus setting for block pruning * Added BlockStore base * Error on replay if base does not have blocks * Handle missing blocks when sending VoteSetMaj23Message * Error message tweak * Properly update blockstore state * Error message fix again * blockchain: ignore peer missing blocks * Added FIXME * Added test for block replay with truncated history * Handle peer base in blockchain reactor * Improved replay error handling * Added tests for Store.PruneBlocks() * Fix non-RPC handling of truncated block history * Panic on missing block meta in needProofBlock() * Updated changelog * Handle truncated block history in RPC layer * Added info about earliest block in /status RPC * Reorder height and base in blockchain reactor messages * Updated changelog * Fix tests * Appease linter * Minor review fixes * Non-empty BlockStores should always have base > 0 * Update code to assume base > 0 invariant * Added blockstore tests for pruning to 0 * Make sure we don't prune below the current base * Added BlockStore.Size() * config: added retain_blocks recommendations * Update v1 blockchain reactor to handle blockstore base * Added state database pruning * Propagate errors on missing validator sets * Comment tweaks * Improved error message Co-Authored-By: Anton Kaliaev <anton.kalyaev@gmail.com> * use ABCI field ResponseCommit.retain_height instead of retain-blocks config option * remove State.RetainHeight, return value instead * fix minor issues * rename pruneHeights() to pruneBlocks() * noop to fix GitHub borkage Co-authored-by: Anton Kaliaev <anton.kalyaev@gmail.com>
5 years ago
add support for block pruning via ABCI Commit response (#4588) * Added BlockStore.DeleteBlock() * Added initial block pruner prototype * wip * Added BlockStore.PruneBlocks() * Added consensus setting for block pruning * Added BlockStore base * Error on replay if base does not have blocks * Handle missing blocks when sending VoteSetMaj23Message * Error message tweak * Properly update blockstore state * Error message fix again * blockchain: ignore peer missing blocks * Added FIXME * Added test for block replay with truncated history * Handle peer base in blockchain reactor * Improved replay error handling * Added tests for Store.PruneBlocks() * Fix non-RPC handling of truncated block history * Panic on missing block meta in needProofBlock() * Updated changelog * Handle truncated block history in RPC layer * Added info about earliest block in /status RPC * Reorder height and base in blockchain reactor messages * Updated changelog * Fix tests * Appease linter * Minor review fixes * Non-empty BlockStores should always have base > 0 * Update code to assume base > 0 invariant * Added blockstore tests for pruning to 0 * Make sure we don't prune below the current base * Added BlockStore.Size() * config: added retain_blocks recommendations * Update v1 blockchain reactor to handle blockstore base * Added state database pruning * Propagate errors on missing validator sets * Comment tweaks * Improved error message Co-Authored-By: Anton Kaliaev <anton.kalyaev@gmail.com> * use ABCI field ResponseCommit.retain_height instead of retain-blocks config option * remove State.RetainHeight, return value instead * fix minor issues * rename pruneHeights() to pruneBlocks() * noop to fix GitHub borkage Co-authored-by: Anton Kaliaev <anton.kalyaev@gmail.com>
5 years ago
  1. package blocksync
  2. import (
  3. "context"
  4. "fmt"
  5. mrand "math/rand"
  6. "testing"
  7. "time"
  8. "github.com/stretchr/testify/assert"
  9. "github.com/stretchr/testify/require"
  10. "github.com/tendermint/tendermint/libs/log"
  11. tmrand "github.com/tendermint/tendermint/libs/rand"
  12. "github.com/tendermint/tendermint/types"
  13. )
  14. func init() {
  15. peerTimeout = 2 * time.Second
  16. }
  17. type testPeer struct {
  18. id types.NodeID
  19. base int64
  20. height int64
  21. inputChan chan inputData // make sure each peer's data is sequential
  22. }
  23. type inputData struct {
  24. t *testing.T
  25. pool *BlockPool
  26. request BlockRequest
  27. }
  28. func (p testPeer) runInputRoutine() {
  29. go func() {
  30. for input := range p.inputChan {
  31. p.simulateInput(input)
  32. }
  33. }()
  34. }
  35. // Request desired, pretend like we got the block immediately.
  36. func (p testPeer) simulateInput(input inputData) {
  37. block := &types.Block{Header: types.Header{Height: input.request.Height}}
  38. input.pool.AddBlock(input.request.PeerID, block, 123)
  39. // TODO: uncommenting this creates a race which is detected by:
  40. // https://github.com/golang/go/blob/2bd767b1022dd3254bcec469f0ee164024726486/src/testing/testing.go#L854-L856
  41. // see: https://github.com/tendermint/tendermint/issues/3390#issue-418379890
  42. // input.t.Logf("Added block from peer %v (height: %v)", input.request.PeerID, input.request.Height)
  43. }
  44. type testPeers map[types.NodeID]testPeer
  45. func (ps testPeers) start() {
  46. for _, v := range ps {
  47. v.runInputRoutine()
  48. }
  49. }
  50. func (ps testPeers) stop() {
  51. for _, v := range ps {
  52. close(v.inputChan)
  53. }
  54. }
  55. func makePeers(numPeers int, minHeight, maxHeight int64) testPeers {
  56. peers := make(testPeers, numPeers)
  57. for i := 0; i < numPeers; i++ {
  58. peerID := types.NodeID(tmrand.Str(12))
  59. height := minHeight + mrand.Int63n(maxHeight-minHeight)
  60. base := minHeight + int64(i)
  61. if base > height {
  62. base = height
  63. }
  64. peers[peerID] = testPeer{peerID, base, height, make(chan inputData, 10)}
  65. }
  66. return peers
  67. }
  68. func TestBlockPoolBasic(t *testing.T) {
  69. ctx, cancel := context.WithCancel(context.Background())
  70. defer cancel()
  71. start := int64(42)
  72. peers := makePeers(10, start+1, 1000)
  73. errorsCh := make(chan peerError, 1000)
  74. requestsCh := make(chan BlockRequest, 1000)
  75. pool := NewBlockPool(log.TestingLogger(), start, requestsCh, errorsCh)
  76. if err := pool.Start(ctx); err != nil {
  77. t.Error(err)
  78. }
  79. t.Cleanup(func() { cancel(); pool.Wait() })
  80. peers.start()
  81. defer peers.stop()
  82. // Introduce each peer.
  83. go func() {
  84. for _, peer := range peers {
  85. pool.SetPeerRange(peer.id, peer.base, peer.height)
  86. }
  87. }()
  88. // Start a goroutine to pull blocks
  89. go func() {
  90. for {
  91. if !pool.IsRunning() {
  92. return
  93. }
  94. first, second := pool.PeekTwoBlocks()
  95. if first != nil && second != nil {
  96. pool.PopRequest()
  97. } else {
  98. time.Sleep(1 * time.Second)
  99. }
  100. }
  101. }()
  102. // Pull from channels
  103. for {
  104. select {
  105. case err := <-errorsCh:
  106. t.Error(err)
  107. case request := <-requestsCh:
  108. t.Logf("Pulled new BlockRequest %v", request)
  109. if request.Height == 300 {
  110. return // Done!
  111. }
  112. peers[request.PeerID].inputChan <- inputData{t, pool, request}
  113. }
  114. }
  115. }
  116. func TestBlockPoolTimeout(t *testing.T) {
  117. ctx, cancel := context.WithCancel(context.Background())
  118. defer cancel()
  119. start := int64(42)
  120. peers := makePeers(10, start+1, 1000)
  121. errorsCh := make(chan peerError, 1000)
  122. requestsCh := make(chan BlockRequest, 1000)
  123. pool := NewBlockPool(log.TestingLogger(), start, requestsCh, errorsCh)
  124. err := pool.Start(ctx)
  125. if err != nil {
  126. t.Error(err)
  127. }
  128. t.Cleanup(func() { cancel(); pool.Wait() })
  129. for _, peer := range peers {
  130. t.Logf("Peer %v", peer.id)
  131. }
  132. // Introduce each peer.
  133. go func() {
  134. for _, peer := range peers {
  135. pool.SetPeerRange(peer.id, peer.base, peer.height)
  136. }
  137. }()
  138. // Start a goroutine to pull blocks
  139. go func() {
  140. for {
  141. if !pool.IsRunning() {
  142. return
  143. }
  144. first, second := pool.PeekTwoBlocks()
  145. if first != nil && second != nil {
  146. pool.PopRequest()
  147. } else {
  148. time.Sleep(1 * time.Second)
  149. }
  150. }
  151. }()
  152. // Pull from channels
  153. counter := 0
  154. timedOut := map[types.NodeID]struct{}{}
  155. for {
  156. select {
  157. case err := <-errorsCh:
  158. t.Log(err)
  159. // consider error to be always timeout here
  160. if _, ok := timedOut[err.peerID]; !ok {
  161. counter++
  162. if counter == len(peers) {
  163. return // Done!
  164. }
  165. }
  166. case request := <-requestsCh:
  167. t.Logf("Pulled new BlockRequest %+v", request)
  168. }
  169. }
  170. }
  171. func TestBlockPoolRemovePeer(t *testing.T) {
  172. ctx, cancel := context.WithCancel(context.Background())
  173. defer cancel()
  174. peers := make(testPeers, 10)
  175. for i := 0; i < 10; i++ {
  176. peerID := types.NodeID(fmt.Sprintf("%d", i+1))
  177. height := int64(i + 1)
  178. peers[peerID] = testPeer{peerID, 0, height, make(chan inputData)}
  179. }
  180. requestsCh := make(chan BlockRequest)
  181. errorsCh := make(chan peerError)
  182. pool := NewBlockPool(log.TestingLogger(), 1, requestsCh, errorsCh)
  183. err := pool.Start(ctx)
  184. require.NoError(t, err)
  185. t.Cleanup(func() { cancel(); pool.Wait() })
  186. // add peers
  187. for peerID, peer := range peers {
  188. pool.SetPeerRange(peerID, peer.base, peer.height)
  189. }
  190. assert.EqualValues(t, 10, pool.MaxPeerHeight())
  191. // remove not-existing peer
  192. assert.NotPanics(t, func() { pool.RemovePeer(types.NodeID("Superman")) })
  193. // remove peer with biggest height
  194. pool.RemovePeer(types.NodeID("10"))
  195. assert.EqualValues(t, 9, pool.MaxPeerHeight())
  196. // remove all peers
  197. for peerID := range peers {
  198. pool.RemovePeer(peerID)
  199. }
  200. assert.EqualValues(t, 0, pool.MaxPeerHeight())
  201. }