You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

234 lines
5.3 KiB

add support for block pruning via ABCI Commit response (#4588) * Added BlockStore.DeleteBlock() * Added initial block pruner prototype * wip * Added BlockStore.PruneBlocks() * Added consensus setting for block pruning * Added BlockStore base * Error on replay if base does not have blocks * Handle missing blocks when sending VoteSetMaj23Message * Error message tweak * Properly update blockstore state * Error message fix again * blockchain: ignore peer missing blocks * Added FIXME * Added test for block replay with truncated history * Handle peer base in blockchain reactor * Improved replay error handling * Added tests for Store.PruneBlocks() * Fix non-RPC handling of truncated block history * Panic on missing block meta in needProofBlock() * Updated changelog * Handle truncated block history in RPC layer * Added info about earliest block in /status RPC * Reorder height and base in blockchain reactor messages * Updated changelog * Fix tests * Appease linter * Minor review fixes * Non-empty BlockStores should always have base > 0 * Update code to assume base > 0 invariant * Added blockstore tests for pruning to 0 * Make sure we don't prune below the current base * Added BlockStore.Size() * config: added retain_blocks recommendations * Update v1 blockchain reactor to handle blockstore base * Added state database pruning * Propagate errors on missing validator sets * Comment tweaks * Improved error message Co-Authored-By: Anton Kaliaev <anton.kalyaev@gmail.com> * use ABCI field ResponseCommit.retain_height instead of retain-blocks config option * remove State.RetainHeight, return value instead * fix minor issues * rename pruneHeights() to pruneBlocks() * noop to fix GitHub borkage Co-authored-by: Anton Kaliaev <anton.kalyaev@gmail.com>
5 years ago
add support for block pruning via ABCI Commit response (#4588) * Added BlockStore.DeleteBlock() * Added initial block pruner prototype * wip * Added BlockStore.PruneBlocks() * Added consensus setting for block pruning * Added BlockStore base * Error on replay if base does not have blocks * Handle missing blocks when sending VoteSetMaj23Message * Error message tweak * Properly update blockstore state * Error message fix again * blockchain: ignore peer missing blocks * Added FIXME * Added test for block replay with truncated history * Handle peer base in blockchain reactor * Improved replay error handling * Added tests for Store.PruneBlocks() * Fix non-RPC handling of truncated block history * Panic on missing block meta in needProofBlock() * Updated changelog * Handle truncated block history in RPC layer * Added info about earliest block in /status RPC * Reorder height and base in blockchain reactor messages * Updated changelog * Fix tests * Appease linter * Minor review fixes * Non-empty BlockStores should always have base > 0 * Update code to assume base > 0 invariant * Added blockstore tests for pruning to 0 * Make sure we don't prune below the current base * Added BlockStore.Size() * config: added retain_blocks recommendations * Update v1 blockchain reactor to handle blockstore base * Added state database pruning * Propagate errors on missing validator sets * Comment tweaks * Improved error message Co-Authored-By: Anton Kaliaev <anton.kalyaev@gmail.com> * use ABCI field ResponseCommit.retain_height instead of retain-blocks config option * remove State.RetainHeight, return value instead * fix minor issues * rename pruneHeights() to pruneBlocks() * noop to fix GitHub borkage Co-authored-by: Anton Kaliaev <anton.kalyaev@gmail.com>
5 years ago
add support for block pruning via ABCI Commit response (#4588) * Added BlockStore.DeleteBlock() * Added initial block pruner prototype * wip * Added BlockStore.PruneBlocks() * Added consensus setting for block pruning * Added BlockStore base * Error on replay if base does not have blocks * Handle missing blocks when sending VoteSetMaj23Message * Error message tweak * Properly update blockstore state * Error message fix again * blockchain: ignore peer missing blocks * Added FIXME * Added test for block replay with truncated history * Handle peer base in blockchain reactor * Improved replay error handling * Added tests for Store.PruneBlocks() * Fix non-RPC handling of truncated block history * Panic on missing block meta in needProofBlock() * Updated changelog * Handle truncated block history in RPC layer * Added info about earliest block in /status RPC * Reorder height and base in blockchain reactor messages * Updated changelog * Fix tests * Appease linter * Minor review fixes * Non-empty BlockStores should always have base > 0 * Update code to assume base > 0 invariant * Added blockstore tests for pruning to 0 * Make sure we don't prune below the current base * Added BlockStore.Size() * config: added retain_blocks recommendations * Update v1 blockchain reactor to handle blockstore base * Added state database pruning * Propagate errors on missing validator sets * Comment tweaks * Improved error message Co-Authored-By: Anton Kaliaev <anton.kalyaev@gmail.com> * use ABCI field ResponseCommit.retain_height instead of retain-blocks config option * remove State.RetainHeight, return value instead * fix minor issues * rename pruneHeights() to pruneBlocks() * noop to fix GitHub borkage Co-authored-by: Anton Kaliaev <anton.kalyaev@gmail.com>
5 years ago
10 years ago
10 years ago
add support for block pruning via ABCI Commit response (#4588) * Added BlockStore.DeleteBlock() * Added initial block pruner prototype * wip * Added BlockStore.PruneBlocks() * Added consensus setting for block pruning * Added BlockStore base * Error on replay if base does not have blocks * Handle missing blocks when sending VoteSetMaj23Message * Error message tweak * Properly update blockstore state * Error message fix again * blockchain: ignore peer missing blocks * Added FIXME * Added test for block replay with truncated history * Handle peer base in blockchain reactor * Improved replay error handling * Added tests for Store.PruneBlocks() * Fix non-RPC handling of truncated block history * Panic on missing block meta in needProofBlock() * Updated changelog * Handle truncated block history in RPC layer * Added info about earliest block in /status RPC * Reorder height and base in blockchain reactor messages * Updated changelog * Fix tests * Appease linter * Minor review fixes * Non-empty BlockStores should always have base > 0 * Update code to assume base > 0 invariant * Added blockstore tests for pruning to 0 * Make sure we don't prune below the current base * Added BlockStore.Size() * config: added retain_blocks recommendations * Update v1 blockchain reactor to handle blockstore base * Added state database pruning * Propagate errors on missing validator sets * Comment tweaks * Improved error message Co-Authored-By: Anton Kaliaev <anton.kalyaev@gmail.com> * use ABCI field ResponseCommit.retain_height instead of retain-blocks config option * remove State.RetainHeight, return value instead * fix minor issues * rename pruneHeights() to pruneBlocks() * noop to fix GitHub borkage Co-authored-by: Anton Kaliaev <anton.kalyaev@gmail.com>
5 years ago
10 years ago
10 years ago
10 years ago
10 years ago
add support for block pruning via ABCI Commit response (#4588) * Added BlockStore.DeleteBlock() * Added initial block pruner prototype * wip * Added BlockStore.PruneBlocks() * Added consensus setting for block pruning * Added BlockStore base * Error on replay if base does not have blocks * Handle missing blocks when sending VoteSetMaj23Message * Error message tweak * Properly update blockstore state * Error message fix again * blockchain: ignore peer missing blocks * Added FIXME * Added test for block replay with truncated history * Handle peer base in blockchain reactor * Improved replay error handling * Added tests for Store.PruneBlocks() * Fix non-RPC handling of truncated block history * Panic on missing block meta in needProofBlock() * Updated changelog * Handle truncated block history in RPC layer * Added info about earliest block in /status RPC * Reorder height and base in blockchain reactor messages * Updated changelog * Fix tests * Appease linter * Minor review fixes * Non-empty BlockStores should always have base > 0 * Update code to assume base > 0 invariant * Added blockstore tests for pruning to 0 * Make sure we don't prune below the current base * Added BlockStore.Size() * config: added retain_blocks recommendations * Update v1 blockchain reactor to handle blockstore base * Added state database pruning * Propagate errors on missing validator sets * Comment tweaks * Improved error message Co-Authored-By: Anton Kaliaev <anton.kalyaev@gmail.com> * use ABCI field ResponseCommit.retain_height instead of retain-blocks config option * remove State.RetainHeight, return value instead * fix minor issues * rename pruneHeights() to pruneBlocks() * noop to fix GitHub borkage Co-authored-by: Anton Kaliaev <anton.kalyaev@gmail.com>
5 years ago
add support for block pruning via ABCI Commit response (#4588) * Added BlockStore.DeleteBlock() * Added initial block pruner prototype * wip * Added BlockStore.PruneBlocks() * Added consensus setting for block pruning * Added BlockStore base * Error on replay if base does not have blocks * Handle missing blocks when sending VoteSetMaj23Message * Error message tweak * Properly update blockstore state * Error message fix again * blockchain: ignore peer missing blocks * Added FIXME * Added test for block replay with truncated history * Handle peer base in blockchain reactor * Improved replay error handling * Added tests for Store.PruneBlocks() * Fix non-RPC handling of truncated block history * Panic on missing block meta in needProofBlock() * Updated changelog * Handle truncated block history in RPC layer * Added info about earliest block in /status RPC * Reorder height and base in blockchain reactor messages * Updated changelog * Fix tests * Appease linter * Minor review fixes * Non-empty BlockStores should always have base > 0 * Update code to assume base > 0 invariant * Added blockstore tests for pruning to 0 * Make sure we don't prune below the current base * Added BlockStore.Size() * config: added retain_blocks recommendations * Update v1 blockchain reactor to handle blockstore base * Added state database pruning * Propagate errors on missing validator sets * Comment tweaks * Improved error message Co-Authored-By: Anton Kaliaev <anton.kalyaev@gmail.com> * use ABCI field ResponseCommit.retain_height instead of retain-blocks config option * remove State.RetainHeight, return value instead * fix minor issues * rename pruneHeights() to pruneBlocks() * noop to fix GitHub borkage Co-authored-by: Anton Kaliaev <anton.kalyaev@gmail.com>
5 years ago
  1. package blocksync
  2. import (
  3. "context"
  4. "fmt"
  5. mrand "math/rand"
  6. "testing"
  7. "time"
  8. "github.com/stretchr/testify/assert"
  9. "github.com/stretchr/testify/require"
  10. "github.com/tendermint/tendermint/libs/log"
  11. tmrand "github.com/tendermint/tendermint/libs/rand"
  12. "github.com/tendermint/tendermint/types"
  13. )
  14. func init() {
  15. peerTimeout = 2 * time.Second
  16. }
  17. type testPeer struct {
  18. id types.NodeID
  19. base int64
  20. height int64
  21. inputChan chan inputData // make sure each peer's data is sequential
  22. }
  23. type inputData struct {
  24. t *testing.T
  25. pool *BlockPool
  26. request BlockRequest
  27. }
  28. func (p testPeer) runInputRoutine() {
  29. go func() {
  30. for input := range p.inputChan {
  31. p.simulateInput(input)
  32. }
  33. }()
  34. }
  35. // Request desired, pretend like we got the block immediately.
  36. func (p testPeer) simulateInput(input inputData) {
  37. block := &types.Block{Header: types.Header{Height: input.request.Height}}
  38. input.pool.AddBlock(input.request.PeerID, block, 123)
  39. // TODO: uncommenting this creates a race which is detected by:
  40. // https://github.com/golang/go/blob/2bd767b1022dd3254bcec469f0ee164024726486/src/testing/testing.go#L854-L856
  41. // see: https://github.com/tendermint/tendermint/issues/3390#issue-418379890
  42. // input.t.Logf("Added block from peer %v (height: %v)", input.request.PeerID, input.request.Height)
  43. }
  44. type testPeers map[types.NodeID]testPeer
  45. func (ps testPeers) start() {
  46. for _, v := range ps {
  47. v.runInputRoutine()
  48. }
  49. }
  50. func (ps testPeers) stop() {
  51. for _, v := range ps {
  52. close(v.inputChan)
  53. }
  54. }
  55. func makePeers(numPeers int, minHeight, maxHeight int64) testPeers {
  56. peers := make(testPeers, numPeers)
  57. for i := 0; i < numPeers; i++ {
  58. peerID := types.NodeID(tmrand.Str(12))
  59. height := minHeight + mrand.Int63n(maxHeight-minHeight)
  60. base := minHeight + int64(i)
  61. if base > height {
  62. base = height
  63. }
  64. peers[peerID] = testPeer{peerID, base, height, make(chan inputData, 10)}
  65. }
  66. return peers
  67. }
  68. func TestBlockPoolBasic(t *testing.T) {
  69. ctx, cancel := context.WithCancel(context.Background())
  70. defer cancel()
  71. start := int64(42)
  72. peers := makePeers(10, start+1, 1000)
  73. errorsCh := make(chan peerError, 1000)
  74. requestsCh := make(chan BlockRequest, 1000)
  75. pool := NewBlockPool(log.TestingLogger(), start, requestsCh, errorsCh)
  76. if err := pool.Start(ctx); err != nil {
  77. t.Error(err)
  78. }
  79. t.Cleanup(func() { cancel(); pool.Wait() })
  80. peers.start()
  81. defer peers.stop()
  82. // Introduce each peer.
  83. go func() {
  84. for _, peer := range peers {
  85. pool.SetPeerRange(peer.id, peer.base, peer.height)
  86. }
  87. }()
  88. // Start a goroutine to pull blocks
  89. go func() {
  90. for {
  91. if !pool.IsRunning() {
  92. return
  93. }
  94. first, second := pool.PeekTwoBlocks()
  95. if first != nil && second != nil {
  96. pool.PopRequest()
  97. } else {
  98. time.Sleep(1 * time.Second)
  99. }
  100. }
  101. }()
  102. // Pull from channels
  103. for {
  104. select {
  105. case err := <-errorsCh:
  106. t.Error(err)
  107. case request := <-requestsCh:
  108. if request.Height == 300 {
  109. return // Done!
  110. }
  111. peers[request.PeerID].inputChan <- inputData{t, pool, request}
  112. }
  113. }
  114. }
  115. func TestBlockPoolTimeout(t *testing.T) {
  116. ctx, cancel := context.WithCancel(context.Background())
  117. defer cancel()
  118. logger := log.TestingLogger()
  119. start := int64(42)
  120. peers := makePeers(10, start+1, 1000)
  121. errorsCh := make(chan peerError, 1000)
  122. requestsCh := make(chan BlockRequest, 1000)
  123. pool := NewBlockPool(logger, start, requestsCh, errorsCh)
  124. err := pool.Start(ctx)
  125. if err != nil {
  126. t.Error(err)
  127. }
  128. t.Cleanup(func() { cancel(); pool.Wait() })
  129. // Introduce each peer.
  130. go func() {
  131. for _, peer := range peers {
  132. pool.SetPeerRange(peer.id, peer.base, peer.height)
  133. }
  134. }()
  135. // Start a goroutine to pull blocks
  136. go func() {
  137. for {
  138. if !pool.IsRunning() {
  139. return
  140. }
  141. first, second := pool.PeekTwoBlocks()
  142. if first != nil && second != nil {
  143. pool.PopRequest()
  144. } else {
  145. time.Sleep(1 * time.Second)
  146. }
  147. }
  148. }()
  149. // Pull from channels
  150. counter := 0
  151. timedOut := map[types.NodeID]struct{}{}
  152. for {
  153. select {
  154. case err := <-errorsCh:
  155. // consider error to be always timeout here
  156. if _, ok := timedOut[err.peerID]; !ok {
  157. counter++
  158. if counter == len(peers) {
  159. return // Done!
  160. }
  161. }
  162. case request := <-requestsCh:
  163. logger.Debug("received request",
  164. "counter", counter,
  165. "request", request)
  166. }
  167. }
  168. }
  169. func TestBlockPoolRemovePeer(t *testing.T) {
  170. ctx, cancel := context.WithCancel(context.Background())
  171. defer cancel()
  172. peers := make(testPeers, 10)
  173. for i := 0; i < 10; i++ {
  174. peerID := types.NodeID(fmt.Sprintf("%d", i+1))
  175. height := int64(i + 1)
  176. peers[peerID] = testPeer{peerID, 0, height, make(chan inputData)}
  177. }
  178. requestsCh := make(chan BlockRequest)
  179. errorsCh := make(chan peerError)
  180. pool := NewBlockPool(log.TestingLogger(), 1, requestsCh, errorsCh)
  181. err := pool.Start(ctx)
  182. require.NoError(t, err)
  183. t.Cleanup(func() { cancel(); pool.Wait() })
  184. // add peers
  185. for peerID, peer := range peers {
  186. pool.SetPeerRange(peerID, peer.base, peer.height)
  187. }
  188. assert.EqualValues(t, 10, pool.MaxPeerHeight())
  189. // remove not-existing peer
  190. assert.NotPanics(t, func() { pool.RemovePeer(types.NodeID("Superman")) })
  191. // remove peer with biggest height
  192. pool.RemovePeer(types.NodeID("10"))
  193. assert.EqualValues(t, 9, pool.MaxPeerHeight())
  194. // remove all peers
  195. for peerID := range peers {
  196. pool.RemovePeer(peerID)
  197. }
  198. assert.EqualValues(t, 0, pool.MaxPeerHeight())
  199. }