You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

107 lines
3.0 KiB

  1. package main
  2. import (
  3. "context"
  4. "errors"
  5. "fmt"
  6. "time"
  7. rpchttp "github.com/tendermint/tendermint/rpc/client/http"
  8. rpctypes "github.com/tendermint/tendermint/rpc/core/types"
  9. e2e "github.com/tendermint/tendermint/test/e2e/pkg"
  10. "github.com/tendermint/tendermint/types"
  11. )
  12. // waitForHeight waits for the network to reach a certain height (or above),
  13. // returning the highest height seen. Errors if the network is not making
  14. // progress at all.
  15. func waitForHeight(testnet *e2e.Testnet, height int64) (*types.Block, *types.BlockID, error) {
  16. var (
  17. err error
  18. maxResult *rpctypes.ResultBlock
  19. clients = map[string]*rpchttp.HTTP{}
  20. lastIncrease = time.Now()
  21. )
  22. for {
  23. for _, node := range testnet.Nodes {
  24. if node.Mode == e2e.ModeSeed {
  25. continue
  26. }
  27. client, ok := clients[node.Name]
  28. if !ok {
  29. client, err = node.Client()
  30. if err != nil {
  31. continue
  32. }
  33. clients[node.Name] = client
  34. }
  35. ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
  36. defer cancel()
  37. result, err := client.Block(ctx, nil)
  38. if err != nil {
  39. continue
  40. }
  41. if result.Block != nil && (maxResult == nil || result.Block.Height >= maxResult.Block.Height) {
  42. maxResult = result
  43. lastIncrease = time.Now()
  44. }
  45. if maxResult != nil && maxResult.Block.Height >= height {
  46. return maxResult.Block, &maxResult.BlockID, nil
  47. }
  48. }
  49. if len(clients) == 0 {
  50. return nil, nil, errors.New("unable to connect to any network nodes")
  51. }
  52. if time.Since(lastIncrease) >= 10*time.Second {
  53. if maxResult == nil {
  54. return nil, nil, errors.New("chain stalled at unknown height")
  55. }
  56. return nil, nil, fmt.Errorf("chain stalled at height %v", maxResult.Block.Height)
  57. }
  58. time.Sleep(1 * time.Second)
  59. }
  60. }
  61. // waitForNode waits for a node to become available and catch up to the given block height.
  62. func waitForNode(node *e2e.Node, height int64, timeout time.Duration) (*rpctypes.ResultStatus, error) {
  63. client, err := node.Client()
  64. if err != nil {
  65. return nil, err
  66. }
  67. ctx, cancel := context.WithTimeout(context.Background(), timeout)
  68. defer cancel()
  69. for {
  70. status, err := client.Status(ctx)
  71. switch {
  72. case errors.Is(err, context.DeadlineExceeded):
  73. return nil, fmt.Errorf("timed out waiting for %v to reach height %v", node.Name, height)
  74. case errors.Is(err, context.Canceled):
  75. return nil, err
  76. case err == nil && status.SyncInfo.LatestBlockHeight >= height:
  77. return status, nil
  78. }
  79. time.Sleep(200 * time.Millisecond)
  80. }
  81. }
  82. // waitForAllNodes waits for all nodes to become available and catch up to the given block height.
  83. func waitForAllNodes(testnet *e2e.Testnet, height int64, timeout time.Duration) (int64, error) {
  84. lastHeight := int64(0)
  85. for _, node := range testnet.Nodes {
  86. if node.Mode == e2e.ModeSeed {
  87. continue
  88. }
  89. status, err := waitForNode(node, height, 20*time.Second)
  90. if err != nil {
  91. return 0, err
  92. }
  93. if status.SyncInfo.LatestBlockHeight > lastHeight {
  94. lastHeight = status.SyncInfo.LatestBlockHeight
  95. }
  96. }
  97. return lastHeight, nil
  98. }