You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

116 lines
3.0 KiB

  1. package main
  2. import (
  3. "context"
  4. "errors"
  5. "fmt"
  6. "time"
  7. rpchttp "github.com/tendermint/tendermint/rpc/client/http"
  8. rpctypes "github.com/tendermint/tendermint/rpc/core/types"
  9. e2e "github.com/tendermint/tendermint/test/e2e/pkg"
  10. "github.com/tendermint/tendermint/types"
  11. )
  12. // waitForHeight waits for the network to reach a certain height (or above),
  13. // returning the highest height seen. Errors if the network is not making
  14. // progress at all.
  15. func waitForHeight(testnet *e2e.Testnet, height int64) (*types.Block, *types.BlockID, error) {
  16. var (
  17. err error
  18. maxResult *rpctypes.ResultBlock
  19. clients = map[string]*rpchttp.HTTP{}
  20. lastIncrease = time.Now()
  21. )
  22. for {
  23. for _, node := range testnet.Nodes {
  24. if node.Mode == e2e.ModeSeed {
  25. continue
  26. }
  27. client, ok := clients[node.Name]
  28. if !ok {
  29. client, err = node.Client()
  30. if err != nil {
  31. continue
  32. }
  33. clients[node.Name] = client
  34. }
  35. ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
  36. defer cancel()
  37. result, err := client.Block(ctx, nil)
  38. if err != nil {
  39. continue
  40. }
  41. if result.Block != nil && (maxResult == nil || result.Block.Height >= maxResult.Block.Height) {
  42. maxResult = result
  43. lastIncrease = time.Now()
  44. }
  45. if maxResult != nil && maxResult.Block.Height >= height {
  46. return maxResult.Block, &maxResult.BlockID, nil
  47. }
  48. }
  49. if len(clients) == 0 {
  50. return nil, nil, errors.New("unable to connect to any network nodes")
  51. }
  52. if time.Since(lastIncrease) >= 20*time.Second {
  53. if maxResult == nil {
  54. return nil, nil, errors.New("chain stalled at unknown height")
  55. }
  56. return nil, nil, fmt.Errorf("chain stalled at height %v", maxResult.Block.Height)
  57. }
  58. time.Sleep(1 * time.Second)
  59. }
  60. }
  61. // waitForNode waits for a node to become available and catch up to the given block height.
  62. func waitForNode(node *e2e.Node, height int64, timeout time.Duration) (*rpctypes.ResultStatus, error) {
  63. if node.Mode == e2e.ModeSeed {
  64. return nil, nil
  65. }
  66. client, err := node.Client()
  67. if err != nil {
  68. return nil, err
  69. }
  70. ctx, cancel := context.WithTimeout(context.Background(), timeout)
  71. defer cancel()
  72. for {
  73. status, err := client.Status(ctx)
  74. switch {
  75. case errors.Is(err, context.DeadlineExceeded):
  76. return nil, fmt.Errorf("timed out waiting for %v to reach height %v", node.Name, height)
  77. case errors.Is(err, context.Canceled):
  78. return nil, err
  79. case err == nil && status.SyncInfo.LatestBlockHeight >= height:
  80. return status, nil
  81. }
  82. time.Sleep(300 * time.Millisecond)
  83. }
  84. }
  85. // waitForAllNodes waits for all nodes to become available and catch up to the given block height.
  86. func waitForAllNodes(testnet *e2e.Testnet, height int64, timeout time.Duration) (int64, error) {
  87. var lastHeight int64
  88. for _, node := range testnet.Nodes {
  89. if node.Mode == e2e.ModeSeed {
  90. continue
  91. }
  92. status, err := waitForNode(node, height, timeout)
  93. if err != nil {
  94. return 0, err
  95. }
  96. if status.SyncInfo.LatestBlockHeight > lastHeight {
  97. lastHeight = status.SyncInfo.LatestBlockHeight
  98. }
  99. }
  100. return lastHeight, nil
  101. }