You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

82 lines
2.4 KiB

  1. package main
  2. import (
  3. "fmt"
  4. "time"
  5. rpctypes "github.com/tendermint/tendermint/rpc/core/types"
  6. e2e "github.com/tendermint/tendermint/test/e2e/pkg"
  7. )
  8. // Perturbs a running testnet.
  9. func Perturb(testnet *e2e.Testnet) error {
  10. for _, node := range testnet.Nodes {
  11. for _, perturbation := range node.Perturbations {
  12. _, err := PerturbNode(node, perturbation)
  13. if err != nil {
  14. return err
  15. }
  16. time.Sleep(20 * time.Second) // give network some time to recover between each
  17. }
  18. }
  19. return nil
  20. }
  21. // PerturbNode perturbs a node with a given perturbation, returning its status
  22. // after recovering.
  23. func PerturbNode(node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.ResultStatus, error) {
  24. testnet := node.Testnet
  25. switch perturbation {
  26. case e2e.PerturbationDisconnect:
  27. logger.Info(fmt.Sprintf("Disconnecting node %v...", node.Name))
  28. if err := execDocker("network", "disconnect", testnet.Name+"_"+testnet.Name, node.Name); err != nil {
  29. return nil, err
  30. }
  31. time.Sleep(10 * time.Second)
  32. if err := execDocker("network", "connect", testnet.Name+"_"+testnet.Name, node.Name); err != nil {
  33. return nil, err
  34. }
  35. case e2e.PerturbationKill:
  36. logger.Info(fmt.Sprintf("Killing node %v...", node.Name))
  37. if err := execCompose(testnet.Dir, "kill", "-s", "SIGKILL", node.Name); err != nil {
  38. return nil, err
  39. }
  40. time.Sleep(10 * time.Second)
  41. if err := execCompose(testnet.Dir, "start", node.Name); err != nil {
  42. return nil, err
  43. }
  44. case e2e.PerturbationPause:
  45. logger.Info(fmt.Sprintf("Pausing node %v...", node.Name))
  46. if err := execCompose(testnet.Dir, "pause", node.Name); err != nil {
  47. return nil, err
  48. }
  49. time.Sleep(10 * time.Second)
  50. if err := execCompose(testnet.Dir, "unpause", node.Name); err != nil {
  51. return nil, err
  52. }
  53. case e2e.PerturbationRestart:
  54. logger.Info(fmt.Sprintf("Restarting node %v...", node.Name))
  55. if err := execCompose(testnet.Dir, "restart", node.Name); err != nil {
  56. return nil, err
  57. }
  58. default:
  59. return nil, fmt.Errorf("unexpected perturbation %q", perturbation)
  60. }
  61. // Seed nodes do not have an RPC endpoint exposed so we cannot assert that
  62. // the node recovered. All we can do is hope.
  63. if node.Mode == e2e.ModeSeed {
  64. return nil, nil
  65. }
  66. status, err := waitForNode(node, 0, 3*time.Minute)
  67. if err != nil {
  68. return nil, err
  69. }
  70. logger.Info(fmt.Sprintf("Node %v recovered at height %v", node.Name, status.SyncInfo.LatestBlockHeight))
  71. return status, nil
  72. }