package main import ( "context" "fmt" "time" rpctypes "github.com/tendermint/tendermint/rpc/core/types" e2e "github.com/tendermint/tendermint/test/e2e/pkg" ) // Perturbs a running testnet. func Perturb(ctx context.Context, testnet *e2e.Testnet) error { timer := time.NewTimer(0) // first tick fires immediately; reset below defer timer.Stop() for _, node := range testnet.Nodes { for _, perturbation := range node.Perturbations { select { case <-ctx.Done(): return ctx.Err() case <-timer.C: _, err := PerturbNode(ctx, node, perturbation) if err != nil { return err } // give network some time to recover between each timer.Reset(20 * time.Second) } } } return nil } // PerturbNode perturbs a node with a given perturbation, returning its status // after recovering. func PerturbNode(ctx context.Context, node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.ResultStatus, error) { testnet := node.Testnet switch perturbation { case e2e.PerturbationDisconnect: logger.Info(fmt.Sprintf("Disconnecting node %v...", node.Name)) if err := execDocker("network", "disconnect", testnet.Name+"_"+testnet.Name, node.Name); err != nil { return nil, err } time.Sleep(10 * time.Second) if err := execDocker("network", "connect", testnet.Name+"_"+testnet.Name, node.Name); err != nil { return nil, err } case e2e.PerturbationKill: logger.Info(fmt.Sprintf("Killing node %v...", node.Name)) if err := execCompose(testnet.Dir, "kill", "-s", "SIGKILL", node.Name); err != nil { return nil, err } time.Sleep(10 * time.Second) if err := execCompose(testnet.Dir, "start", node.Name); err != nil { return nil, err } case e2e.PerturbationPause: logger.Info(fmt.Sprintf("Pausing node %v...", node.Name)) if err := execCompose(testnet.Dir, "pause", node.Name); err != nil { return nil, err } time.Sleep(10 * time.Second) if err := execCompose(testnet.Dir, "unpause", node.Name); err != nil { return nil, err } case e2e.PerturbationRestart: logger.Info(fmt.Sprintf("Restarting node %v...", node.Name)) if err := execCompose(testnet.Dir, "kill", "-s", "SIGTERM", node.Name); err != nil { return nil, err } time.Sleep(10 * time.Second) if err := execCompose(testnet.Dir, "start", node.Name); err != nil { return nil, err } default: return nil, fmt.Errorf("unexpected perturbation %q", perturbation) } // Seed nodes do not have an RPC endpoint exposed so we cannot assert that // the node recovered. All we can do is hope. if node.Mode == e2e.ModeSeed { return nil, nil } ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) defer cancel() status, err := waitForNode(ctx, node, 0) if err != nil { return nil, err } logger.Info(fmt.Sprintf("Node %v recovered at height %v", node.Name, status.SyncInfo.LatestBlockHeight)) return status, nil }