From ceea64ec28ce3d572d626e519d6a59e6cc3573b6 Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Mon, 26 Oct 2020 11:09:46 +0100 Subject: [PATCH] test: fix handling of start height in generated E2E testnets (#5563) In #5488 the E2E testnet generator changed to setting explicit `StartAt` heights for initial nodes. This broke the runner, which expected all initial nodes to have `StartAt: 0`, as well as validator set scheduling in the generator. Testnet loading now normalizes initial nodes to have `StartAt: 0`. This also tweaks waiting for misbehavior heights to only use an additional wait if there actually is any misbehavior in the testnet, and to output information when waiting. --- test/e2e/generator/generate.go | 2 +- test/e2e/pkg/testnet.go | 23 ++++++++++++++++++++++- test/e2e/runner/main.go | 11 +++++++---- test/e2e/runner/start.go | 6 ++++++ test/e2e/runner/wait.go | 31 +++++++------------------------ 5 files changed, 43 insertions(+), 30 deletions(-) diff --git a/test/e2e/generator/generate.go b/test/e2e/generator/generate.go index 2ccbd3a6b..a7da44596 100644 --- a/test/e2e/generator/generate.go +++ b/test/e2e/generator/generate.go @@ -104,7 +104,7 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er name := fmt.Sprintf("validator%02d", i) manifest.Nodes[name] = generateNode(r, e2e.ModeValidator, startAt, i <= 2) - if startAt == 0 { + if startAt == manifest.InitialHeight { (*manifest.Validators)[name] = int64(30 + r.Intn(71)) } else { manifest.ValidatorUpdates[fmt.Sprint(startAt+5)] = map[string]int64{ diff --git a/test/e2e/pkg/testnet.go b/test/e2e/pkg/testnet.go index c2f55bc3d..3425d7041 100644 --- a/test/e2e/pkg/testnet.go +++ b/test/e2e/pkg/testnet.go @@ -151,6 +151,9 @@ func LoadTestnet(file string) (*Testnet, error) { Perturbations: []Perturbation{}, Misbehaviors: make(map[int64]string), } + if node.StartAt == testnet.InitialHeight { + node.StartAt = 0 // normalize to 0 for initial nodes, since code expects this + } if nodeManifest.Mode != "" { node.Mode = Mode(nodeManifest.Mode) } @@ -341,7 +344,12 @@ func (n Node) Validate(testnet Testnet) error { for height, misbehavior := range n.Misbehaviors { if height < n.StartAt { - return fmt.Errorf("misbehavior height %d is before start height %d", height, n.StartAt) + return fmt.Errorf("misbehavior height %d is below node start height %d", + height, n.StartAt) + } + if height < testnet.InitialHeight { + return fmt.Errorf("misbehavior height %d is below network initial height %d", + height, testnet.InitialHeight) } exists := false for possibleBehaviors := range mcs.MisbehaviorList { @@ -395,6 +403,19 @@ func (t Testnet) IPv6() bool { return t.IP.IP.To4() == nil } +// LastMisbehaviorHeight returns the height of the last misbehavior. +func (t Testnet) LastMisbehaviorHeight() int64 { + lastHeight := int64(0) + for _, node := range t.Nodes { + for height := range node.Misbehaviors { + if height > lastHeight { + lastHeight = height + } + } + } + return lastHeight +} + // Address returns a P2P endpoint address for the node. func (n Node) AddressP2P(withID bool) string { ip := n.IP.String() diff --git a/test/e2e/runner/main.go b/test/e2e/runner/main.go index bcca5b899..b20454e6b 100644 --- a/test/e2e/runner/main.go +++ b/test/e2e/runner/main.go @@ -69,13 +69,16 @@ func NewCLI() *CLI { if err := Start(cli.testnet); err != nil { return err } - if err := waitForAllMisbehaviors(cli.testnet); err != nil { - return err + if lastMisbehavior := cli.testnet.LastMisbehaviorHeight(); lastMisbehavior > 0 { + // wait for misbehaviors before starting perturbations + if err := WaitUntil(cli.testnet, lastMisbehavior+5); err != nil { + return err + } } if err := Perturb(cli.testnet); err != nil { return err } - if err := Wait(cli.testnet, interphaseWaitPeriod); err != nil { // allow some txs to go through + if err := Wait(cli.testnet, 5); err != nil { // allow some txs to go through return err } @@ -84,7 +87,7 @@ func NewCLI() *CLI { return err } // wait for network to settle before tests - if err := Wait(cli.testnet, interphaseWaitPeriod); err != nil { + if err := Wait(cli.testnet, 5); err != nil { return err } if err := Test(cli.testnet); err != nil { diff --git a/test/e2e/runner/start.go b/test/e2e/runner/start.go index 915222faf..b755f8965 100644 --- a/test/e2e/runner/start.go +++ b/test/e2e/runner/start.go @@ -15,6 +15,12 @@ func Start(testnet *e2e.Testnet) error { sort.SliceStable(nodeQueue, func(i, j int) bool { return nodeQueue[i].StartAt < nodeQueue[j].StartAt }) + if len(nodeQueue) == 0 { + return fmt.Errorf("no nodes in testnet") + } + if nodeQueue[0].StartAt > 0 { + return fmt.Errorf("no initial nodes in testnet") + } // Start initial nodes (StartAt: 0) logger.Info("Starting initial network nodes...") diff --git a/test/e2e/runner/wait.go b/test/e2e/runner/wait.go index c53032c30..8e9030856 100644 --- a/test/e2e/runner/wait.go +++ b/test/e2e/runner/wait.go @@ -7,8 +7,6 @@ import ( e2e "github.com/tendermint/tendermint/test/e2e/pkg" ) -const interphaseWaitPeriod = 5 - // Wait waits for a number of blocks to be produced, and for all nodes to catch // up with it. func Wait(testnet *e2e.Testnet, blocks int64) error { @@ -16,30 +14,15 @@ func Wait(testnet *e2e.Testnet, blocks int64) error { if err != nil { return err } - waitFor := block.Height + blocks - logger.Info(fmt.Sprintf("Waiting for all nodes to reach height %v...", waitFor)) - _, err = waitForAllNodes(testnet, waitFor, 20*time.Second) + return WaitUntil(testnet, block.Height+blocks) +} + +// WaitUntil waits until a given height has been reached. +func WaitUntil(testnet *e2e.Testnet, height int64) error { + logger.Info(fmt.Sprintf("Waiting for all nodes to reach height %v...", height)) + _, err := waitForAllNodes(testnet, height, 20*time.Second) if err != nil { return err } return nil } - -// WaitForAllMisbehaviors calculates the height of the last misbehavior and ensures the entire -// testnet has surpassed this height before moving on to the next phase -func waitForAllMisbehaviors(testnet *e2e.Testnet) error { - _, _, err := waitForHeight(testnet, lastMisbehaviorHeight(testnet)) - return err -} - -func lastMisbehaviorHeight(testnet *e2e.Testnet) int64 { - lastHeight := testnet.InitialHeight - for _, n := range testnet.Nodes { - for height := range n.Misbehaviors { - if height > lastHeight { - lastHeight = height - } - } - } - return lastHeight + interphaseWaitPeriod -}