e2e: longer test harness timeouts (#6728)

.github/workflows/e2e-nightly-master.yml

@@ -35,7 +35,7 @@ jobs:
       - name: Generate testnets
         working-directory: test/e2e
         # When changing -g, also change the matrix groups above
-        run: ./build/generator -g 2 -d networks/nightly/${{ matrix.p2p }} -p ${{ matrix.p2p }} 
+        run: ./build/generator -g 2 -d networks/nightly/${{ matrix.p2p }} -p ${{ matrix.p2p }}
 
       - name: Run ${{ matrix.p2p }} p2p testnets in group ${{ matrix.group }}
         working-directory: test/e2e

test/e2e/runner/evidence.go

@@ -91,6 +91,13 @@ func InjectEvidence(testnet *e2e.Testnet, amount int) error {
 		}
 	}
 
+	// wait for the node to reach the height above the forged height so that
+	// it is able to validate the evidence
+	_, err = waitForNode(targetNode, blockRes.Block.Height+2, 10*time.Second)
+	if err != nil {
+		return err
+	}
+
 	logger.Info(fmt.Sprintf("Finished sending evidence (height %d)", blockRes.Block.Height+2))
 
 	return nil
@@ -186,7 +193,12 @@ func generateDuplicateVoteEvidence(
 	if err != nil {
 		return nil, err
 	}
-	return types.NewDuplicateVoteEvidence(voteA, voteB, time, vals), nil
+	ev := types.NewDuplicateVoteEvidence(voteA, voteB, time, vals)
+	if ev == nil {
+		return nil, fmt.Errorf("could not generate evidence a=%v b=%v vals=%v", voteA, voteB, vals)
+	}
+
+	return ev, nil
 }
 
 func readPrivKey(keyFilePath string) (crypto.PrivKey, error) {

test/e2e/runner/main.go

@@ -85,7 +85,7 @@ func NewCLI() *CLI {
 				if err := InjectEvidence(cli.testnet, cli.testnet.Evidence); err != nil {
 					return err
 				}
-				if err := Wait(cli.testnet, 1); err != nil { // ensure chain progress
+				if err := Wait(cli.testnet, 5); err != nil { // ensure chain progress
 					return err
 				}
 			}

test/e2e/runner/perturb.go

@@ -16,7 +16,7 @@ func Perturb(testnet *e2e.Testnet) error {
 			if err != nil {
 				return err
 			}
-			time.Sleep(15 * time.Second) // give network some time to recover between each
+			time.Sleep(20 * time.Second) // give network some time to recover between each
 		}
 	}
 	return nil
@@ -42,7 +42,7 @@ func PerturbNode(node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.Resul
 		if err := execCompose(testnet.Dir, "kill", "-s", "SIGKILL", node.Name); err != nil {
 			return nil, err
 		}
-		time.Sleep(2 * time.Second)
+		time.Sleep(10 * time.Second)
 		if err := execCompose(testnet.Dir, "start", node.Name); err != nil {
 			return nil, err
 		}
@@ -73,7 +73,7 @@ func PerturbNode(node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.Resul
 		return nil, nil
 	}
 
-	status, err := waitForNode(node, 0, 2*time.Minute)
+	status, err := waitForNode(node, 0, 3*time.Minute)
 	if err != nil {
 		return nil, err
 	}

test/e2e/runner/start.go

@@ -75,7 +75,7 @@ func Start(testnet *e2e.Testnet) error {
 		if err := execCompose(testnet.Dir, "up", "-d", node.Name); err != nil {
 			return err
 		}
-		status, err := waitForNode(node, node.StartAt, 5*time.Minute)
+		status, err := waitForNode(node, node.StartAt, 8*time.Minute)
 		if err != nil {
 			return err
 		}

test/e2e/runner/wait.go

@@ -30,5 +30,5 @@ func WaitUntil(testnet *e2e.Testnet, height int64) error {
 // waitingTime estimates how long it should take for a node to reach the height.
 // More nodes in a network implies we may expect a slower network and may have to wait longer.
 func waitingTime(nodes int) time.Duration {
-	return time.Minute + (time.Duration(nodes) * (15 * time.Second))
+	return time.Minute + (time.Duration(nodes) * (30 * time.Second))
 }