From 8b4f0dba7018381a3aafab4fbdc8765fd2726dc8 Mon Sep 17 00:00:00 2001
From: Erik Grinaker <erik@interchain.berlin>
Date: Thu, 22 Oct 2020 22:53:27 +0200
Subject: [PATCH] test: run remaining E2E testnets on run-multiple.sh failure
 (#5557)

Fixes #5542.
---
 test/e2e/run-multiple.sh | 26 ++++++++++++++++++++------
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/test/e2e/run-multiple.sh b/test/e2e/run-multiple.sh
index e9699cf1e..5d6a20ef9 100755
--- a/test/e2e/run-multiple.sh
+++ b/test/e2e/run-multiple.sh
@@ -3,33 +3,47 @@
 # This is a convenience script that takes a list of testnet manifests
 # as arguments and runs each one of them sequentially. If a testnet
 # fails, the container logs are dumped to stdout along with the testnet
-# manifest.
+# manifest, but the remaining testnets are still run.
 #
 # This is mostly used to run generated networks in nightly CI jobs.
 #
 
-# Don't set -e, since we explicitly check status codes ourselves.
-set -u
+set -euo pipefail
 
 if [[ $# == 0 ]]; then
 	echo "Usage: $0 [MANIFEST...]" >&2
 	exit 1
 fi
 
+FAILED=()
+
 for MANIFEST in "$@"; do
 	START=$SECONDS
 	echo "==> Running testnet $MANIFEST..."
-	./build/runner -f "$MANIFEST"
 
-	if [[ $? -ne 0 ]]; then
+	if ! ./build/runner -f "$MANIFEST"; then
 		echo "==> Testnet $MANIFEST failed, dumping manifest..."
 		cat "$MANIFEST"
 
 		echo "==> Dumping container logs for $MANIFEST..."
 		./build/runner -f "$MANIFEST" logs
-		exit 1
+
+		echo "==> Cleaning up failed testnet $MANIFEST..."
+		./build/runner -f "$MANIFEST" cleanup
+
+		FAILED+=("$MANIFEST")
 	fi
 
 	echo "==> Completed testnet $MANIFEST in $(( SECONDS - START ))s"
 	echo ""
 done
+
+if [[ ${#FAILED[@]} -ne 0 ]]; then
+	echo "${#FAILED[@]} testnets failed:"
+	for MANIFEST in "${FAILED[@]}"; do
+		echo "- $MANIFEST"
+	done
+	exit 1
+else
+	echo "All testnets successful"
+fi