diff --git a/test/e2e/run-multiple.sh b/test/e2e/run-multiple.sh index e9699cf1e..5d6a20ef9 100755 --- a/test/e2e/run-multiple.sh +++ b/test/e2e/run-multiple.sh @@ -3,33 +3,47 @@ # This is a convenience script that takes a list of testnet manifests # as arguments and runs each one of them sequentially. If a testnet # fails, the container logs are dumped to stdout along with the testnet -# manifest. +# manifest, but the remaining testnets are still run. # # This is mostly used to run generated networks in nightly CI jobs. # -# Don't set -e, since we explicitly check status codes ourselves. -set -u +set -euo pipefail if [[ $# == 0 ]]; then echo "Usage: $0 [MANIFEST...]" >&2 exit 1 fi +FAILED=() + for MANIFEST in "$@"; do START=$SECONDS echo "==> Running testnet $MANIFEST..." - ./build/runner -f "$MANIFEST" - if [[ $? -ne 0 ]]; then + if ! ./build/runner -f "$MANIFEST"; then echo "==> Testnet $MANIFEST failed, dumping manifest..." cat "$MANIFEST" echo "==> Dumping container logs for $MANIFEST..." ./build/runner -f "$MANIFEST" logs - exit 1 + + echo "==> Cleaning up failed testnet $MANIFEST..." + ./build/runner -f "$MANIFEST" cleanup + + FAILED+=("$MANIFEST") fi echo "==> Completed testnet $MANIFEST in $(( SECONDS - START ))s" echo "" done + +if [[ ${#FAILED[@]} -ne 0 ]]; then + echo "${#FAILED[@]} testnets failed:" + for MANIFEST in "${FAILED[@]}"; do + echo "- $MANIFEST" + done + exit 1 +else + echo "All testnets successful" +fi