test/p2p: kill and restart all nodes

.gitignore

@@ -11,3 +11,4 @@ remote_dump
 .revision
 vendor
 .vagrant
+test/p2p/data/

test/p2p/kill_all/check_peers.sh

@@ -0,0 +1,48 @@
+#! /bin/bash
+set -eu
+
+NUM_OF_PEERS=$1
+
+# how many attempts for each peer to catch up by height
+MAX_ATTEMPTS_TO_CATCH_UP=10
+
+echo "Waiting for nodes to come online"
+set +e
+for i in $(seq 1 "$NUM_OF_PEERS"); do
+  addr=$(test/p2p/ip.sh "$i"):46657
+  curl -s "$addr/status" > /dev/null
+  ERR=$?
+  while [ "$ERR" != 0 ]; do
+    sleep 1
+    curl -s "$addr/status" > /dev/null
+    ERR=$?
+  done
+  echo "... node $i is up"
+done
+set -e
+
+# get the first peer's height
+addr=$(test/p2p/ip.sh 1):46657
+h1=$(curl -s "$addr/status" | jq .result[1].latest_block_height)
+echo "1st peer is on height $h1"
+
+echo "Waiting until other peers reporting a height higher than the 1st one"
+for i in $(seq 2 "$NUM_OF_PEERS"); do
+  attempt=1
+  hi=0
+
+  while [[ $hi -le $h1 ]] ; do
+    addr=$(test/p2p/ip.sh "$i"):46657
+    hi=$(curl -s "$addr/status" | jq .result[1].latest_block_height)
+
+    echo "... peer $i is on height $hi"
+
+    ((attempt++))
+    if [ "$attempt" -ge $MAX_ATTEMPTS_TO_CATCH_UP ] ; then
+      echo "$attempt unsuccessful attempts were made to catch up"
+      exit 1
+    fi
+
+    sleep 1
+  done
+done

test/p2p/kill_all/test.sh

@@ -0,0 +1,29 @@
+#! /bin/bash
+set -eu
+
+DOCKER_IMAGE=$1
+NETWORK_NAME=$2
+NUM_OF_PEERS=$3
+NUM_OF_CRASHES=$4
+
+cd "$GOPATH/src/github.com/tendermint/tendermint"
+
+###############################################################
+# NUM_OF_CRASHES times:
+# 	restart all peers
+# 	wait for them to sync and check that they are making progress
+###############################################################
+
+for i in $(seq 1 "$NUM_OF_CRASHES"); do
+  # restart all peers
+  for i in $(seq 1 "$NUM_OF_PEERS"); do
+    docker stop "local_testnet_$i"
+    docker start "local_testnet_$i"
+  done
+
+  bash test/p2p/client.sh "$DOCKER_IMAGE" "$NETWORK_NAME" kill_all "test/p2p/kill_all/check_peers.sh $NUM_OF_PEERS"
+done
+
+echo ""
+echo "PASS"
+echo ""

test/p2p/local_testnet_stop.sh

@@ -6,7 +6,7 @@ N=$2
 
 for i in `seq 1 $N`; do
   docker stop local_testnet_$i
-  docker rm local_testnet_$i
+  docker rm -vf local_testnet_$i
 done
 
 docker network rm $NETWORK_NAME

test/p2p/peer.sh

@@ -19,5 +19,6 @@ docker run -d \
 	--ip=$(test/p2p/ip.sh $ID) \
 	--name local_testnet_$ID \
 	--entrypoint tendermint \
+  -v $GOPATH/src/github.com/tendermint/tendermint/test/p2p/:/go/src/github.com/tendermint/tendermint/test/p2p \
 	-e TMROOT=/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$ID/core \
 	$DOCKER_IMAGE node $SEEDS --proxy_app=dummy

test/p2p/test.sh

@@ -26,3 +26,6 @@ bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME ab "test/p2p/atomic_broadcas
 # test fast sync (from current state of network):
 # for each node, kill it and readd via fast sync
 bash test/p2p/fast_sync/test.sh $DOCKER_IMAGE $NETWORK_NAME $N
+
+# test killing all peers
+bash test/p2p/kill_all/test.sh $DOCKER_IMAGE $NETWORK_NAME $N 3