From 30328548f7616ddd3e78bb145dae8ddf774cff8e Mon Sep 17 00:00:00 2001
From: Anton Kalyaev <anton.kalyaev@gmail.com>
Date: Wed, 21 Dec 2016 01:36:06 +0400
Subject: [PATCH] test/p2p: kill and restart all nodes

---
 .gitignore                       |  1 +
 test/p2p/kill_all/check_peers.sh | 48 ++++++++++++++++++++++++++++++++
 test/p2p/kill_all/test.sh        | 29 +++++++++++++++++++
 test/p2p/local_testnet_stop.sh   |  2 +-
 test/p2p/peer.sh                 |  1 +
 test/p2p/test.sh                 |  3 ++
 6 files changed, 83 insertions(+), 1 deletion(-)
 create mode 100644 test/p2p/kill_all/check_peers.sh
 create mode 100644 test/p2p/kill_all/test.sh

diff --git a/.gitignore b/.gitignore
index f3af0c402..acc957a9e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,3 +11,4 @@ remote_dump
 .revision
 vendor
 .vagrant
+test/p2p/data/
diff --git a/test/p2p/kill_all/check_peers.sh b/test/p2p/kill_all/check_peers.sh
new file mode 100644
index 000000000..31f6d5504
--- /dev/null
+++ b/test/p2p/kill_all/check_peers.sh
@@ -0,0 +1,48 @@
+#! /bin/bash
+set -eu
+
+NUM_OF_PEERS=$1
+
+# how many attempts for each peer to catch up by height
+MAX_ATTEMPTS_TO_CATCH_UP=10
+
+echo "Waiting for nodes to come online"
+set +e
+for i in $(seq 1 "$NUM_OF_PEERS"); do
+  addr=$(test/p2p/ip.sh "$i"):46657
+  curl -s "$addr/status" > /dev/null
+  ERR=$?
+  while [ "$ERR" != 0 ]; do
+    sleep 1
+    curl -s "$addr/status" > /dev/null
+    ERR=$?
+  done
+  echo "... node $i is up"
+done
+set -e
+
+# get the first peer's height
+addr=$(test/p2p/ip.sh 1):46657
+h1=$(curl -s "$addr/status" | jq .result[1].latest_block_height)
+echo "1st peer is on height $h1"
+
+echo "Waiting until other peers reporting a height higher than the 1st one"
+for i in $(seq 2 "$NUM_OF_PEERS"); do
+  attempt=1
+  hi=0
+
+  while [[ $hi -le $h1 ]] ; do
+    addr=$(test/p2p/ip.sh "$i"):46657
+    hi=$(curl -s "$addr/status" | jq .result[1].latest_block_height)
+
+    echo "... peer $i is on height $hi"
+
+    ((attempt++))
+    if [ "$attempt" -ge $MAX_ATTEMPTS_TO_CATCH_UP ] ; then
+      echo "$attempt unsuccessful attempts were made to catch up"
+      exit 1
+    fi
+
+    sleep 1
+  done
+done
diff --git a/test/p2p/kill_all/test.sh b/test/p2p/kill_all/test.sh
new file mode 100644
index 000000000..e8157d7be
--- /dev/null
+++ b/test/p2p/kill_all/test.sh
@@ -0,0 +1,29 @@
+#! /bin/bash
+set -eu
+
+DOCKER_IMAGE=$1
+NETWORK_NAME=$2
+NUM_OF_PEERS=$3
+NUM_OF_CRASHES=$4
+
+cd "$GOPATH/src/github.com/tendermint/tendermint"
+
+###############################################################
+# NUM_OF_CRASHES times:
+# 	restart all peers
+# 	wait for them to sync and check that they are making progress
+###############################################################
+
+for i in $(seq 1 "$NUM_OF_CRASHES"); do
+  # restart all peers
+  for i in $(seq 1 "$NUM_OF_PEERS"); do
+    docker stop "local_testnet_$i"
+    docker start "local_testnet_$i"
+  done
+
+  bash test/p2p/client.sh "$DOCKER_IMAGE" "$NETWORK_NAME" kill_all "test/p2p/kill_all/check_peers.sh $NUM_OF_PEERS"
+done
+
+echo ""
+echo "PASS"
+echo ""
diff --git a/test/p2p/local_testnet_stop.sh b/test/p2p/local_testnet_stop.sh
index 6fe23ab2f..8edd3eeb7 100644
--- a/test/p2p/local_testnet_stop.sh
+++ b/test/p2p/local_testnet_stop.sh
@@ -6,7 +6,7 @@ N=$2
 
 for i in `seq 1 $N`; do
   docker stop local_testnet_$i
-  docker rm local_testnet_$i
+  docker rm -vf local_testnet_$i
 done
 
 docker network rm $NETWORK_NAME
diff --git a/test/p2p/peer.sh b/test/p2p/peer.sh
index d1405eff1..32e696e76 100644
--- a/test/p2p/peer.sh
+++ b/test/p2p/peer.sh
@@ -19,5 +19,6 @@ docker run -d \
 	--ip=$(test/p2p/ip.sh $ID) \
 	--name local_testnet_$ID \
 	--entrypoint tendermint \
+  -v $GOPATH/src/github.com/tendermint/tendermint/test/p2p/:/go/src/github.com/tendermint/tendermint/test/p2p \
 	-e TMROOT=/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$ID/core \
 	$DOCKER_IMAGE node $SEEDS --proxy_app=dummy
diff --git a/test/p2p/test.sh b/test/p2p/test.sh
index 58beb9a4a..2a45e7f8c 100644
--- a/test/p2p/test.sh
+++ b/test/p2p/test.sh
@@ -26,3 +26,6 @@ bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME ab "test/p2p/atomic_broadcas
 # test fast sync (from current state of network):
 # for each node, kill it and readd via fast sync
 bash test/p2p/fast_sync/test.sh $DOCKER_IMAGE $NETWORK_NAME $N
+
+# test killing all peers
+bash test/p2p/kill_all/test.sh $DOCKER_IMAGE $NETWORK_NAME $N 3