From 1bfd67dfc642b3dac87ce2f102c27df9cb52d23b Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Thu, 25 Aug 2016 18:27:21 -0400
Subject: [PATCH 1/3] test: refactor bash; test fastsync (failing)

---
 scripts/install_tmsp_apps.sh                  |  1 +
 test/{ => docker}/Dockerfile                  |  3 +-
 test/docker/DockerfileDev                     |  3 ++
 test/docker/build.sh                          |  3 ++
 test/docker/update.sh                         |  9 ++++
 test/net/test.sh                              |  2 +
 test/p2p/a.sh                                 |  8 ++++
 .../{run_test.sh => atomic_broadcast/test.sh} | 25 ++++++++---
 test/p2p/clean.sh                             |  1 +
 test/p2p/{test_client.sh => client.sh}        |  6 ++-
 test/p2p/fast_sync/test.sh                    | 44 +++++++++++++++++++
 test/p2p/ip.sh                                |  7 +++
 test/p2p/local_testnet.sh                     | 13 ++----
 test/p2p/peer.sh                              | 23 ++++++++++
 test/p2p/test.sh                              | 28 +++++++++++-
 test/test.sh                                  |  8 ++--
 16 files changed, 158 insertions(+), 26 deletions(-)
 rename test/{ => docker}/Dockerfile (99%)
 create mode 100644 test/docker/DockerfileDev
 create mode 100644 test/docker/build.sh
 create mode 100644 test/docker/update.sh
 create mode 100644 test/p2p/a.sh
 rename test/p2p/{run_test.sh => atomic_broadcast/test.sh} (76%)
 rename test/p2p/{test_client.sh => client.sh} (74%)
 create mode 100644 test/p2p/fast_sync/test.sh
 create mode 100755 test/p2p/ip.sh
 create mode 100644 test/p2p/peer.sh

diff --git a/scripts/install_tmsp_apps.sh b/scripts/install_tmsp_apps.sh
index 035bc8d3e..6abe4da09 100644
--- a/scripts/install_tmsp_apps.sh
+++ b/scripts/install_tmsp_apps.sh
@@ -2,6 +2,7 @@
 
 go get github.com/tendermint/tmsp/...
 
+# get the tmsp commit used by tendermint
 COMMIT=`bash scripts/glide/parse.sh $(pwd)/glide.lock tmsp`
 
 cd $GOPATH/src/github.com/tendermint/tmsp
diff --git a/test/Dockerfile b/test/docker/Dockerfile
similarity index 99%
rename from test/Dockerfile
rename to test/docker/Dockerfile
index 26995de38..1de79b2a1 100644
--- a/test/Dockerfile
+++ b/test/docker/Dockerfile
@@ -14,10 +14,9 @@ WORKDIR $TENDERMINT_ORG/tendermint
 
 RUN make get_vendor_deps
 
-RUN go install ./cmd/tendermint
 
+RUN go install ./cmd/tendermint
 RUN bash scripts/install_tmsp_apps.sh
 
-
 EXPOSE 46656
 EXPOSE 46657
diff --git a/test/docker/DockerfileDev b/test/docker/DockerfileDev
new file mode 100644
index 000000000..7b31b1bfc
--- /dev/null
+++ b/test/docker/DockerfileDev
@@ -0,0 +1,3 @@
+FROM tester
+
+VOLUME /go/bin
diff --git a/test/docker/build.sh b/test/docker/build.sh
new file mode 100644
index 000000000..39df08720
--- /dev/null
+++ b/test/docker/build.sh
@@ -0,0 +1,3 @@
+#! /bin/bash
+
+docker build -t tester -f ./test/docker/Dockerfile .
diff --git a/test/docker/update.sh b/test/docker/update.sh
new file mode 100644
index 000000000..b946bf395
--- /dev/null
+++ b/test/docker/update.sh
@@ -0,0 +1,9 @@
+#! /bin/bash
+
+# update the `tester` image by copying in the latest tendermint binary
+
+docker run --name builder tester true
+docker cp $GOPATH/bin/tendermint builder:/go/bin/tendermint
+docker commit builder tester
+docker rm -vf builder
+
diff --git a/test/net/test.sh b/test/net/test.sh
index 9ec03e9a2..e3029a0dd 100644
--- a/test/net/test.sh
+++ b/test/net/test.sh
@@ -1,6 +1,8 @@
 #! /bin/bash
 set -eu
 
+# start a testnet and benchmark throughput using mintnet+netmon via the network_testing repo
+
 DATACENTER=single
 VALSETSIZE=4
 BLOCKSIZE=8092
diff --git a/test/p2p/a.sh b/test/p2p/a.sh
new file mode 100644
index 000000000..189548f9e
--- /dev/null
+++ b/test/p2p/a.sh
@@ -0,0 +1,8 @@
+#! /bin/bash
+
+h1=10
+h2=0
+while [ $h2 -lt $(($h1+3)) ]; do
+	echo "$h2"
+	h2=$(($h2+1))
+done
diff --git a/test/p2p/run_test.sh b/test/p2p/atomic_broadcast/test.sh
similarity index 76%
rename from test/p2p/run_test.sh
rename to test/p2p/atomic_broadcast/test.sh
index 87ab8e1f7..3b78166d6 100644
--- a/test/p2p/run_test.sh
+++ b/test/p2p/atomic_broadcast/test.sh
@@ -1,9 +1,20 @@
 #! /bin/bash
 
+###################################################################
+# wait for all peers to come online
+# for each peer:
+# 	wait to have 3 peers
+#	wait to be at height > 1
+#	send a tx, wait for commit
+#	assert app hash on every peer reflects the post tx state
+###################################################################
+
+N=4
+
 # wait for everyone to come online
 echo "Waiting for nodes to come online"
-for i in `seq 1 4`; do
-	addr="172.57.0.$((100+$i)):46657"
+for i in `seq 1 $N`; do
+	addr=$(test/p2p/ip.sh $i):46657
 	curl -s $addr/status > /dev/null
 	ERR=$?
 	while [ "$ERR" != 0 ]; do
@@ -16,8 +27,8 @@ done
 
 echo ""
 # run the test on each of them
-for i in `seq 1 4`; do
-	addr="172.57.0.$((100+$i)):46657"
+for i in `seq 1 $N`; do
+	addr=$(test/p2p/ip.sh $i):46657
 
 	# - assert everyone has 3 other peers
 	N_PEERS=`curl -s $addr/net_info | jq '.result[1].peers | length'`
@@ -61,9 +72,10 @@ for i in `seq 1 4`; do
 	fi
 
 	# check we get the same new hash on all other nodes
-	for j in `seq 1 4`; do
+	for j in `seq 1 $N`; do
 		if [[ "$i" != "$j" ]]; then
-			HASH3=`curl -s 172.57.0.$((100+$j)):46657/status | jq .result[1].latest_app_hash`
+			addrJ=$(test/p2p/ip.sh $j):46657
+			HASH3=`curl -s $addrJ/status | jq .result[1].latest_app_hash`
 			
 			if [[ "$HASH2" != "$HASH3" ]]; then
 				echo "App hash for node $j doesn't match. Got $HASH3, expected $HASH2"
@@ -77,3 +89,4 @@ done
 
 echo ""
 echo "PASS"
+echo ""
diff --git a/test/p2p/clean.sh b/test/p2p/clean.sh
index e0fb36956..44a1276ef 100644
--- a/test/p2p/clean.sh
+++ b/test/p2p/clean.sh
@@ -1,4 +1,5 @@
 #! /bin/bash
 
+# clean everything
 docker rm -vf $(docker ps -aq)
 docker network rm local_testnet
diff --git a/test/p2p/test_client.sh b/test/p2p/client.sh
similarity index 74%
rename from test/p2p/test_client.sh
rename to test/p2p/client.sh
index d9fe64c5c..3cf39d5b5 100644
--- a/test/p2p/test_client.sh
+++ b/test/p2p/client.sh
@@ -5,12 +5,14 @@ DOCKER_IMAGE=$1
 NETWORK_NAME=$2
 CMD=$3
 
+echo "starting test client container with CMD=$CMD"
 # run the test container on the local network
 docker run -t \
-	--rm \
 	-v $GOPATH/src/github.com/tendermint/tendermint/test/p2p/:/go/src/github.com/tendermint/tendermint/test/p2p \
 	--net=$NETWORK_NAME \
-	--ip=172.57.0.99 \
+	--ip=$(test/p2p/ip.sh "-1") \
 	--name test_container \
 	--entrypoint bash \
 	$DOCKER_IMAGE $CMD
+
+docker rm -vf test_container
diff --git a/test/p2p/fast_sync/test.sh b/test/p2p/fast_sync/test.sh
new file mode 100644
index 000000000..7a5453f00
--- /dev/null
+++ b/test/p2p/fast_sync/test.sh
@@ -0,0 +1,44 @@
+#! /bin/bash
+set -eu
+set -o pipefail
+
+###############################################################
+# for each peer:
+# 	kill peer
+# 	bring it back online via fast sync
+# 	check app hash
+###############################################################
+
+ID=$1
+
+addr=$(test/p2p/ip.sh $ID):46657
+peerID=$(( $(($ID % 4)) + 1  )) # 1->2 ... 3->4 ... 4->1
+peer_addr=$(test/p2p/ip.sh $peerID):46657
+
+# get another peer's height
+h1=`curl -s $peer_addr/status | jq .result[1].latest_block_height`
+
+# get another peer's state
+root1=`curl -s $peer_addr/status | jq .result[1].latest_app_hash`
+
+echo "Other peer is on height $h1 with state $root1"
+echo "Waiting for peer $ID to catch up"
+
+# wait for it to sync to past its previous height
+set +e
+set +o pipefail
+h2="0"
+while [[ "$h2" -lt "$(($h1+3))" ]]; do
+	sleep 1
+	h2=`curl -s $addr/status | jq .result[1].latest_block_height`
+	echo "... $h2"
+done
+
+# check the app hash
+root2=`curl -s $addr/status | jq .result[1].latest_app_hash`
+
+if [[ "$root1" != "$root2" ]]; then
+	echo "App hash after fast sync does not match. Got $root2; expected $root1"
+	exit 1
+fi
+echo "... fast sync successful"
diff --git a/test/p2p/ip.sh b/test/p2p/ip.sh
new file mode 100755
index 000000000..33ea890db
--- /dev/null
+++ b/test/p2p/ip.sh
@@ -0,0 +1,7 @@
+#! /bin/bash
+set -eu
+
+ID=$1
+echo "172.57.0.$((100+$ID))"
+
+
diff --git a/test/p2p/local_testnet.sh b/test/p2p/local_testnet.sh
index 2b4183b95..9380adfd4 100644
--- a/test/p2p/local_testnet.sh
+++ b/test/p2p/local_testnet.sh
@@ -10,19 +10,12 @@ cd $GOPATH/src/github.com/tendermint/tendermint
 docker network create --driver bridge --subnet 172.57.0.0/16 $NETWORK_NAME
 
 N=4
-seeds="172.57.0.101:46656"
+seeds="$(test/p2p/ip.sh 1):46656"
 for i in `seq 2 $N`; do
-	seeds="$seeds,172.57.0.$((100+$i)):46656"
+	seeds="$seeds,$(test/p2p/ip.sh $i):46656"
 done
 echo "Seeds: $seeds"
 
 for i in `seq 1 $N`; do
-	# start tendermint container
-	docker run -d \
-		--net=$NETWORK_NAME \
-		--ip=172.57.0.$((100+$i)) \
-		--name local_testnet_$i \
-		--entrypoint tendermint \
-		-e TMROOT=/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$i/core \
-		$DOCKER_IMAGE node --seeds $seeds --proxy_app=dummy
+	bash test/p2p/peer.sh $DOCKER_IMAGE $NETWORK_NAME $i $seeds
 done
diff --git a/test/p2p/peer.sh b/test/p2p/peer.sh
new file mode 100644
index 000000000..d1405eff1
--- /dev/null
+++ b/test/p2p/peer.sh
@@ -0,0 +1,23 @@
+#! /bin/bash
+set -eu
+
+DOCKER_IMAGE=$1
+NETWORK_NAME=$2
+ID=$3
+
+set +u
+SEEDS=$4
+set -u
+if [[ "$SEEDS" != "" ]]; then
+	SEEDS=" --seeds $SEEDS "
+fi
+
+echo "starting tendermint peer ID=$ID"
+# start tendermint container on the network
+docker run -d \
+	--net=$NETWORK_NAME \
+	--ip=$(test/p2p/ip.sh $ID) \
+	--name local_testnet_$ID \
+	--entrypoint tendermint \
+	-e TMROOT=/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$ID/core \
+	$DOCKER_IMAGE node $SEEDS --proxy_app=dummy
diff --git a/test/p2p/test.sh b/test/p2p/test.sh
index f139fd675..2a109f0bb 100644
--- a/test/p2p/test.sh
+++ b/test/p2p/test.sh
@@ -1,4 +1,5 @@
 #! /bin/bash
+set -eu
 
 DOCKER_IMAGE=$1
 NETWORK_NAME=local_testnet
@@ -6,5 +7,28 @@ NETWORK_NAME=local_testnet
 # start the testnet on a local network
 bash test/p2p/local_testnet.sh $DOCKER_IMAGE $NETWORK_NAME
 
-# run the test
-bash test/p2p/test_client.sh $DOCKER_IMAGE $NETWORK_NAME test/p2p/run_test.sh
+# test atomic broadcast
+bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME test/p2p/atomic_broadcast/test.sh
+
+# test fast sync (from current state of network)
+# run it on each of them
+N=4
+for i in `seq 1 $N`; do
+	echo "Testing fasysync on node $i"
+
+	# kill peer 
+	docker rm -vf local_testnet_$i
+
+	# restart peer - should have an empty blockchain
+	SEEDS="$(test/p2p/ip.sh 1):46656"
+	for j in `seq 2 $N`; do
+		SEEDS="$SEEDS,$(test/p2p/ip.sh $j):46656"
+	done
+	bash test/p2p/peer.sh $DOCKER_IMAGE $NETWORK_NAME $i $SEEDS
+
+	bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME "test/p2p/fast_sync/test.sh $i"
+done
+echo ""
+echo "PASS"
+echo ""
+
diff --git a/test/test.sh b/test/test.sh
index 0f486f677..6e2925902 100644
--- a/test/test.sh
+++ b/test/test.sh
@@ -5,17 +5,17 @@ set -eu
 # See the github.com/tendermint/tendermint/test/README.md
 
 echo ""
-echo "* building docker file"
-docker build -t tester -f ./test/Dockerfile .
+echo "* building docker image"
+bash ./test/docker/build.sh
 
 echo ""
-echo "* running go tests and app tests"
+echo "* running go tests and app tests in docker container"
 docker run -t tester bash test/run_test.sh
 
 # test basic network connectivity
 # by starting a local testnet and checking peers connect and make blocks
 echo ""
-echo "* running basic peer tests"
+echo "* running p2p tests on a local docker network"
 bash test/p2p/test.sh tester
 
 # only run the cloud benchmark for releases

From 7bec3330178e6f09a2c3ff36ae107addcc79e875 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Sat, 27 Aug 2016 14:15:18 -0400
Subject: [PATCH 2/3] fix fast sync

---
 blockchain/reactor.go | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/blockchain/reactor.go b/blockchain/reactor.go
index 53aa311cc..1cf78dcee 100644
--- a/blockchain/reactor.go
+++ b/blockchain/reactor.go
@@ -234,16 +234,18 @@ FOR_LOOP:
 					err := bcR.state.ExecBlock(bcR.evsw, bcR.proxyAppConn, first, firstPartsHeader)
 					if err != nil {
 						// TODO This is bad, are we zombie?
-						PanicQ(Fmt("Failed to process committed block: %v", err))
+						PanicQ(Fmt("Failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err))
+					}
+					// NOTE: we could improve performance if we
+					// didn't make the app commit to disk every block
+					// ... but we would need a way to get the hash without it persisting
+					res := bcR.proxyAppConn.CommitSync()
+					if res.IsErr() {
+						// TODO Handle gracefully.
+						PanicQ(Fmt("Failed to commit block at application: %v", res))
 					}
-					/*
-						err = bcR.proxyAppConn.CommitSync()
-						if err != nil {
-							// TODO Handle gracefully.
-							PanicQ(Fmt("Failed to commit block at application: %v", err))
-						}
-					*/
 					bcR.store.SaveBlock(first, firstParts, second.LastCommit)
+					bcR.state.AppHash = res.Data // ...
 					bcR.state.Save()
 				}
 			}

From bf7ba17932e9cf18fbddbf4129d96173ad2709e1 Mon Sep 17 00:00:00 2001
From: Ethan Buchman <ethan@coinculture.info>
Date: Sat, 27 Aug 2016 16:01:52 -0400
Subject: [PATCH 3/3] test/p2p: name client conts so we dont need to rm them
 because circle

---
 blockchain/reactor.go | 3 ++-
 test/p2p/a.sh         | 8 --------
 test/p2p/client.sh    | 6 +++---
 test/p2p/test.sh      | 8 ++++++--
 4 files changed, 11 insertions(+), 14 deletions(-)
 delete mode 100644 test/p2p/a.sh

diff --git a/blockchain/reactor.go b/blockchain/reactor.go
index 1cf78dcee..e471633c0 100644
--- a/blockchain/reactor.go
+++ b/blockchain/reactor.go
@@ -231,6 +231,7 @@ FOR_LOOP:
 					break SYNC_LOOP
 				} else {
 					bcR.pool.PopRequest()
+					// TODO: use ApplyBlock instead of Exec/Commit/SetAppHash/Save
 					err := bcR.state.ExecBlock(bcR.evsw, bcR.proxyAppConn, first, firstPartsHeader)
 					if err != nil {
 						// TODO This is bad, are we zombie?
@@ -245,7 +246,7 @@ FOR_LOOP:
 						PanicQ(Fmt("Failed to commit block at application: %v", res))
 					}
 					bcR.store.SaveBlock(first, firstParts, second.LastCommit)
-					bcR.state.AppHash = res.Data // ...
+					bcR.state.AppHash = res.Data
 					bcR.state.Save()
 				}
 			}
diff --git a/test/p2p/a.sh b/test/p2p/a.sh
deleted file mode 100644
index 189548f9e..000000000
--- a/test/p2p/a.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#! /bin/bash
-
-h1=10
-h2=0
-while [ $h2 -lt $(($h1+3)) ]; do
-	echo "$h2"
-	h2=$(($h2+1))
-done
diff --git a/test/p2p/client.sh b/test/p2p/client.sh
index 3cf39d5b5..efc5096ef 100644
--- a/test/p2p/client.sh
+++ b/test/p2p/client.sh
@@ -3,7 +3,8 @@ set -eu
 
 DOCKER_IMAGE=$1
 NETWORK_NAME=$2
-CMD=$3
+ID=$3
+CMD=$4
 
 echo "starting test client container with CMD=$CMD"
 # run the test container on the local network
@@ -11,8 +12,7 @@ docker run -t \
 	-v $GOPATH/src/github.com/tendermint/tendermint/test/p2p/:/go/src/github.com/tendermint/tendermint/test/p2p \
 	--net=$NETWORK_NAME \
 	--ip=$(test/p2p/ip.sh "-1") \
-	--name test_container \
+	--name test_container_$ID \
 	--entrypoint bash \
 	$DOCKER_IMAGE $CMD
 
-docker rm -vf test_container
diff --git a/test/p2p/test.sh b/test/p2p/test.sh
index 2a109f0bb..4d021a4ee 100644
--- a/test/p2p/test.sh
+++ b/test/p2p/test.sh
@@ -4,11 +4,13 @@ set -eu
 DOCKER_IMAGE=$1
 NETWORK_NAME=local_testnet
 
+cd $GOPATH/src/github.com/tendermint/tendermint
+
 # start the testnet on a local network
 bash test/p2p/local_testnet.sh $DOCKER_IMAGE $NETWORK_NAME
 
 # test atomic broadcast
-bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME test/p2p/atomic_broadcast/test.sh
+bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME ab test/p2p/atomic_broadcast/test.sh
 
 # test fast sync (from current state of network)
 # run it on each of them
@@ -17,7 +19,9 @@ for i in `seq 1 $N`; do
 	echo "Testing fasysync on node $i"
 
 	# kill peer 
+	set +e # circle sigh :(
 	docker rm -vf local_testnet_$i
+	set -e 
 
 	# restart peer - should have an empty blockchain
 	SEEDS="$(test/p2p/ip.sh 1):46656"
@@ -26,7 +30,7 @@ for i in `seq 1 $N`; do
 	done
 	bash test/p2p/peer.sh $DOCKER_IMAGE $NETWORK_NAME $i $SEEDS
 
-	bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME "test/p2p/fast_sync/test.sh $i"
+	bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME fs_$i "test/p2p/fast_sync/test.sh $i"
 done
 echo ""
 echo "PASS"