Merge pull request #267 from tendermint/fastsync_fix

Fastsync fix

blockchain/reactor.go

@@ -231,19 +231,22 @@ FOR_LOOP:
 					break SYNC_LOOP
 				} else {
 					bcR.pool.PopRequest()
+					// TODO: use ApplyBlock instead of Exec/Commit/SetAppHash/Save
 					err := bcR.state.ExecBlock(bcR.evsw, bcR.proxyAppConn, first, firstPartsHeader)
 					if err != nil {
 						// TODO This is bad, are we zombie?
-						PanicQ(Fmt("Failed to process committed block: %v", err))
+						PanicQ(Fmt("Failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err))
+					}
+					// NOTE: we could improve performance if we
+					// didn't make the app commit to disk every block
+					// ... but we would need a way to get the hash without it persisting
+					res := bcR.proxyAppConn.CommitSync()
+					if res.IsErr() {
+						// TODO Handle gracefully.
+						PanicQ(Fmt("Failed to commit block at application: %v", res))
 					}
-					/*
-						err = bcR.proxyAppConn.CommitSync()
-						if err != nil {
-							// TODO Handle gracefully.
-							PanicQ(Fmt("Failed to commit block at application: %v", err))
-						}
-					*/
 					bcR.store.SaveBlock(first, firstParts, second.LastCommit)
+					bcR.state.AppHash = res.Data
 					bcR.state.Save()
 				}
 			}

scripts/install_tmsp_apps.sh

@@ -2,6 +2,7 @@
 
 go get github.com/tendermint/tmsp/...
 
+# get the tmsp commit used by tendermint
 COMMIT=`bash scripts/glide/parse.sh $(pwd)/glide.lock tmsp`
 
 cd $GOPATH/src/github.com/tendermint/tmsp

test/docker/DockerfileDev

@@ -0,0 +1,3 @@
+FROM tester
+
+VOLUME /go/bin

test/docker/build.sh

@@ -0,0 +1,3 @@
+#! /bin/bash
+
+docker build -t tester -f ./test/docker/Dockerfile .

test/docker/update.sh

@@ -0,0 +1,9 @@
+#! /bin/bash
+
+# update the `tester` image by copying in the latest tendermint binary
+
+docker run --name builder tester true
+docker cp $GOPATH/bin/tendermint builder:/go/bin/tendermint
+docker commit builder tester
+docker rm -vf builder
+

test/net/test.sh

@@ -1,6 +1,8 @@
 #! /bin/bash
 set -eu
 
+# start a testnet and benchmark throughput using mintnet+netmon via the network_testing repo
+
 DATACENTER=single
 VALSETSIZE=4
 BLOCKSIZE=8092

test/p2p/clean.sh

@@ -1,4 +1,5 @@
 #! /bin/bash
 
+# clean everything
 docker rm -vf $(docker ps -aq)
 docker network rm local_testnet

test/p2p/fast_sync/test.sh

@@ -0,0 +1,44 @@
+#! /bin/bash
+set -eu
+set -o pipefail
+
+###############################################################
+# for each peer:
+# 	kill peer
+# 	bring it back online via fast sync
+# 	check app hash
+###############################################################
+
+ID=$1
+
+addr=$(test/p2p/ip.sh $ID):46657
+peerID=$(( $(($ID % 4)) + 1  )) # 1->2 ... 3->4 ... 4->1
+peer_addr=$(test/p2p/ip.sh $peerID):46657
+
+# get another peer's height
+h1=`curl -s $peer_addr/status | jq .result[1].latest_block_height`
+
+# get another peer's state
+root1=`curl -s $peer_addr/status | jq .result[1].latest_app_hash`
+
+echo "Other peer is on height $h1 with state $root1"
+echo "Waiting for peer $ID to catch up"
+
+# wait for it to sync to past its previous height
+set +e
+set +o pipefail
+h2="0"
+while [[ "$h2" -lt "$(($h1+3))" ]]; do
+	sleep 1
+	h2=`curl -s $addr/status | jq .result[1].latest_block_height`
+	echo "... $h2"
+done
+
+# check the app hash
+root2=`curl -s $addr/status | jq .result[1].latest_app_hash`
+
+if [[ "$root1" != "$root2" ]]; then
+	echo "App hash after fast sync does not match. Got $root2; expected $root1"
+	exit 1
+fi
+echo "... fast sync successful"

test/p2p/ip.sh

@@ -0,0 +1,7 @@
+#! /bin/bash
+set -eu
+
+ID=$1
+echo "172.57.0.$((100+$ID))"
+
+

test/p2p/local_testnet.sh

@@ -10,19 +10,12 @@ cd $GOPATH/src/github.com/tendermint/tendermint
 docker network create --driver bridge --subnet 172.57.0.0/16 $NETWORK_NAME
 
 N=4
-seeds="172.57.0.101:46656"
+seeds="$(test/p2p/ip.sh 1):46656"
 for i in `seq 2 $N`; do
-	seeds="$seeds,172.57.0.$((100+$i)):46656"
+	seeds="$seeds,$(test/p2p/ip.sh $i):46656"
 done
 echo "Seeds: $seeds"
 
 for i in `seq 1 $N`; do
-	# start tendermint container
-	docker run -d \
-		--net=$NETWORK_NAME \
-		--ip=172.57.0.$((100+$i)) \
-		--name local_testnet_$i \
-		--entrypoint tendermint \
-		-e TMROOT=/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$i/core \
-		$DOCKER_IMAGE node --seeds $seeds --proxy_app=dummy
+	bash test/p2p/peer.sh $DOCKER_IMAGE $NETWORK_NAME $i $seeds
 done

test/p2p/peer.sh

@@ -0,0 +1,23 @@
+#! /bin/bash
+set -eu
+
+DOCKER_IMAGE=$1
+NETWORK_NAME=$2
+ID=$3
+
+set +u
+SEEDS=$4
+set -u
+if [[ "$SEEDS" != "" ]]; then
+	SEEDS=" --seeds $SEEDS "
+fi
+
+echo "starting tendermint peer ID=$ID"
+# start tendermint container on the network
+docker run -d \
+	--net=$NETWORK_NAME \
+	--ip=$(test/p2p/ip.sh $ID) \
+	--name local_testnet_$ID \
+	--entrypoint tendermint \
+	-e TMROOT=/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$ID/core \
+	$DOCKER_IMAGE node $SEEDS --proxy_app=dummy

test/p2p/test.sh

@@ -1,10 +1,38 @@
 #! /bin/bash
+set -eu
 
 DOCKER_IMAGE=$1
 NETWORK_NAME=local_testnet
 
+cd $GOPATH/src/github.com/tendermint/tendermint
+
 # start the testnet on a local network
 bash test/p2p/local_testnet.sh $DOCKER_IMAGE $NETWORK_NAME
 
-# run the test
-bash test/p2p/test_client.sh $DOCKER_IMAGE $NETWORK_NAME test/p2p/run_test.sh
+# test atomic broadcast
+bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME ab test/p2p/atomic_broadcast/test.sh
+
+# test fast sync (from current state of network)
+# run it on each of them
+N=4
+for i in `seq 1 $N`; do
+	echo "Testing fasysync on node $i"
+
+	# kill peer 
+	set +e # circle sigh :(
+	docker rm -vf local_testnet_$i
+	set -e 
+
+	# restart peer - should have an empty blockchain
+	SEEDS="$(test/p2p/ip.sh 1):46656"
+	for j in `seq 2 $N`; do
+		SEEDS="$SEEDS,$(test/p2p/ip.sh $j):46656"
+	done
+	bash test/p2p/peer.sh $DOCKER_IMAGE $NETWORK_NAME $i $SEEDS
+
+	bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME fs_$i "test/p2p/fast_sync/test.sh $i"
+done
+echo ""
+echo "PASS"
+echo ""
+

test/test.sh

@@ -5,17 +5,17 @@ set -eu
 # See the github.com/tendermint/tendermint/test/README.md
 
 echo ""
-echo "* building docker file"
-docker build -t tester -f ./test/Dockerfile .
+echo "* building docker image"
+bash ./test/docker/build.sh
 
 echo ""
-echo "* running go tests and app tests"
+echo "* running go tests and app tests in docker container"
 docker run -t tester bash test/run_test.sh
 
 # test basic network connectivity
 # by starting a local testnet and checking peers connect and make blocks
 echo ""
-echo "* running basic peer tests"
+echo "* running p2p tests on a local docker network"
 bash test/p2p/test.sh tester
 
 # only run the cloud benchmark for releases