Testing refactor for Jenkins (#1098)

* de-mystify tests & run them in parallel (#1031)

* test optimization for jenkins (#1093)

* makefile cleanup

* tests: split fast and slow go tests, closes #1055

* pr comments

* restore circle conditions

* fix need_abci

* ...

* docker run: no :Z for circle?

* Remove cmd breaking comment

Makefile

@@ -5,7 +5,7 @@ PACKAGES=$(shell go list ./... | grep -v '/vendor/')
 BUILD_TAGS?=tendermint
 BUILD_FLAGS = -ldflags "-X github.com/tendermint/tendermint/version.GitCommit=`git rev-parse --short=8 HEAD`"
 
-all: check build test install
+all: check build test_integrations install
 
 check: check_tools ensure_deps
 
@@ -73,16 +73,56 @@ get_deps_bin_size:
 ########################################
 ### Testing
 
-test:
-	@echo "--> Running go test"
-	@go test $(PACKAGES)
-
-test_race:
-	@echo "--> Running go test --race"
-	@go test -race $(PACKAGES)
+## required to be run first by most tests
+build_docker_test_image:
+	docker build -t tester -f ./test/docker/Dockerfile .
+
+### coverage, app, persistence, and libs tests
+test_cover:
+	# run the go unit tests with coverage
+	bash test/test_cover.sh
+	
+test_apps:
+	# run the app tests using bash
+	# requires `abci-cli` and `tendermint` binaries installed
+	bash test/app/test.sh
+
+test_persistence:
+	# run the persistence tests using bash
+	# requires `abci-cli` installed
+	docker run --name run_persistence -t tester bash test/persist/test_failure_indices.sh
+
+	# TODO undockerize
+	# bash test/persist/test_failure_indices.sh
+
+test_p2p:
+	docker rm -f rsyslog || true
+	rm -rf test/logs || true
+	mkdir test/logs
+	cd test/
+	docker run -d -v "logs:/var/log/" -p 127.0.0.1:5514:514/udp --name rsyslog voxxit/rsyslog
+	cd ..
+	# requires 'tester' the image from above
+	bash test/p2p/test.sh tester
+
+need_abci:
+	bash scripts/install_abci_apps.sh
 
 test_integrations:
-	@bash ./test/test.sh
+	make build_docker_test_image
+	make get_tools
+	make get_vendor_deps
+	make install
+	make need_abci
+	make test_cover
+	make test_apps
+	make test_persistence
+	make test_p2p
+
+test_libs:
+	# checkout every github.com/tendermint dir and run its tests
+	# NOTE: on release-* or master branches only (set by Jenkins)
+	docker run --name run_libs -t tester bash test/test_libs.sh
 
 test_release:
 	@go test -tags release $(PACKAGES)
@@ -92,10 +132,17 @@ test100:
 
 vagrant_test:
 	vagrant up
-	vagrant ssh -c 'make install'
-	vagrant ssh -c 'make test_race'
 	vagrant ssh -c 'make test_integrations'
 
+### go tests
+test:
+	@echo "--> Running go test"
+	@go test $(PACKAGES)
+
+test_race:
+	@echo "--> Running go test --race"
+	@go test -v -race $(PACKAGES)
+
 
 ########################################
 ### Formatting, linting, and vetting
@@ -136,8 +183,7 @@ metalinter_all:
 	@echo "--> Running linter (all)"
 	gometalinter.v2 --vendor --deadline=600s --enable-all --disable=lll ./...
 
-
 # To avoid unintended conflicts with file names, always add to .PHONY
 # unless there is a reason not to.
 # https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html
-.PHONY: check build build_race dist install check_tools get_tools update_tools get_vendor_deps draw_deps get_deps_bin_size test test_race test_integrations test_release test100 vagrant_test fmt metalinter metalinter_all
+.PHONY: check build build_race dist install check_tools get_tools update_tools get_vendor_deps draw_depsbuild_test_docker_image test_cover test_apps test_persistence test_p2p test test_race test_libs test_integrations test_release test100 vagrant_test fmt

test/docker/Dockerfile

@@ -26,6 +26,7 @@ ADD scripts scripts
 RUN bash scripts/install_abci_apps.sh
 
 # Now copy in the code
+# NOTE: this will overwrite whatever is in vendor/
 COPY . $REPO
 
 

test/persist/test.sh

@@ -1,5 +0,0 @@
-#! /bin/bash
-
-cd "$GOPATH/src/github.com/tendermint/tendermint"
-
-bash ./test/persist/test_failure_indices.sh

test/persist/test_failure_indices.sh

@@ -25,22 +25,22 @@ function start_procs(){
     PID_DUMMY=$!
 
     # before starting tendermint, remove the rpc socket
-    rm $RPC_ADDR
+    rm -f $RPC_ADDR
     if [[ "$indexToFail" == "" ]]; then
         # run in background, dont fail
-        if [[ "$CIRCLECI" == true ]]; then
-            $TM_CMD &
-        else
+		if [[ "$CIRCLECI" == true ]]; then
+			$TM_CMD &
+		else
             $TM_CMD &> "tendermint_${name}.log" & 
-        fi
+		fi
         PID_TENDERMINT=$!
     else
         # run in foreground, fail
-        if [[ "$CIRCLECI" == true ]]; then
-            FAIL_TEST_INDEX=$indexToFail $TM_CMD
-        else 
+		if [[ "$CIRCLECI" == true ]]; then
+			FAIL_TEST_INDEX=$indexToFail $TM_CMD
+		else
             FAIL_TEST_INDEX=$indexToFail $TM_CMD &> "tendermint_${name}.log"
-        fi
+		fi
         PID_TENDERMINT=$!
     fi
 }

test/run_test.sh

@@ -1,23 +0,0 @@
-#! /bin/bash
-set -e
-
-pwd
-
-BRANCH=$(git rev-parse --abbrev-ref HEAD)
-echo "Current branch: $BRANCH"
-
-# run the go unit tests with coverage
-bash test/test_cover.sh
-
-# run the app tests using bash
-bash test/app/test.sh
-
-# run the persistence tests using bash
-bash test/persist/test.sh
-
-if [[ "$BRANCH" == "master" || $(echo "$BRANCH" | grep "release-") != "" ]]; then
-	echo ""
-	echo "* branch $BRANCH; testing libs"
-	# checkout every github.com/tendermint dir and run its tests
-	bash test/test_libs.sh
-fi

test/test.sh

@@ -1,50 +0,0 @@
-#! /bin/bash
-set -eu
-
-# Get the directory of where this script is.
-SOURCE="${BASH_SOURCE[0]}"
-while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done
-DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
-
-LOGS_DIR="$DIR/logs"
-echo
-echo "* [$(date +"%T")] cleaning up $LOGS_DIR"
-rm -rf "$LOGS_DIR"
-mkdir -p "$LOGS_DIR"
-
-set +e
-echo
-echo "* [$(date +"%T")] removing run_test container"
-docker rm -vf run_test
-set -e
-
-echo
-echo "* [$(date +"%T")] starting rsyslog container"
-docker rm -f rsyslog || true
-docker run -d -v "$LOGS_DIR:/var/log/" -p 127.0.0.1:5514:514/udp --name rsyslog voxxit/rsyslog
-
-set +u
-if [[ "$SKIP_BUILD" == "" ]]; then
-	echo
-	echo "* [$(date +"%T")] building docker image"
-	bash "$DIR/docker/build.sh"
-fi
-
-echo
-echo "* [$(date +"%T")] running go tests and app tests in docker container"
-# sometimes its helpful to mount the local test folder
-# -v $DIR:/go/src/github.com/tendermint/tendermint/test
-if [[ "$CIRCLECI" == true ]]; then
-	docker run --name run_test -e CIRCLECI=true -t tester bash test/run_test.sh
-else
-	docker run --name run_test -t tester bash test/run_test.sh
-fi
-
-# copy the coverage results out of docker container
-docker cp run_test:/go/src/github.com/tendermint/tendermint/coverage.txt .
-
-# test basic network connectivity
-# by starting a local testnet and checking peers connect and make blocks
-echo
-echo "* [$(date +"%T")] running p2p tests on a local docker network"
-bash "$DIR/p2p/test.sh" tester

test/test_cover.sh

@@ -3,6 +3,7 @@
 PKGS=$(go list github.com/tendermint/tendermint/... | grep -v /vendor/)
 
 set -e
+
 echo "mode: atomic" > coverage.txt
 for pkg in ${PKGS[@]}; do
 	go test -v -timeout 30m -race -coverprofile=profile.out -covermode=atomic "$pkg"