Merge pull request #1283 from tendermint/feature/xla-run-integration-release

Speed up CircleCI builds

.circleci/config.yml

@@ -0,0 +1,222 @@
+version: 2
+
+defaults: &defaults
+  working_directory: /go/src/github.com/tendermint/tendermint
+  docker:
+    - image: circleci/golang:1.10.0
+  environment:
+    GOBIN: /tmp/workspace/bin
+
+jobs:
+  setup_dependencies:
+    <<: *defaults
+    steps:
+      - run: mkdir -p /tmp/workspace/bin
+      - run: mkdir -p /tmp/workspace/profiles
+      - checkout
+      - restore_cache:
+          keys:
+            - v1-pkg-cache
+      - run:
+          name: tools
+          command: |
+            export PATH="$GOBIN:$PATH"
+            make get_tools
+      - run:
+          name: dependencies
+          command: |
+            export PATH="$GOBIN:$PATH"
+            make get_vendor_deps
+      - run:
+          name: binaries
+          command: |
+            export PATH="$GOBIN:$PATH"
+            make install
+      - persist_to_workspace:
+          root: /tmp/workspace
+          paths:
+            - bin
+            - profiles
+      - save_cache:
+          key: v1-pkg-cache
+          paths:
+            - /go/pkg
+      - save_cache:
+          key: v1-tree-{{ .Environment.CIRCLE_SHA1 }}
+          paths:
+            - /go/src/github.com/tendermint/tendermint
+
+  setup_abci:
+    <<: *defaults
+    steps:
+      - attach_workspace:
+          at: /tmp/workspace
+      - restore_cache:
+          key: v1-pkg-cache
+      - restore_cache:
+          key: v1-tree-{{ .Environment.CIRCLE_SHA1 }}
+      - run:
+          name: Checkout abci
+          command: |
+            commit=$(bash scripts/dep_utils/parse.sh abci)
+            go get -v -u -d github.com/tendermint/abci/...
+            cd /go/src/github.com/tendermint/abci
+            git checkout "$commit"
+      - run:
+          working_directory: /go/src/github.com/tendermint/abci
+          name: Install abci
+          command: |
+            set -ex
+            export PATH="$GOBIN:$PATH"
+            make get_tools
+            make get_vendor_deps
+            make install
+      - run: ls -lah /tmp/workspace/bin
+      - persist_to_workspace:
+          root: /tmp/workspace
+          paths:
+            - "bin/abci*"
+
+  lint:
+    <<: *defaults
+    steps:
+      - attach_workspace:
+          at: /tmp/workspace
+      - restore_cache:
+          key: v1-pkg-cache
+      - restore_cache:
+          key: v1-tree-{{ .Environment.CIRCLE_SHA1 }}
+      - run:
+          name: metalinter
+          command: |
+            set -ex
+            export PATH="$GOBIN:$PATH"
+            make metalinter
+
+  test_apps:
+    <<: *defaults
+    steps:
+      - attach_workspace:
+          at: /tmp/workspace
+      - restore_cache:
+          key: v1-pkg-cache
+      - restore_cache:
+          key: v1-tree-{{ .Environment.CIRCLE_SHA1 }}
+      - run: sudo apt-get update && sudo apt-get install -y --no-install-recommends bsdmainutils
+      - run:
+          name: Run tests
+          command: bash test/app/test.sh
+
+  test_cover:
+    <<: *defaults
+    parallelism: 4
+    steps:
+      - attach_workspace:
+          at: /tmp/workspace
+      - restore_cache:
+          key: v1-pkg-cache
+      - restore_cache:
+          key: v1-tree-{{ .Environment.CIRCLE_SHA1 }}
+      - run:
+          name: Run tests
+          command: |
+            for pkg in $(go list github.com/tendermint/tendermint/... | grep -v /vendor/ | circleci tests split --split-by=timings); do
+              id=$(basename "$pkg")
+
+              go test -timeout 5m -race -coverprofile=/tmp/workspace/profiles/$id.out -covermode=atomic "$pkg"
+            done
+      - persist_to_workspace:
+          root: /tmp/workspace
+          paths:
+            - "profiles/*"
+
+  test_libs:
+    <<: *defaults
+    steps:
+      - attach_workspace:
+          at: /tmp/workspace
+      - restore_cache:
+          key: v1-pkg-cache
+      - restore_cache:
+          key: v1-tree-{{ .Environment.CIRCLE_SHA1 }}
+      - run:
+          name: Run tests
+          command: bash test/test_libs.sh
+
+  test_persistence:
+    <<: *defaults
+    steps:
+      - attach_workspace:
+          at: /tmp/workspace
+      - restore_cache:
+          key: v1-pkg-cache
+      - restore_cache:
+          key: v1-tree-{{ .Environment.CIRCLE_SHA1 }}
+      - run:
+          name: Run tests
+          command: bash test/persist/test_failure_indices.sh
+
+  test_p2p:
+    environment:
+      GOBIN: /home/circleci/.go_workspace/bin
+      GOPATH: /home/circleci/.go_workspace
+    machine:
+      image: circleci/classic:latest
+    steps:
+      - checkout
+      - run: mkdir -p $GOPATH/src/github.com/tendermint
+      - run: ln -sf /home/circleci/project $GOPATH/src/github.com/tendermint/tendermint
+      - run: bash test/circleci/p2p.sh
+
+  upload_coverage:
+    <<: *defaults
+    steps:
+      - attach_workspace:
+          at: /tmp/workspace
+      - restore_cache:
+          key: v1-tree-{{ .Environment.CIRCLE_SHA1 }}
+      - run:
+          name: gather
+          command: |
+            set -ex
+
+            echo "mode: atomic" > coverage.txt
+            for prof in $(ls /tmp/workspace/profiles/); do
+              tail -n +2 /tmp/workspace/profiles/"$prof" >> coverage.txt
+            done
+      - run:
+          name: upload
+          command: bash <(curl -s https://codecov.io/bash) -f coverage.txt
+
+workflows:
+  version: 2
+  test-suite:
+    jobs:
+      - setup_dependencies
+      - setup_abci:
+          requires:
+            - setup_dependencies
+      - lint:
+          requires:
+            - setup_dependencies
+      - test_apps:
+          requires:
+            - setup_abci
+      - test_cover:
+          requires:
+            - setup_dependencies
+      - test_libs:
+          filters:
+            branches:
+              only:
+                - develop
+                - master
+          requires:
+            - setup_dependencies
+      - test_persistence:
+          requires:
+            - setup_abci
+      - test_p2p
+      - upload_coverage:
+          requires:
+            - test_cover

Makefile

@@ -1,6 +1,6 @@
 GOTOOLS = \
 	github.com/golang/dep/cmd/dep \
-	# gopkg.in/alecthomas/gometalinter.v2
+	gopkg.in/alecthomas/gometalinter.v2
 PACKAGES=$(shell go list ./... | grep -v '/vendor/')
 BUILD_TAGS?=tendermint
 BUILD_FLAGS = -ldflags "-X github.com/tendermint/tendermint/version.GitCommit=`git rev-parse --short=8 HEAD`"
@@ -40,7 +40,7 @@ check_tools:
 get_tools:
 	@echo "--> Installing tools"
 	go get -u -v $(GOTOOLS)
-	# @gometalinter.v2 --install
+	@gometalinter.v2 --install
 
 update_tools:
 	@echo "--> Updating tools"
@@ -79,7 +79,7 @@ test:
 
 test_race:
 	@echo "--> Running go test --race"
-	@go test -v -race $(PACKAGES)
+	@go test -race $(PACKAGES)
 
 test_integrations:
 	@bash ./test/test.sh
@@ -105,7 +105,7 @@ fmt:
 
 metalinter:
 	@echo "--> Running linter"
-	gometalinter.v2 --vendor --deadline=600s --disable-all  \
+	@gometalinter.v2 --vendor --deadline=600s --disable-all  \
 		--enable=deadcode \
 		--enable=gosimple \
 	 	--enable=misspell \

circle.yml

@@ -1,35 +0,0 @@
----
-machine:
-  environment:
-    MACH_PREFIX: tendermint-test-mach
-    DOCKER_VERSION: 1.10.0
-    DOCKER_MACHINE_VERSION: 0.9.0
-    GOPATH: "$HOME/.go_project"
-    PROJECT_PARENT_PATH: "$GOPATH/src/github.com/$CIRCLE_PROJECT_USERNAME"
-    PROJECT_PATH: "$PROJECT_PARENT_PATH/$CIRCLE_PROJECT_REPONAME"
-    PATH: "$HOME/.go_project/bin:${PATH}"
-  hosts:
-    localhost: 127.0.0.1
-
-dependencies:
-  override:
-    - curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | sudo bash -s -- $DOCKER_VERSION
-    - sudo start docker
-    - sudo curl -sSL -o /usr/bin/docker-machine "https://github.com/docker/machine/releases/download/v$DOCKER_MACHINE_VERSION/docker-machine-`uname -s`-`uname -m`"; sudo chmod 0755 /usr/bin/docker-machine
-    - mkdir -p "$PROJECT_PARENT_PATH"
-    - ln -sf "$HOME/$CIRCLE_PROJECT_REPONAME/" "$PROJECT_PATH"
-  post:
-    - go version
-    - docker version
-    - docker-machine version
-
-test:
-  override:
-    - cd "$PROJECT_PATH" && set -o pipefail && make test_integrations 2>&1 | tee test_integrations.log:
-        timeout: 1800
-  post:
-    - cd "$PROJECT_PATH" && mv test_integrations.log "${CIRCLE_ARTIFACTS}"
-    - cd "$PROJECT_PATH" && bash <(curl -s https://codecov.io/bash) -f coverage.txt
-    - cd "$PROJECT_PATH" && mv coverage.txt "${CIRCLE_ARTIFACTS}"
-    - cd "$PROJECT_PATH" && cp test/logs/messages "${CIRCLE_ARTIFACTS}/docker.log"
-    - cd "${CIRCLE_ARTIFACTS}" && tar czf logs.tar.gz *.log

p2p/switch_test.go

@@ -248,6 +248,7 @@ func TestSwitchStopsNonPersistentPeerOnError(t *testing.T) {
 	require.Nil(err)
 
 	peer := sw.Peers().Get(rp.ID())
+	require.NotNil(peer)
 
 	// simulate failure by closing connection
 	pc.CloseConn()
@@ -274,10 +275,11 @@ func TestSwitchReconnectsToPersistentPeer(t *testing.T) {
 	pc, err := newOutboundPeerConn(rp.Addr(), DefaultPeerConfig(), true, sw.nodeKey.PrivKey)
 	//	sw.reactorsByCh, sw.chDescs, sw.StopPeerForError, sw.nodeKey.PrivKey,
 	require.Nil(err)
-	err = sw.addPeer(pc)
-	require.Nil(err)
+
+	require.Nil(sw.addPeer(pc))
 
 	peer := sw.Peers().Get(rp.ID())
+	require.NotNil(peer)
 
 	// simulate failure by closing connection
 	pc.CloseConn()

scripts/dep_utils/checkout.sh

@@ -1,5 +1,7 @@
 #! /bin/bash
 
+set -ex
+
 set +u
 if [[ "$DEP" == "" ]]; then
 	DEP=$GOPATH/src/github.com/tendermint/tendermint/Gopkg.lock

test/app/test.sh

@@ -1,5 +1,5 @@
 #! /bin/bash
-set -e
+set -ex
 
 #- kvstore over socket, curl
 #- counter over socket, curl
@@ -8,6 +8,7 @@ set -e
 
 # TODO: install everything
 
+export PATH="$GOBIN:$PATH"
 export TMHOME=$HOME/.tendermint_app
 
 function kvstore_over_socket(){

test/circleci/p2p.sh

@@ -0,0 +1,35 @@
+#! /bin/bash
+set -eux
+
+# Get the directory of where this script is.
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done
+DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+LOGS_DIR="$DIR/../logs"
+echo
+echo "* [$(date +"%T")] cleaning up $LOGS_DIR"
+rm -rf "$LOGS_DIR"
+mkdir -p "$LOGS_DIR"
+
+set +e
+echo
+echo "* [$(date +"%T")] removing run_test container"
+docker rm -vf run_test
+set -e
+
+echo
+echo "* [$(date +"%T")] starting rsyslog container"
+docker rm -f rsyslog || true
+docker run -d -v "$LOGS_DIR:/var/log/" -p 127.0.0.1:5514:514/udp --name rsyslog voxxit/rsyslog
+
+set +u
+if [[ "$SKIP_BUILD" == "" ]]; then
+	echo
+	echo "* [$(date +"%T")] building docker image"
+	bash "$DIR/../docker/build.sh"
+fi
+
+echo
+echo "* [$(date +"%T")] running p2p tests on a local docker network"
+bash "$DIR/../p2p/test.sh" tester

test/docker/Dockerfile

@@ -10,6 +10,7 @@ RUN apt-get update && \
 
 # Setup tendermint repo
 ENV REPO $GOPATH/src/github.com/tendermint/tendermint
+ENV GOBIN $GOPATH/bin
 WORKDIR $REPO
 
 # Install the vendored dependencies before copying code

test/persist/test_failure_indices.sh

@@ -1,5 +1,6 @@
 #! /bin/bash
 
+export PATH="$GOBIN:$PATH"
 export TMHOME=$HOME/.tendermint_persist
 
 rm -rf "$TMHOME"

test/test_libs.sh

@@ -1,5 +1,7 @@
 #! /bin/bash
-set -e
+set -ex
+
+export PATH="$GOBIN:$PATH"
 
 # Get the parent directory of where this script is.
 SOURCE="${BASH_SOURCE[0]}"
@@ -18,6 +20,7 @@ for lib in "${LIBS[@]}"; do
 
 	echo "Testing $lib ..."
 	cd "$GOPATH/src/github.com/tendermint/$lib"
+	make get_tools
 	make get_vendor_deps
 	make test
 	if [[ "$?" != 0 ]]; then

types/priv_validator/socket.go

@@ -18,9 +18,8 @@ import (
 )
 
 const (
-	defaultConnDeadlineSeconds      = 3
-	defaultDialRetryIntervalSeconds = 1
-	defaultDialRetryMax             = 10
+	defaultConnDeadlineSeconds = 3
+	defaultDialRetryMax        = 10
 )
 
 // Socket errors.
@@ -56,7 +55,7 @@ type socketClient struct {
 // Check that socketClient implements PrivValidator2.
 var _ types.PrivValidator2 = (*socketClient)(nil)
 
-// NewsocketClient returns an instance of socketClient.
+// NewSocketClient returns an instance of socketClient.
 func NewSocketClient(
 	logger log.Logger,
 	socketAddr string,
@@ -448,7 +447,7 @@ func readMsg(r io.Reader) (PrivValidatorSocketMsg, error) {
 
 	w, ok := read.(struct{ PrivValidatorSocketMsg })
 	if !ok {
-		return nil, errors.New("unknwon type")
+		return nil, errors.New("unknown type")
 	}
 
 	return w.PrivValidatorSocketMsg, nil