diff --git a/.circleci/config.yml b/.circleci/config.yml index be2d9690b..babba409d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,136 +1,126 @@ -version: 2 +version: 2.1 -defaults: &defaults - working_directory: /go/src/github.com/tendermint/tendermint - docker: - - image: circleci/golang - environment: - GOBIN: /tmp/workspace/bin - -docs_update_config: &docs_update_config - working_directory: ~/repo - docker: - - image: tendermintdev/jq_curl - environment: - AWS_REGION: us-east-1 +executors: + golang: + docker: + - image: tendermintdev/docker-tendermint-build + working_directory: /go/src/github.com/tendermint/tendermint + environment: + GOBIN: /tmp/bin + release: + machine: true + docs: + docker: + - image: tendermintdev/jq_curl + environment: + AWS_REGION: us-east-1 -release_management_docker: &release_management_docker - machine: true +commands: + run_test: + parameters: + script_path: + type: string + steps: + - attach_workspace: + at: /tmp/bin + - restore_cache: + name: "Restore source code cache" + keys: + - go-src-v1-{{ .Revision }} + - checkout + - restore_cache: + name: "Restore go modules cache" + keys: + - go-mod-v1-{{ checksum "go.sum" }} + - run: + name: "Running test" + command: | + bash << parameters.script_path >> jobs: setup_dependencies: - <<: *defaults + executor: golang steps: - - run: mkdir -p /tmp/workspace/bin - - run: mkdir -p /tmp/workspace/profiles - checkout - restore_cache: + name: "Restore go modules cache" keys: - - v4-pkg-cache + - go-mod-v1-{{ checksum "go.sum" }} - run: - name: tools command: | - export PATH="$GOBIN:$PATH" - make get_tools + mkdir -p /tmp/bin - run: - name: binaries - command: | - export PATH="$GOBIN:$PATH" - make install install_abci - - persist_to_workspace: - root: /tmp/workspace - paths: - - bin - - profiles + name: Cache go modules + command: make go-mod-cache + - run: + name: tools + command: make tools + - run: + name: "Build binaries" + command: make install install_abci - save_cache: - key: v4-pkg-cache + name: "Save go modules cache" + key: go-mod-v1-{{ checksum "go.sum" }} paths: - - /go/pkg + - "/go/pkg/mod" - save_cache: - key: v3-tree-{{ .Environment.CIRCLE_SHA1 }} + name: "Save source code cache" + key: go-src-v1-{{ .Revision }} paths: - - /go/src/github.com/tendermint/tendermint - - build_slate: - <<: *defaults - steps: - - attach_workspace: - at: /tmp/workspace - - restore_cache: - key: v4-pkg-cache - - restore_cache: - key: v3-tree-{{ .Environment.CIRCLE_SHA1 }} - - run: - name: slate docs - command: | - set -ex - export PATH="$GOBIN:$PATH" - make build-slate + - ".git" + - persist_to_workspace: + root: "/tmp/bin" + paths: + - "." test_abci_apps: - <<: *defaults + executor: golang steps: - - attach_workspace: - at: /tmp/workspace - - restore_cache: - key: v4-pkg-cache - - restore_cache: - key: v3-tree-{{ .Environment.CIRCLE_SHA1 }} - - run: - name: Run abci apps tests - command: | - export PATH="$GOBIN:$PATH" - bash abci/tests/test_app/test.sh + - run_test: + script_path: abci/tests/test_app/test.sh # if this test fails, fix it and update the docs at: # https://github.com/tendermint/tendermint/blob/develop/docs/abci-cli.md test_abci_cli: - <<: *defaults + executor: golang steps: - - attach_workspace: - at: /tmp/workspace - - restore_cache: - key: v4-pkg-cache - - restore_cache: - key: v3-tree-{{ .Environment.CIRCLE_SHA1 }} - - run: - name: Run abci-cli tests - command: | - export PATH="$GOBIN:$PATH" - bash abci/tests/test_cli/test.sh + - run_test: + script_path: abci/tests/test_cli/test.sh test_apps: - <<: *defaults + executor: golang steps: - - attach_workspace: - at: /tmp/workspace - - restore_cache: - key: v4-pkg-cache - - restore_cache: - key: v3-tree-{{ .Environment.CIRCLE_SHA1 }} - - run: sudo apt-get update && sudo apt-get install -y --no-install-recommends bsdmainutils - - run: - name: Run tests - command: bash test/app/test.sh + - run_test: + script_path: test/app/test.sh + + test_persistence: + executor: golang + steps: + - run_test: + script_path: test/persist/test_failure_indices.sh test_cover: - <<: *defaults + executor: golang parallelism: 4 steps: - - attach_workspace: - at: /tmp/workspace - restore_cache: - key: v4-pkg-cache + name: "Restore source code cache" + keys: + - go-src-v1-{{ .Revision }} + - checkout - restore_cache: - key: v3-tree-{{ .Environment.CIRCLE_SHA1 }} - - run: mkdir -p /tmp/logs + name: "Restore go module cache" + keys: + - go-mod-v2-{{ checksum "go.sum" }} - run: - name: Run tests + name: "Run tests" command: | + export VERSION="$(git describe --tags --long | sed 's/v\(.*\)/\1/')" + export GO111MODULE=on + mkdir -p /tmp/logs /tmp/workspace/profiles for pkg in $(go list github.com/tendermint/tendermint/... | circleci tests split --split-by=timings); do id=$(basename "$pkg") - - GO111MODULE=on go test -v -timeout 5m -mod=readonly -race -coverprofile=/tmp/workspace/profiles/$id.out -covermode=atomic "$pkg" | tee "/tmp/logs/$id-$RANDOM.log" + go test -v -timeout 5m -mod=readonly -race -coverprofile=/tmp/workspace/profiles/$id.out -covermode=atomic "$pkg" | tee "/tmp/logs/$id-$RANDOM.log" done - persist_to_workspace: root: /tmp/workspace @@ -139,19 +129,6 @@ jobs: - store_artifacts: path: /tmp/logs - test_persistence: - <<: *defaults - steps: - - attach_workspace: - at: /tmp/workspace - - restore_cache: - key: v4-pkg-cache - - restore_cache: - key: v3-tree-{{ .Environment.CIRCLE_SHA1 }} - - run: - name: Run tests - command: bash test/persist/test_failure_indices.sh - localnet: working_directory: /home/circleci/.go_workspace/src/github.com/tendermint/tendermint machine: @@ -187,19 +164,22 @@ jobs: path: /home/circleci/project/test/p2p/logs upload_coverage: - <<: *defaults + executor: golang steps: - attach_workspace: at: /tmp/workspace - restore_cache: - key: v4-pkg-cache + name: "Restore source code cache" + keys: + - go-src-v1-{{ .Revision }} + - checkout - restore_cache: - key: v3-tree-{{ .Environment.CIRCLE_SHA1 }} + name: "Restore go module cache" + keys: + - go-mod-v2-{{ checksum "go.sum" }} - run: name: gather command: | - set -ex - echo "mode: atomic" > coverage.txt for prof in $(ls /tmp/workspace/profiles/); do tail -n +2 /tmp/workspace/profiles/"$prof" >> coverage.txt @@ -209,8 +189,12 @@ jobs: command: bash .circleci/codecov.sh -f coverage.txt deploy_docs: - <<: *docs_update_config + executor: docs steps: + - restore_cache: + name: "Restore source code cache" + keys: + - go-src-v1-{{ .Revision }} - checkout - run: name: Trigger website build @@ -233,8 +217,12 @@ jobs: fi prepare_build: - <<: *defaults + executor: golang steps: + - restore_cache: + name: "Restore source code cache" + keys: + - go-src-v1-{{ .Revision }} - checkout - run: name: Get next release number @@ -250,8 +238,7 @@ jobs: echo "export CIRCLE_TAG=\"${NEXT_TAG}\"" > release-version.source - run: name: Build dependencies - command: | - make get_tools + command: make tools - persist_to_workspace: root: . paths: @@ -262,11 +249,16 @@ jobs: - "/go/pkg/mod" build_artifacts: - <<: *defaults + executor: golang parallelism: 4 steps: + - restore_cache: + name: "Restore source code cache" + keys: + - go-src-v1-{{ .Revision }} - checkout - restore_cache: + name: "Restore release dependencies cache" keys: - v2-release-deps-{{ checksum "go.sum" }} - attach_workspace: @@ -287,13 +279,17 @@ jobs: - "tendermint_linux_amd64" release_artifacts: - <<: *defaults + executor: golang steps: + - restore_cache: + name: "Restore source code cache" + keys: + - go-src-v1-{{ .Revision }} - checkout - attach_workspace: at: /tmp/workspace - run: - name: Deploy to GitHub + name: "Deploy to GitHub" command: | # Setting CIRCLE_TAG because we do not tag the release ourselves. source /tmp/workspace/release-version.source @@ -315,27 +311,36 @@ jobs: python -u scripts/release_management/github-publish.py --id "${RELEASE_ID}" release_docker: - <<: *release_management_docker + machine: + image: ubuntu-1604:201903-01 steps: + - restore_cache: + name: "Restore source code cache" + keys: + - go-src-v1-{{ .Revision }} - checkout - attach_workspace: at: /tmp/workspace - run: - name: Deploy to Docker Hub + name: "Deploy to Docker Hub" command: | # Setting CIRCLE_TAG because we do not tag the release ourselves. source /tmp/workspace/release-version.source cp /tmp/workspace/tendermint_linux_amd64 DOCKER/tendermint docker build --label="tendermint" --tag="tendermint/tendermint:${CIRCLE_TAG}" --tag="tendermint/tendermint:latest" "DOCKER" - docker login -u "${DOCKERHUB_USER}" --password-stdin <<< "${DOCKERHUB_PASS}" + docker login -u "${DOCKERHUB_USER}" --password-stdin \<<< "${DOCKERHUB_PASS}" docker push "tendermint/tendermint" docker logout reproducible_builds: - <<: *defaults + executor: golang steps: - attach_workspace: at: /tmp/workspace + - restore_cache: + name: "Restore source code cache" + keys: + - go-src-v1-{{ .Revision }} - checkout - setup_remote_docker: docker_layer_caching: true diff --git a/Makefile b/Makefile index 03a88a853..6870258f6 100644 --- a/Makefile +++ b/Makefile @@ -7,8 +7,6 @@ GOBIN?=${GOPATH}/bin PACKAGES=$(shell go list ./...) OUTPUT?=build/tendermint -export GO111MODULE = on - INCLUDE = -I=. -I=${GOPATH}/src -I=${GOPATH}/src/github.com/gogo/protobuf/protobuf BUILD_TAGS?='tendermint' LD_FLAGS = -X github.com/tendermint/tendermint/version.GitCommit=`git rev-parse --short=8 HEAD` -s -w @@ -16,7 +14,9 @@ BUILD_FLAGS = -mod=readonly -ldflags "$(LD_FLAGS)" all: check build test install -check: check_tools +# The below include contains the tools. +include scripts/devtools/Makefile +include tests.mk ######################################## ### Build Tendermint @@ -27,13 +27,6 @@ build: build_c: CGO_ENABLED=1 go build $(BUILD_FLAGS) -tags "$(BUILD_TAGS) cleveldb" -o $(OUTPUT) ./cmd/tendermint/ -# Runs `make build_c` from within an Amazon Linux (v2)-based Docker build -# container in order to build an Amazon Linux-compatible binary. Produces a -# compatible binary at ./build/tendermint -build_c-amazonlinux: - $(MAKE) -C ./DOCKER build_amazonlinux_buildimage - docker run --rm -it -v `pwd`:/tendermint tendermint/tendermint:build_c-amazonlinux - build_race: CGO_ENABLED=1 go build -race $(BUILD_FLAGS) -tags $(BUILD_TAGS) -o $(OUTPUT) ./cmd/tendermint @@ -78,22 +71,6 @@ install_abci: dist: @BUILD_TAGS=$(BUILD_TAGS) sh -c "'$(CURDIR)/scripts/dist.sh'" -######################################## -### Tools & dependencies - -check_tools: - @# https://stackoverflow.com/a/25668869 - @echo "Found tools: $(foreach tool,$(notdir $(GOTOOLS)),\ - $(if $(shell which $(tool)),$(tool),$(error "No $(tool) in PATH")))" - -get_tools: - @echo "--> Installing tools" - ./scripts/get_tools.sh - -update_tools: - @echo "--> Updating tools" - ./scripts/get_tools.sh - #For ABCI and libs get_protoc: @# https://github.com/google/protobuf/releases @@ -107,6 +84,16 @@ get_protoc: cd .. && \ rm -rf protobuf-3.6.1 +go-mod-cache: go.sum + @echo "--> Download go modules to local cache" + @go mod download +.PHONY: go-mod-cache + +go.sum: go.mod + @echo "--> Ensure dependencies have not been modified" + @go mod verify + @go mod tidy + draw_deps: @# requires brew install graphviz or apt-get install graphviz go get github.com/RobotsAndPencils/goviz @@ -153,100 +140,6 @@ protoc_grpc: rpc/grpc/types.pb.go protoc_merkle: crypto/merkle/merkle.pb.go -######################################## -### Testing - -## required to be run first by most tests -build_docker_test_image: - docker build -t tester -f ./test/docker/Dockerfile . - -### coverage, app, persistence, and libs tests -test_cover: - # run the go unit tests with coverage - bash test/test_cover.sh - -test_apps: - # run the app tests using bash - # requires `abci-cli` and `tendermint` binaries installed - bash test/app/test.sh - -test_abci_apps: - bash abci/tests/test_app/test.sh - -test_abci_cli: - # test the cli against the examples in the tutorial at: - # ./docs/abci-cli.md - # if test fails, update the docs ^ - @ bash abci/tests/test_cli/test.sh - -test_persistence: - # run the persistence tests using bash - # requires `abci-cli` installed - docker run --name run_persistence -t tester bash test/persist/test_failure_indices.sh - - # TODO undockerize - # bash test/persist/test_failure_indices.sh - -test_p2p: - docker rm -f rsyslog || true - rm -rf test/logs || true - mkdir test/logs - cd test/ - docker run -d -v "logs:/var/log/" -p 127.0.0.1:5514:514/udp --name rsyslog voxxit/rsyslog - cd .. - # requires 'tester' the image from above - bash test/p2p/test.sh tester - # the `docker cp` takes a really long time; uncomment for debugging - # - # mkdir -p test/p2p/logs && docker cp rsyslog:/var/log test/p2p/logs - -test_integrations: - make build_docker_test_image - make get_tools - make install - make test_cover - make test_apps - make test_abci_apps - make test_abci_cli - make test_libs - make test_persistence - make test_p2p - -test_release: - @go test -tags release $(PACKAGES) - -test100: - @for i in {1..100}; do make test; done - -vagrant_test: - vagrant up - vagrant ssh -c 'make test_integrations' - -### go tests -test: - @echo "--> Running go test" - @go test -p 1 $(PACKAGES) - -test_race: - @echo "--> Running go test --race" - @go test -p 1 -v -race $(PACKAGES) - -# uses https://github.com/sasha-s/go-deadlock/ to detect potential deadlocks -test_with_deadlock: - make set_with_deadlock - make test - make cleanup_after_test_with_deadlock - -set_with_deadlock: - find . -name "*.go" | grep -v "vendor/" | xargs -n 1 sed -i.bak 's/sync.RWMutex/deadlock.RWMutex/' - find . -name "*.go" | grep -v "vendor/" | xargs -n 1 sed -i.bak 's/sync.Mutex/deadlock.Mutex/' - find . -name "*.go" | grep -v "vendor/" | xargs -n 1 goimports -w - -# cleanes up after you ran test_with_deadlock -cleanup_after_test_with_deadlock: - find . -name "*.go" | grep -v "vendor/" | xargs -n 1 sed -i.bak 's/deadlock.RWMutex/sync.RWMutex/' - find . -name "*.go" | grep -v "vendor/" | xargs -n 1 sed -i.bak 's/deadlock.Mutex/sync.Mutex/' - find . -name "*.go" | grep -v "vendor/" | xargs -n 1 goimports -w ######################################## ### Formatting, linting, and vetting @@ -276,12 +169,19 @@ build-docker: ### Local testnet using docker # Build linux binary on other platforms -build-linux: get_tools +build-linux: tools GOOS=linux GOARCH=amd64 $(MAKE) build build-docker-localnode: @cd networks/local && make +# Runs `make build_c` from within an Amazon Linux (v2)-based Docker build +# container in order to build an Amazon Linux-compatible binary. Produces a +# compatible binary at ./build/tendermint +build_c-amazonlinux: + $(MAKE) -C ./DOCKER build_amazonlinux_buildimage + docker run --rm -it -v `pwd`:/tendermint tendermint/tendermint:build_c-amazonlinux + # Run a 4-node testnet locally localnet-start: localnet-stop build-docker-localnode @if ! [ -f build/node0/config/genesis.json ]; then docker run --rm -v $(CURDIR)/build:/tendermint:Z tendermint/localnode testnet --config /etc/tendermint/config-template.toml --v 4 --o . --populate-persistent-peers --starting-ip-address 192.167.10.2; fi @@ -311,10 +211,6 @@ sentry-stop: @if [ -z "$(DO_API_TOKEN)" ]; then echo "DO_API_TOKEN environment variable not set." ; false ; fi cd networks/remote/terraform && terraform destroy -var DO_API_TOKEN="$(DO_API_TOKEN)" -var SSH_KEY_FILE="$(HOME)/.ssh/id_rsa.pub" -# meant for the CI, inspect script & adapt accordingly -build-slate: - bash scripts/slate.sh - # Build hooks for dredd, to skip or add information on some steps build-contract-tests-hooks: ifeq ($(OS),Windows_NT) @@ -334,4 +230,8 @@ contract-tests: # To avoid unintended conflicts with file names, always add to .PHONY # unless there is a reason not to. # https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html -.PHONY: check build build_race build_abci dist install install_abci check_tools get_tools update_tools draw_deps get_protoc protoc_abci protoc_libs gen_certs clean_certs grpc_dbserver test_cover test_apps test_persistence test_p2p test test_race test_integrations test_release test100 vagrant_test fmt rpc-docs build-linux localnet-start localnet-stop build-docker build-docker-localnode sentry-start sentry-config sentry-stop build-slate protoc_grpc protoc_all build_c install_c test_with_deadlock cleanup_after_test_with_deadlock lint build-contract-tests-hooks contract-tests build_c-amazonlinux +.PHONY: check build build_race build_abci dist install install_abci check_tools tools update_tools draw_deps \ + get_protoc protoc_abci protoc_libs gen_certs clean_certs grpc_dbserver fmt rpc-docs build-linux localnet-start \ + localnet-stop build-docker build-docker-localnode sentry-start sentry-config sentry-stop protoc_grpc protoc_all \ + build_c install_c test_with_deadlock cleanup_after_test_with_deadlock lint build-contract-tests-hooks contract-tests \ + build_c-amazonlinux diff --git a/abci/tests/test_app/test.sh b/abci/tests/test_app/test.sh index c0bdace27..0d8301831 100755 --- a/abci/tests/test_app/test.sh +++ b/abci/tests/test_app/test.sh @@ -3,9 +3,8 @@ set -e # These tests spawn the counter app and server by execing the ABCI_APP command and run some simple client tests against it -export GO111MODULE=on - # Get the directory of where this script is. +export PATH="$GOBIN:$PATH" SOURCE="${BASH_SOURCE[0]}" while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" diff --git a/abci/tests/test_cli/test.sh b/abci/tests/test_cli/test.sh index ce074f513..cc880603d 100755 --- a/abci/tests/test_cli/test.sh +++ b/abci/tests/test_cli/test.sh @@ -2,6 +2,7 @@ set -e # Get the root directory. +export PATH="$GOBIN:$PATH" SOURCE="${BASH_SOURCE[0]}" while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done DIR="$( cd -P "$( dirname "$SOURCE" )/../.." && pwd )" diff --git a/scripts/devtools/Makefile b/scripts/devtools/Makefile new file mode 100644 index 000000000..1ca24bf9d --- /dev/null +++ b/scripts/devtools/Makefile @@ -0,0 +1,92 @@ +### +# Find OS and Go environment +# GO contains the Go binary +# FS contains the OS file separator +### +ifeq ($(OS),Windows_NT) + GO := $(shell where go.exe 2> NUL) + FS := "\\" +else + GO := $(shell command -v go 2> /dev/null) + FS := "/" +endif + +ifeq ($(GO),) + $(error could not find go. Is it in PATH? $(GO)) +endif + +GOPATH ?= $(shell $(GO) env GOPATH) +GITHUBDIR := $(GOPATH)$(FS)src$(FS)github.com + +### +# Functions +### + +go_get = $(if $(findstring Windows_NT,$(OS)),\ +IF NOT EXIST $(GITHUBDIR)$(FS)$(1)$(FS) ( mkdir $(GITHUBDIR)$(FS)$(1) ) else (cd .) &\ +IF NOT EXIST $(GITHUBDIR)$(FS)$(1)$(FS)$(2)$(FS) ( cd $(GITHUBDIR)$(FS)$(1) && git clone https://github.com/$(1)/$(2) ) else (cd .) &\ +,\ +mkdir -p $(GITHUBDIR)$(FS)$(1) &&\ +(test ! -d $(GITHUBDIR)$(FS)$(1)$(FS)$(2) && cd $(GITHUBDIR)$(FS)$(1) && git clone https://github.com/$(1)/$(2)) || true &&\ +)\ +cd $(GITHUBDIR)$(FS)$(1)$(FS)$(2) && git fetch origin && git checkout -q $(3) + +mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST))) +mkfile_dir := $(shell cd $(shell dirname $(mkfile_path)); pwd) + +### +# tools +### + +TOOLS_DESTDIR ?= $(GOPATH)/bin + +GOIMPORTS = $(TOOLS_DESTDIR)/goimports +CERTSTRAP = $(TOOLS_DESTDIR)/certstrap +PROTOBUF = $(TOOLS_DESTDIR)/protoc +GOX = $(TOOLS_DESTDIR)/gox +GOODMAN = $(TOOLS_DESTDIR)/goodman + +all: tools + +tools: goimports certstrap protobuf gox goodman + +check: check_tools + +check_tools: + @# https://stackoverflow.com/a/25668869 + @echo "Found tools: $(foreach tool,$(notdir $(GOTOOLS)),\ + $(if $(shell which $(tool)),$(tool),$(error "No $(tool) in PATH")))" + +goimports: $(GOIMPORTS) +$(GOIMPORTS): + @echo "Get goimports@v0.0.0-20190628034336-212fb13d595e" + @go get golang.org/x/tools/cmd/goimports@v0.0.0-20190628034336-212fb13d595e + +certstrap: $(CERTSTRAP) +$(CERTSTRAP): + @echo "Get Certstrap" + @go get github.com/square/certstrap@338204a88c4349b1c135eac1e8c14c693ad007da + +protobuf: $(PROTOBUF) +$(PROTOBUF): + @echo "Get Protobuf" + ## protobuf v1.3.0 + @go get github.com/gogo/protobuf/protoc-gen-gogo@0ca988a254f991240804bf9821f3450d87ccbb1b + +gox: $(GOX) +$(GOX): + @echo "Get Gox" +# used to build tm-monitor & tm-bench binaries + ## gox v1.0.1 + @go get github.com/mitchellh/gox@d8caaff5a9dc98f4cfa1fcce6e7265a04689f641 + +goodman: $(GOODMAN) +$(GOODMAN): + @echo "Get Goodman" + @go get github.com/snikch/goodman/cmd/goodman@10e37e294daa3c9a90abded60ff9924bafab3888 + +tools-clean: + rm -f $(CERTSTRAP) $(GOIMPORTS) $(PROTOBUF) $(GOX) $(GOODMAN) + rm -f tools-stamp + +.PHONY: all tools tools-clean diff --git a/scripts/get_tools.sh b/scripts/get_tools.sh deleted file mode 100755 index 2adc782b8..000000000 --- a/scripts/get_tools.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env bash -set -e - -# This file downloads all of the binary dependencies we have, and checks out a -# specific git hash. -# -# repos it installs: -# github.com/golang/dep/cmd/dep -# github.com/gogo/protobuf/protoc-gen-gogo -# github.com/square/certstrap -# github.com/mitchellh/gox -# github.com/golangci/golangci-lint -# github.com/petermattis/goid -# github.com/sasha-s/go-deadlock -# goimports - -## check if GOPATH is set -if [ -z ${GOPATH+x} ]; then - echo "please set GOPATH (https://github.com/golang/go/wiki/SettingGOPATH)" - exit 1 -fi - -mkdir -p "$GOPATH/src/github.com" -cd "$GOPATH/src/github.com" || exit 1 - -installFromGithub() { - repo=$1 - commit=$2 - # optional - subdir=$3 - echo "--> Installing $repo ($commit)..." - if [ ! -d "$repo" ]; then - mkdir -p "$repo" - git clone "https://github.com/$repo.git" "$repo" - fi - if [ ! -z ${subdir+x} ] && [ ! -d "$repo/$subdir" ]; then - echo "ERROR: no such directory $repo/$subdir" - exit 1 - fi - pushd "$repo" && \ - git fetch origin && \ - git checkout -q "$commit" && \ - if [ ! -z ${subdir+x} ]; then cd "$subdir" || exit 1; fi && \ - go install && \ - if [ ! -z ${subdir+x} ]; then cd - || exit 1; fi && \ - popd || exit 1 - echo "--> Done" - echo "" -} - -######################## DEVELOPER TOOLS ##################################### -## protobuf v1.3.0 -installFromGithub gogo/protobuf 0ca988a254f991240804bf9821f3450d87ccbb1b protoc-gen-gogo - -installFromGithub square/certstrap 338204a88c4349b1c135eac1e8c14c693ad007da - -# used to build tm-monitor & tm-bench binaries -## gox v1.0.1 -installFromGithub mitchellh/gox d8caaff5a9dc98f4cfa1fcce6e7265a04689f641 - -## Trying to install golangci with Go 1.13 gives: -## go: github.com/go-critic/go-critic@v0.0.0-20181204210945-1df300866540: invalid pseudo-version: does not match version-control timestamp (2019-05-26T07:48:19Z) -## golangci-lint v1.17.1 -# installFromGithub golangci/golangci-lint 4ba2155996359eabd8800d1fbf3e3a9777c80490 cmd/golangci-lint - -## Trying to install golangci with Go 1.13 gives: -## go: cannot find main module, but found .git/config in /go/src/github.com/petermattis/goid -## make test_with_deadlock -## XXX: https://github.com/tendermint/tendermint/issues/3242 -# installFromGithub petermattis/goid b0b1615b78e5ee59739545bb38426383b2cda4c9 -# installFromGithub sasha-s/go-deadlock d68e2bc52ae3291765881b9056f2c1527f245f1e -# go get golang.org/x/tools/cmd/goimports -# installFromGithub snikch/goodman 10e37e294daa3c9a90abded60ff9924bafab3888 cmd/goodman diff --git a/scripts/install-golangci-lint.sh b/scripts/install-golangci-lint.sh new file mode 100644 index 000000000..b95713828 --- /dev/null +++ b/scripts/install-golangci-lint.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -euo pipefail + +f_sha256() { + local l_file + l_file=$1 + python -sBc "import hashlib;print(hashlib.sha256(open('$l_file','rb').read()).hexdigest())" +} + +installer="$(mktemp)" +trap "rm -f ${installer}" EXIT + +GOBIN="${1}" +VERSION="${2}" +HASHSUM="${3}" +CURL="$(which curl)" + +echo "Downloading golangci-lint ${VERSION} installer ..." >&2 +"${CURL}" -sfL "https://raw.githubusercontent.com/golangci/golangci-lint/${VERSION}/install.sh" > "${installer}" + +echo "Checking hashsum ..." >&2 +[ "${HASHSUM}" = "$(f_sha256 ${installer})" ] +chmod +x "${installer}" + +echo "Launching installer ..." >&2 +exec "${installer}" -d -b "${GOBIN}" "${VERSION}" diff --git a/test/docker/Dockerfile b/test/docker/Dockerfile index b39277bd9..fb5458e82 100644 --- a/test/docker/Dockerfile +++ b/test/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.12 +FROM golang:1.13 # Add testing deps for curl RUN echo 'deb http://httpredir.debian.org/debian testing main non-free contrib' >> /etc/apt/sources.list @@ -19,7 +19,7 @@ COPY . $REPO # Install the vendored dependencies # docker caching prevents reinstall on code change! -RUN make get_tools +RUN make tools # install ABCI CLI RUN make install_abci diff --git a/tests.mk b/tests.mk new file mode 100644 index 000000000..18caef496 --- /dev/null +++ b/tests.mk @@ -0,0 +1,106 @@ +#!/usr/bin/make -f + +######################################## +### Testing + +BINDIR ?= $(GOPATH)/bin + +## required to be run first by most tests +build_docker_test_image: + docker build -t tester -f ./test/docker/Dockerfile . + +### coverage, app, persistence, and libs tests +test_cover: + # run the go unit tests with coverage + bash test/test_cover.sh + +test_apps: + # run the app tests using bash + # requires `abci-cli` and `tendermint` binaries installed + bash test/app/test.sh + +test_abci_apps: + bash abci/tests/test_app/test.sh + +test_abci_cli: + # test the cli against the examples in the tutorial at: + # ./docs/abci-cli.md + # if test fails, update the docs ^ + @ bash abci/tests/test_cli/test.sh + +test_persistence: + # run the persistence tests using bash + # requires `abci-cli` installed + docker run --name run_persistence -t tester bash test/persist/test_failure_indices.sh + + # TODO undockerize + # bash test/persist/test_failure_indices.sh + +test_p2p: + docker rm -f rsyslog || true + rm -rf test/logs || true + mkdir test/logs + cd test/ + docker run -d -v "logs:/var/log/" -p 127.0.0.1:5514:514/udp --name rsyslog voxxit/rsyslog + cd .. + # requires 'tester' the image from above + bash test/p2p/test.sh tester + # the `docker cp` takes a really long time; uncomment for debugging + # + # mkdir -p test/p2p/logs && docker cp rsyslog:/var/log test/p2p/logs + +test_integrations: + make build_docker_test_image + make tools + make install + make test_cover + make test_apps + make test_abci_apps + make test_abci_cli + make test_libs + make test_persistence + make test_p2p + +test_release: + @go test -tags release $(PACKAGES) + +test100: + @for i in {1..100}; do make test; done + +vagrant_test: + vagrant up + vagrant ssh -c 'make test_integrations' + +### go tests +test: + @echo "--> Running go test" + @go test -p 1 $(PACKAGES) + +test_race: + @echo "--> Running go test --race" + @go test -p 1 -v -race $(PACKAGES) + +# uses https://github.com/sasha-s/go-deadlock/ to detect potential deadlocks +test_with_deadlock: + make set_with_deadlock + make test + make cleanup_after_test_with_deadlock + +set_with_deadlock: + @echo "Get Goid" + @go get github.com/petermattis/goid@b0b1615b78e5ee59739545bb38426383b2cda4c9 + @echo "Get Go-Deadlock" + @go get github.com/sasha-s/go-deadlock@d68e2bc52ae3291765881b9056f2c1527f245f1e + find . -name "*.go" | grep -v "vendor/" | xargs -n 1 sed -i.bak 's/sync.RWMutex/deadlock.RWMutex/' + find . -name "*.go" | grep -v "vendor/" | xargs -n 1 sed -i.bak 's/sync.Mutex/deadlock.Mutex/' + find . -name "*.go" | grep -v "vendor/" | xargs -n 1 goimports -w + +# cleanes up after you ran test_with_deadlock +cleanup_after_test_with_deadlock: + find . -name "*.go" | grep -v "vendor/" | xargs -n 1 sed -i.bak 's/deadlock.RWMutex/sync.RWMutex/' + find . -name "*.go" | grep -v "vendor/" | xargs -n 1 sed -i.bak 's/deadlock.Mutex/sync.Mutex/' + find . -name "*.go" | grep -v "vendor/" | xargs -n 1 goimports -w + # cleans up the deps to not include the need libs + go mod tidy + +.PHONY: test_cover test_apps test_persistence test_p2p test test_race test_integrations test_release test100 vagrant_test