From 6c8d7a8c19bc3cbea627aed3b81bf1b93a96d445 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 8 Jan 2018 16:41:23 -0600 Subject: [PATCH] deterministic tooling for releases get rid of gox build target builds inside docker, dev-build - locally Revert "build target builds inside docker, dev-build - locally" This reverts commit 8ba89d5e8c5668e3839ff49952a9166d1158f6e8. add build tags to make build/build_race/install use tendermint's fork of glide instead of tar.gz remove TMHOME unused var + set length for git hash get rid of GOTOOLS_CHECK fixes after review zip needed for distribution --- Makefile | 24 ++++++------ Vagrantfile | 2 +- scripts/dist.sh | 54 +++++++++++++++++++++------ scripts/dist_build.sh | 51 ------------------------- scripts/release.sh | 6 +-- scripts/tendermint-builder/Dockerfile | 12 ------ 6 files changed, 58 insertions(+), 91 deletions(-) delete mode 100755 scripts/dist_build.sh delete mode 100644 scripts/tendermint-builder/Dockerfile diff --git a/Makefile b/Makefile index 5fd599cce..c765d078f 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,9 @@ GOTOOLS = \ - github.com/mitchellh/gox \ - github.com/Masterminds/glide \ - github.com/tcnksm/ghr \ + github.com/tendermint/glide \ # gopkg.in/alecthomas/gometalinter.v2 -GOTOOLS_CHECK = gox glide ghr gometalinter.v2 PACKAGES=$(shell go list ./... | grep -v '/vendor/') BUILD_TAGS?=tendermint -TMHOME = $${TMHOME:-$$HOME/.tendermint} -BUILD_FLAGS = -ldflags "-X github.com/tendermint/tendermint/version.GitCommit=`git rev-parse --short HEAD`" +BUILD_FLAGS = -ldflags "-X github.com/tendermint/tendermint/version.GitCommit=`git rev-parse --short=8 HEAD`" all: check build test install @@ -18,25 +14,27 @@ check: check_tools get_vendor_deps ### Build build: - go build $(BUILD_FLAGS) -o build/tendermint ./cmd/tendermint/ + go build $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' -o build/tendermint ./cmd/tendermint/ build_race: - go build -race $(BUILD_FLAGS) -o build/tendermint ./cmd/tendermint + go build -race $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' -o build/tendermint ./cmd/tendermint + +install: + go install $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' ./cmd/tendermint + +######################################## +### Distribution # dist builds binaries for all platforms and packages them for distribution dist: @BUILD_TAGS='$(BUILD_TAGS)' sh -c "'$(CURDIR)/scripts/dist.sh'" -install: - go install $(BUILD_FLAGS) ./cmd/tendermint - - ######################################## ### Tools & dependencies check_tools: @# https://stackoverflow.com/a/25668869 - @echo "Found tools: $(foreach tool,$(GOTOOLS_CHECK),\ + @echo "Found tools: $(foreach tool,$(notdir $(GOTOOLS)),\ $(if $(shell which $(tool)),$(tool),$(error "No $(tool) in PATH")))" get_tools: diff --git a/Vagrantfile b/Vagrantfile index ee878649d..ac8da0cc1 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -21,7 +21,7 @@ Vagrant.configure("2") do |config| # install base requirements apt-get update - apt-get install -y --no-install-recommends wget curl jq \ + apt-get install -y --no-install-recommends wget curl jq zip \ make shellcheck bsdmainutils psmisc apt-get install -y docker-ce golang-1.9-go apt-get install -y language-pack-en diff --git a/scripts/dist.sh b/scripts/dist.sh index c144c7fdd..40aa71e98 100755 --- a/scripts/dist.sh +++ b/scripts/dist.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash set -e +# WARN: non hermetic build (people must run this script inside docker to +# produce deterministic binaries). + # Get the version from the environment, or try to figure it out. if [ -z $VERSION ]; then VERSION=$(awk -F\" '/Version =/ { print $2; exit }' < version/version.go) @@ -11,22 +14,51 @@ if [ -z "$VERSION" ]; then fi echo "==> Building version $VERSION..." -# Get the parent directory of where this script is. -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done -DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )" - -# Change into that dir because we expect that. -cd "$DIR" - # Delete the old dir echo "==> Removing old directory..." rm -rf build/pkg mkdir -p build/pkg -# Do a hermetic build inside a Docker container. -docker build -t tendermint/tendermint-builder scripts/tendermint-builder/ -docker run --rm -e "BUILD_TAGS=$BUILD_TAGS" -v "$(pwd)":/go/src/github.com/tendermint/tendermint tendermint/tendermint-builder ./scripts/dist_build.sh +# Get the git commit +GIT_COMMIT="$(git rev-parse --short=8 HEAD)" +GIT_IMPORT="github.com/tendermint/tendermint/version" + +# Determine the arch/os combos we're building for +XC_ARCH=${XC_ARCH:-"386 amd64 arm"} +XC_OS=${XC_OS:-"solaris darwin freebsd linux windows"} +XC_EXCLUDE=${XC_EXCLUDE:-" darwin/arm solaris/amd64 solaris/386 solaris/arm freebsd/amd64 windows/arm "} + +# Make sure build tools are available. +make get_tools + +# Get VENDORED dependencies +make get_vendor_deps + +# Build! +# ldflags: -s Omit the symbol table and debug information. +# -w Omit the DWARF symbol table. +echo "==> Building..." +IFS=' ' read -ra arch_list <<< "$XC_ARCH" +IFS=' ' read -ra os_list <<< "$XC_OS" +for arch in "${arch_list[@]}"; do + for os in "${os_list[@]}"; do + if [[ "$XC_EXCLUDE" != *" $os/$arch "* ]]; then + echo "--> $os/$arch" + GOOS=${os} GOARCH=${arch} go build -ldflags "-s -w -X ${GIT_IMPORT}.GitCommit=${GIT_COMMIT}" -tags="${BUILD_TAGS}" -o "build/pkg/${os}_${arch}/tendermint" ./cmd/tendermint + fi + done +done + +# Zip all the files. +echo "==> Packaging..." +for PLATFORM in $(find ./build/pkg -mindepth 1 -maxdepth 1 -type d); do + OSARCH=$(basename "${PLATFORM}") + echo "--> ${OSARCH}" + + pushd "$PLATFORM" >/dev/null 2>&1 + zip "../${OSARCH}.zip" ./* + popd >/dev/null 2>&1 +done # Add "tendermint" and $VERSION prefix to package name. rm -rf ./build/dist diff --git a/scripts/dist_build.sh b/scripts/dist_build.sh deleted file mode 100755 index 337fbacab..000000000 --- a/scripts/dist_build.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash -set -e - -# Get the parent directory of where this script is. -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done -DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )" - -# Change into that dir because we expect that. -cd "$DIR" - -# Get the git commit -GIT_COMMIT="$(git rev-parse --short HEAD)" -GIT_IMPORT="github.com/tendermint/tendermint/version" - -# Determine the arch/os combos we're building for -XC_ARCH=${XC_ARCH:-"386 amd64 arm"} -XC_OS=${XC_OS:-"solaris darwin freebsd linux windows"} - -# Make sure build tools are available. -# TODO: Tools should be "vendored" too. -make get_tools - -# Get VENDORED dependencies -make get_vendor_deps - -# Build! -# ldflags: -s Omit the symbol table and debug information. -# -w Omit the DWARF symbol table. -echo "==> Building..." -"$(which gox)" \ - -os="${XC_OS}" \ - -arch="${XC_ARCH}" \ - -osarch="!darwin/arm !solaris/amd64 !freebsd/amd64" \ - -ldflags "-s -w -X ${GIT_IMPORT}.GitCommit=${GIT_COMMIT}" \ - -output "build/pkg/{{.OS}}_{{.Arch}}/tendermint" \ - -tags="${BUILD_TAGS}" \ - github.com/tendermint/tendermint/cmd/tendermint - -# Zip all the files. -echo "==> Packaging..." -for PLATFORM in $(find ./build/pkg -mindepth 1 -maxdepth 1 -type d); do - OSARCH=$(basename "${PLATFORM}") - echo "--> ${OSARCH}" - - pushd "$PLATFORM" >/dev/null 2>&1 - zip "../${OSARCH}.zip" ./* - popd >/dev/null 2>&1 -done - -exit 0 diff --git a/scripts/release.sh b/scripts/release.sh index 02899ad5d..9a4e508e1 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -25,9 +25,9 @@ sh -c "'$DIR/scripts/dist.sh'" # Pushing binaries to S3 sh -c "'$DIR/scripts/publish.sh'" -echo "==> Crafting a Github release" -today=$(date +"%B-%d-%Y") -ghr -b "https://github.com/tendermint/tendermint/blob/master/CHANGELOG.md#${VERSION//.}-${today,}" "v$VERSION" "$DIR/build/dist" +# echo "==> Crafting a Github release" +# today=$(date +"%B-%d-%Y") +# ghr -b "https://github.com/tendermint/tendermint/blob/master/CHANGELOG.md#${VERSION//.}-${today,}" "v$VERSION" "$DIR/build/dist" # Build and push Docker image diff --git a/scripts/tendermint-builder/Dockerfile b/scripts/tendermint-builder/Dockerfile deleted file mode 100644 index 2d3c0ef5d..000000000 --- a/scripts/tendermint-builder/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM golang:1.9.2 - -RUN apt-get update && apt-get install -y --no-install-recommends \ - zip \ - && rm -rf /var/lib/apt/lists/* - -# We want to ensure that release builds never have any cgo dependencies so we -# switch that off at the highest level. -ENV CGO_ENABLED 0 - -RUN mkdir -p $GOPATH/src/github.com/tendermint/tendermint -WORKDIR $GOPATH/src/github.com/tendermint/tendermint