Merge pull request #1086 from tendermint/966-deterministic-tooling-for-releases

Distribution improvements (freeze glide version + get rid of gox)

Makefile

@@ -1,13 +1,9 @@
 GOTOOLS = \
-	github.com/mitchellh/gox \
-	github.com/Masterminds/glide \
-	github.com/tcnksm/ghr \
+	github.com/tendermint/glide \
 	# gopkg.in/alecthomas/gometalinter.v2
-GOTOOLS_CHECK = gox glide ghr gometalinter.v2
 PACKAGES=$(shell go list ./... | grep -v '/vendor/')
 BUILD_TAGS?=tendermint
-TMHOME = $${TMHOME:-$$HOME/.tendermint}
-BUILD_FLAGS = -ldflags "-X github.com/tendermint/tendermint/version.GitCommit=`git rev-parse --short HEAD`"
+BUILD_FLAGS = -ldflags "-X github.com/tendermint/tendermint/version.GitCommit=`git rev-parse --short=8 HEAD`"
 
 all: check build test install
 
@@ -18,25 +14,27 @@ check: check_tools get_vendor_deps
 ### Build
 
 build:
-	go build $(BUILD_FLAGS) -o build/tendermint ./cmd/tendermint/
+	go build $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' -o build/tendermint ./cmd/tendermint/
 
 build_race:
-	go build -race $(BUILD_FLAGS) -o build/tendermint ./cmd/tendermint
+	go build -race $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' -o build/tendermint ./cmd/tendermint
+
+install:
+	go install $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' ./cmd/tendermint
+
+########################################
+### Distribution
 
 # dist builds binaries for all platforms and packages them for distribution
 dist:
 	@BUILD_TAGS='$(BUILD_TAGS)' sh -c "'$(CURDIR)/scripts/dist.sh'"
 
-install:
-	go install $(BUILD_FLAGS) ./cmd/tendermint
-
-
 ########################################
 ### Tools & dependencies
 
 check_tools:
 	@# https://stackoverflow.com/a/25668869
-	@echo "Found tools: $(foreach tool,$(GOTOOLS_CHECK),\
+	@echo "Found tools: $(foreach tool,$(notdir $(GOTOOLS)),\
         $(if $(shell which $(tool)),$(tool),$(error "No $(tool) in PATH")))"
 
 get_tools:

Vagrantfile

@@ -21,7 +21,7 @@ Vagrant.configure("2") do |config|
 
     # install base requirements
     apt-get update
-    apt-get install -y --no-install-recommends wget curl jq \
+    apt-get install -y --no-install-recommends wget curl jq zip \
         make shellcheck bsdmainutils psmisc
     apt-get install -y docker-ce golang-1.9-go
     apt-get install -y language-pack-en

scripts/dist.sh

@@ -1,6 +1,9 @@
 #!/usr/bin/env bash
 set -e
 
+# WARN: non hermetic build (people must run this script inside docker to
+# produce deterministic binaries).
+
 # Get the version from the environment, or try to figure it out.
 if [ -z $VERSION ]; then
 	VERSION=$(awk -F\" '/Version =/ { print $2; exit }' < version/version.go)
@@ -11,22 +14,51 @@ if [ -z "$VERSION" ]; then
 fi
 echo "==> Building version $VERSION..."
 
-# Get the parent directory of where this script is.
-SOURCE="${BASH_SOURCE[0]}"
-while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done
-DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )"
-
-# Change into that dir because we expect that.
-cd "$DIR"
-
 # Delete the old dir
 echo "==> Removing old directory..."
 rm -rf build/pkg
 mkdir -p build/pkg
 
-# Do a hermetic build inside a Docker container.
-docker build -t tendermint/tendermint-builder scripts/tendermint-builder/
-docker run --rm -e "BUILD_TAGS=$BUILD_TAGS" -v "$(pwd)":/go/src/github.com/tendermint/tendermint tendermint/tendermint-builder ./scripts/dist_build.sh
+# Get the git commit
+GIT_COMMIT="$(git rev-parse --short=8 HEAD)"
+GIT_IMPORT="github.com/tendermint/tendermint/version"
+
+# Determine the arch/os combos we're building for
+XC_ARCH=${XC_ARCH:-"386 amd64 arm"}
+XC_OS=${XC_OS:-"solaris darwin freebsd linux windows"}
+XC_EXCLUDE=${XC_EXCLUDE:-" darwin/arm solaris/amd64 solaris/386 solaris/arm freebsd/amd64 windows/arm "}
+
+# Make sure build tools are available.
+make get_tools
+
+# Get VENDORED dependencies
+make get_vendor_deps
+
+# Build!
+# ldflags: -s Omit the symbol table and debug information.
+#	         -w Omit the DWARF symbol table.
+echo "==> Building..."
+IFS=' ' read -ra arch_list <<< "$XC_ARCH"
+IFS=' ' read -ra os_list <<< "$XC_OS"
+for arch in "${arch_list[@]}"; do
+	for os in "${os_list[@]}"; do
+		if [[ "$XC_EXCLUDE" !=  *" $os/$arch "* ]]; then
+			echo "--> $os/$arch"
+			GOOS=${os} GOARCH=${arch} go build -ldflags "-s -w -X ${GIT_IMPORT}.GitCommit=${GIT_COMMIT}" -tags="${BUILD_TAGS}" -o "build/pkg/${os}_${arch}/tendermint" ./cmd/tendermint
+		fi
+	done
+done
+
+# Zip all the files.
+echo "==> Packaging..."
+for PLATFORM in $(find ./build/pkg -mindepth 1 -maxdepth 1 -type d); do
+	OSARCH=$(basename "${PLATFORM}")
+	echo "--> ${OSARCH}"
+
+	pushd "$PLATFORM" >/dev/null 2>&1
+	zip "../${OSARCH}.zip" ./*
+	popd >/dev/null 2>&1
+done
 
 # Add "tendermint" and $VERSION prefix to package name.
 rm -rf ./build/dist

scripts/dist_build.sh

@@ -1,51 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-# Get the parent directory of where this script is.
-SOURCE="${BASH_SOURCE[0]}"
-while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done
-DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )"
-
-# Change into that dir because we expect that.
-cd "$DIR"
-
-# Get the git commit
-GIT_COMMIT="$(git rev-parse --short HEAD)"
-GIT_IMPORT="github.com/tendermint/tendermint/version"
-
-# Determine the arch/os combos we're building for
-XC_ARCH=${XC_ARCH:-"386 amd64 arm"}
-XC_OS=${XC_OS:-"solaris darwin freebsd linux windows"}
-
-# Make sure build tools are available.
-# TODO: Tools should be "vendored" too.
-make get_tools
-
-# Get VENDORED dependencies
-make get_vendor_deps
-
-# Build!
-# ldflags: -s Omit the symbol table and debug information.
-#	         -w Omit the DWARF symbol table.
-echo "==> Building..."
-"$(which gox)" \
-		-os="${XC_OS}" \
-		-arch="${XC_ARCH}" \
-		-osarch="!darwin/arm !solaris/amd64 !freebsd/amd64" \
-		-ldflags "-s -w -X ${GIT_IMPORT}.GitCommit=${GIT_COMMIT}" \
-		-output "build/pkg/{{.OS}}_{{.Arch}}/tendermint" \
-		-tags="${BUILD_TAGS}" \
-		github.com/tendermint/tendermint/cmd/tendermint
-
-# Zip all the files.
-echo "==> Packaging..."
-for PLATFORM in $(find ./build/pkg -mindepth 1 -maxdepth 1 -type d); do
-		OSARCH=$(basename "${PLATFORM}")
-		echo "--> ${OSARCH}"
-
-		pushd "$PLATFORM" >/dev/null 2>&1
-		zip "../${OSARCH}.zip" ./*
-		popd >/dev/null 2>&1
-done
-
-exit 0

scripts/release.sh

@@ -25,9 +25,9 @@ sh -c "'$DIR/scripts/dist.sh'"
 # Pushing binaries to S3
 sh -c "'$DIR/scripts/publish.sh'"
 
-echo "==> Crafting a Github release"
-today=$(date +"%B-%d-%Y")
-ghr -b "https://github.com/tendermint/tendermint/blob/master/CHANGELOG.md#${VERSION//.}-${today,}" "v$VERSION" "$DIR/build/dist"
+# echo "==> Crafting a Github release"
+# today=$(date +"%B-%d-%Y")
+# ghr -b "https://github.com/tendermint/tendermint/blob/master/CHANGELOG.md#${VERSION//.}-${today,}" "v$VERSION" "$DIR/build/dist"
 
 # Build and push Docker image
 

scripts/tendermint-builder/Dockerfile

@@ -1,12 +0,0 @@
-FROM golang:1.9.2
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-		zip \
-	&& rm -rf /var/lib/apt/lists/*
-
-# We want to ensure that release builds never have any cgo dependencies so we
-# switch that off at the highest level.
-ENV CGO_ENABLED 0
-
-RUN mkdir -p $GOPATH/src/github.com/tendermint/tendermint
-WORKDIR $GOPATH/src/github.com/tendermint/tendermint