infrastructure for reproducible builds (#3770)

* Add deterministic buildsystem

* Update CircleCI config

* Enable build on all branches for testing purposes

* Revert "Enable build on all branches for testing purposes"

This reverts commit bf5cf66da9.

* Remove develop from branch filters

* Remove dangling reference to develop

* Upload binaries too

* Build for stable branches too

.circleci/config.yml

@@ -331,6 +331,34 @@ jobs:
             docker push "tendermint/tendermint"
             docker logout
 
+  reproducible_builds:
+    <<: *defaults
+    steps:
+      - attach_workspace:
+          at: /tmp/workspace
+      - checkout
+      - setup_remote_docker:
+          docker_layer_caching: true
+      - run:
+          name: Build tendermint
+          no_output_timeout: 20m
+          command: |
+            sudo apt-get install -y ruby
+            bash -x ./scripts/gitian-build.sh all
+            for os in darwin linux windows; do
+              cp gitian-build-${os}/result/tendermint-${os}-res.yml .
+              cp gitian-build-${os}/build/out/tendermint-*.tar.gz .
+              rm -rf gitian-build-${os}/
+            done
+      - store_artifacts:
+          path: /go/src/github.com/tendermint/tendermint/tendermint-darwin-res.yml
+      - store_artifacts:
+          path: /go/src/github.com/tendermint/tendermint/tendermint-linux-res.yml
+      - store_artifacts:
+          path: /go/src/github.com/tendermint/tendermint/tendermint-windows-res.yml
+      - store_artifacts:
+          path: /go/src/github.com/tendermint/tendermint/tendermint-*.tar.gz
+
 workflows:
   version: 2
   test-suite:
@@ -340,7 +368,6 @@ workflows:
             branches:
               only:
                 - master
-                - develop
       - setup_dependencies
       - test_abci_apps:
           requires:
@@ -364,6 +391,12 @@ workflows:
       - upload_coverage:
           requires:
             - test_cover
+      - reproducible_builds:
+          filters:
+            branches:
+              only:
+                - master
+                - /v[0-9]+\.[0-9]+/
   release:
     jobs:
       - prepare_build

scripts/gitian-build.sh

@@ -0,0 +1,201 @@
+#!/bin/bash
+
+# symbol prefixes:
+# g_ -> global
+# l_ - local variable
+# f_ -> function
+
+set -euo pipefail
+
+GITIAN_CACHE_DIRNAME='.gitian-builder-cache'
+GO_DEBIAN_RELEASE='1.12.5-1'
+GO_TARBALL="golang-debian-${GO_DEBIAN_RELEASE}.tar.gz"
+GO_TARBALL_URL="https://salsa.debian.org/go-team/compiler/golang/-/archive/debian/${GO_DEBIAN_RELEASE}/${GO_TARBALL}"
+
+# Defaults
+
+DEFAULT_SIGN_COMMAND='gpg --detach-sign'
+DEFAULT_TENDERMINT_SIGS=${TENDERMINT_SIGS:-'tendermint.sigs'}
+DEFAULT_GITIAN_REPO='https://github.com/devrandom/gitian-builder'
+DEFAULT_GBUILD_FLAGS=''
+DEFAULT_SIGS_REPO='https://github.com/tendermint/tendermint.sigs'
+
+# Overrides
+
+SIGN_COMMAND=${SIGN_COMMAND:-${DEFAULT_SIGN_COMMAND}}
+GITIAN_REPO=${GITIAN_REPO:-${DEFAULT_GITIAN_REPO}}
+GBUILD_FLAGS=${GBUILD_FLAGS:-${DEFAULT_GBUILD_FLAGS}}
+
+# Globals
+
+g_workdir=''
+g_gitian_cache=''
+g_cached_gitian=''
+g_cached_go_tarball=''
+g_sign_identity=''
+g_sigs_dir=''
+g_flag_commit=''
+
+
+f_help() {
+  cat >&2 <<EOF
+Usage: $(basename $0) [-h] PLATFORM
+Launch a gitian build from the current source directory for the given PLATFORM.
+The following platforms are supported:
+  darwin
+  linux
+  windows
+  all
+
+  Options:
+   -h               display this help and exit
+   -c               clone the signatures repository and commit signatures;
+                    ignored if sign identity is not supplied
+   -s IDENTITY      sign build as IDENTITY
+
+If a GPG identity is supplied via the -s flag, the build will be signed and verified.
+The signature will be saved in '${DEFAULT_TENDERMINT_SIGS}/'. An alternative output directory
+for signatures can be supplied via the environment variable \$TENDERMINT_SIGS.
+
+The default signing command used to sign the build is '$DEFAULT_SIGN_COMMAND'.
+An alternative signing command can be supplied via the environment
+variable \$SIGN_COMMAND.
+EOF
+}
+
+
+f_builddir() {
+  printf '%s' "${g_workdir}/gitian-build-$1"
+}
+
+f_prep_build() {
+  local l_platforms \
+    l_os \
+    l_builddir
+
+  l_platforms="$1"
+
+  if [ -n "${g_flag_commit}" -a ! -d "${g_sigs_dir}" ]; then
+    git clone ${DEFAULT_SIGS_REPO} "${g_sigs_dir}"
+  fi
+
+  for l_os in ${l_platforms}; do
+    l_builddir="$(f_builddir ${l_os})"
+
+    f_echo_stderr "Preparing build directory $(basename ${l_builddir}), restoring files from cache"
+    cp -ar "${g_cached_gitian}" "${l_builddir}" >&2
+    mkdir "${l_builddir}/inputs/"
+    cp -v "${g_cached_go_tarball}" "${l_builddir}/inputs/"
+  done
+}
+
+f_build() {
+  local l_descriptor
+
+  l_descriptor=$1
+
+  bin/gbuild --commit tendermint="$g_commit" ${GBUILD_FLAGS} "$l_descriptor"
+  libexec/stop-target || f_echo_stderr "warning: couldn't stop target"
+}
+
+f_sign_verify() {
+  local l_descriptor
+
+  l_descriptor=$1
+
+  bin/gsign -p "${SIGN_COMMAND}" -s "${g_sign_identity}" --destination="${g_sigs_dir}" --release=${g_release} ${l_descriptor}
+  bin/gverify --destination="${g_sigs_dir}" --release="${g_release}" ${l_descriptor}
+}
+
+f_commit_sig() {
+  local l_release_name
+
+  l_release_name=$1
+
+  pushd "${g_sigs_dir}"
+  git add . || echo "git add failed" >&2
+  git commit -m "Add ${l_release_name} reproducible build" || echo "git commit failed" >&2
+  popd
+}
+
+f_prep_docker_image() {
+  pushd $1
+  bin/make-base-vm --docker --suite bionic --arch amd64
+  popd
+}
+
+f_ensure_cache() {
+  g_gitian_cache="${g_workdir}/${GITIAN_CACHE_DIRNAME}"
+  [ -d "${g_gitian_cache}" ] || mkdir "${g_gitian_cache}"
+
+  g_cached_go_tarball="${g_gitian_cache}/${GO_TARBALL}"
+  if [ ! -f "${g_cached_go_tarball}" ]; then
+  f_echo_stderr "${g_cached_go_tarball}: cache miss, caching..."
+    curl -L "${GO_TARBALL_URL}" --output "${g_cached_go_tarball}"
+  fi
+
+  g_cached_gitian="${g_gitian_cache}/gitian-builder"
+  if [ ! -d "${g_cached_gitian}" ]; then
+  f_echo_stderr "${g_cached_gitian}: cache miss, caching..."
+    git clone ${GITIAN_REPO} "${g_cached_gitian}"
+  fi
+}
+
+f_demangle_platforms() {
+  case "${1}" in
+  all)
+    printf '%s' 'darwin linux windows' ;;
+  linux|darwin|windows)
+    printf '%s' "${1}" ;;
+  *)
+    echo "invalid platform -- ${1}"
+    exit 1
+  esac
+}
+
+f_echo_stderr() {
+  echo $@ >&2
+}
+
+
+while getopts ":cs:h" opt; do
+  case "${opt}" in
+    h)  f_help ; exit 0 ;;
+    c)  g_flag_commit=y ;;
+    s)  g_sign_identity="${OPTARG}" ;;
+  esac
+done
+
+shift "$((OPTIND-1))"
+
+g_platforms=$(f_demangle_platforms "${1}")
+g_workdir="$(pwd)"
+g_commit="$(git rev-parse HEAD)"
+g_sigs_dir=${TENDERMINT_SIGS:-"${g_workdir}/${DEFAULT_TENDERMINT_SIGS}"}
+
+f_ensure_cache
+
+f_prep_docker_image "${g_cached_gitian}"
+
+f_prep_build "${g_platforms}"
+
+export USE_DOCKER=1
+for g_os in ${g_platforms}; do
+  g_release="$(git describe --tags --abbrev=9 | sed 's/^v//')-${g_os}"
+  g_descriptor="${g_workdir}/scripts/gitian-descriptors/gitian-${g_os}.yml"
+  [ -f ${g_descriptor} ]
+  g_builddir="$(f_builddir ${g_os})"
+
+  pushd "${g_builddir}"
+  f_build "${g_descriptor}"
+  if [ -n "${g_sign_identity}" ]; then
+    f_sign_verify "${g_descriptor}"
+  fi
+  popd
+
+  if [ -n "${g_sign_identity}" -a -n "${g_flag_commit}" ]; then
+    [ -d "${g_sigs_dir}/.git/" ] && f_commit_sig ${g_release} || f_echo_stderr "couldn't commit, ${g_sigs_dir} is not a git clone"
+  fi
+done
+
+exit 0

scripts/gitian-descriptors/gitian-darwin.yml

@@ -0,0 +1,111 @@
+---
+name: "tendermint-darwin"
+enable_cache: true
+distro: "ubuntu"
+suites:
+- "bionic"
+architectures:
+- "amd64"
+packages:
+- "bsdmainutils"
+- "build-essential"
+- "ca-certificates"
+- "curl"
+- "debhelper"
+- "dpkg-dev"
+- "devscripts"
+- "fakeroot"
+- "git"
+- "golang-any"
+- "xxd"
+- "quilt"
+remotes:
+- "url": "https://github.com/tendermint/tendermint.git"
+  "dir": "tendermint"
+files:
+- "golang-debian-1.12.5-1.tar.gz"
+script: |
+  set -e -o pipefail
+
+  GO_SRC_RELEASE=golang-debian-1.12.5-1
+  GO_SRC_TARBALL="${GO_SRC_RELEASE}.tar.gz"
+  # Compile go and configure the environment
+  export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME""
+  export BUILD_DIR=`pwd`
+  tar xf "${GO_SRC_TARBALL}"
+  rm -f "${GO_SRC_TARBALL}"
+  [ -d "${GO_SRC_RELEASE}/" ]
+  mv "${GO_SRC_RELEASE}/" go/
+  pushd go/
+  QUILT_PATCHES=debian/patches quilt push -a
+  fakeroot debian/rules build RUN_TESTS=false GOCACHE=/tmp/go-cache
+  popd
+
+  export GOOS=darwin
+  export GOROOT=${BUILD_DIR}/go
+  export GOPATH=${BUILD_DIR}/gopath
+  mkdir -p ${GOPATH}/bin
+
+  export PATH_orig=${PATH}
+  export PATH=$GOPATH/bin:$GOROOT/bin:$PATH
+
+  export ARCHS='386 amd64'
+  export GO111MODULE=on
+
+  # Make release tarball
+  pushd tendermint
+  VERSION=$(git describe --tags | sed 's/^v//')
+  COMMIT=$(git rev-parse --short=8 HEAD)
+  DISTNAME=tendermint-${VERSION}
+  git archive --format tar.gz --prefix ${DISTNAME}/ -o ${DISTNAME}.tar.gz HEAD
+  SOURCEDIST=`pwd`/`echo tendermint-*.tar.gz`
+  popd
+
+  # Correct tar file order
+  mkdir -p temp
+  pushd temp
+  tar xf $SOURCEDIST
+  rm $SOURCEDIST
+  find tendermint-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > $SOURCEDIST
+  popd
+
+  # Prepare GOPATH and install deps
+  distsrc=${GOPATH}/src/github.com/tendermint/tendermint
+  mkdir -p ${distsrc}
+  pushd ${distsrc}
+  tar --strip-components=1 -xf $SOURCEDIST
+  go mod download
+  popd
+
+  # Configure LDFLAGS for reproducible builds
+  LDFLAGS="-extldflags=-static -buildid=${VERSION} -s -w \
+    -X github.com/tendermint/tendermint/version.GitCommit=${COMMIT}"
+
+  # Extract release tarball and build
+  for arch in ${ARCHS}; do
+    INSTALLPATH=`pwd`/installed/${DISTNAME}-${arch}
+    mkdir -p ${INSTALLPATH}
+
+    # Build tendermint binary
+    pushd ${distsrc}
+    GOARCH=${arch} GOROOT_FINAL=${GOROOT} go build -a \
+      -gcflags=all=-trimpath=${GOPATH} \
+      -asmflags=all=-trimpath=${GOPATH} \
+      -mod=readonly -tags "tendermint" \
+      -ldflags="${LDFLAGS}" \
+      -o ${INSTALLPATH}/tendermint ./cmd/tendermint/
+
+    popd # ${distsrc}
+
+    pushd ${INSTALLPATH}
+    find -type f | sort | tar \
+      --no-recursion --mode='u+rw,go+r-w,a+X' \
+      --numeric-owner --sort=name \
+      --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-darwin-${arch}.tar.gz
+    popd  # installed
+  done
+
+  rm -rf ${distsrc}
+
+  mkdir -p $OUTDIR/src
+  mv $SOURCEDIST $OUTDIR/src

scripts/gitian-descriptors/gitian-linux.yml

@@ -0,0 +1,110 @@
+---
+name: "tendermint-linux"
+enable_cache: true
+distro: "ubuntu"
+suites:
+- "bionic"
+architectures:
+- "amd64"
+packages:
+- "bsdmainutils"
+- "build-essential"
+- "ca-certificates"
+- "curl"
+- "debhelper"
+- "dpkg-dev"
+- "devscripts"
+- "fakeroot"
+- "git"
+- "golang-any"
+- "xxd"
+- "quilt"
+remotes:
+- "url": "https://github.com/tendermint/tendermint.git"
+  "dir": "tendermint"
+files:
+- "golang-debian-1.12.5-1.tar.gz"
+script: |
+  set -e -o pipefail
+
+  GO_SRC_RELEASE=golang-debian-1.12.5-1
+  GO_SRC_TARBALL="${GO_SRC_RELEASE}.tar.gz"
+  # Compile go and configure the environment
+  export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME""
+  export BUILD_DIR=`pwd`
+  tar xf "${GO_SRC_TARBALL}"
+  rm -f "${GO_SRC_TARBALL}"
+  [ -d "${GO_SRC_RELEASE}/" ]
+  mv "${GO_SRC_RELEASE}/" go/
+  pushd go/
+  QUILT_PATCHES=debian/patches quilt push -a
+  fakeroot debian/rules build RUN_TESTS=false GOCACHE=/tmp/go-cache
+  popd
+
+  export GOROOT=${BUILD_DIR}/go
+  export GOPATH=${BUILD_DIR}/gopath
+  mkdir -p ${GOPATH}/bin
+
+  export PATH_orig=${PATH}
+  export PATH=$GOPATH/bin:$GOROOT/bin:$PATH
+
+  export ARCHS='386 amd64 arm arm64'
+  export GO111MODULE=on
+
+  # Make release tarball
+  pushd tendermint
+  VERSION=$(git describe --tags | sed 's/^v//')
+  COMMIT=$(git rev-parse --short=8 HEAD)
+  DISTNAME=tendermint-${VERSION}
+  git archive --format tar.gz --prefix ${DISTNAME}/ -o ${DISTNAME}.tar.gz HEAD
+  SOURCEDIST=`pwd`/`echo tendermint-*.tar.gz`
+  popd
+
+  # Correct tar file order
+  mkdir -p temp
+  pushd temp
+  tar xf $SOURCEDIST
+  rm $SOURCEDIST
+  find tendermint-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > $SOURCEDIST
+  popd
+
+  # Prepare GOPATH and install deps
+  distsrc=${GOPATH}/src/github.com/tendermint/tendermint
+  mkdir -p ${distsrc}
+  pushd ${distsrc}
+  tar --strip-components=1 -xf $SOURCEDIST
+  go mod download
+  popd
+
+  # Configure LDFLAGS for reproducible builds
+  LDFLAGS="-extldflags=-static -buildid=${VERSION} -s -w \
+    -X github.com/tendermint/tendermint/version.GitCommit=${COMMIT}"
+
+  # Extract release tarball and build
+  for arch in ${ARCHS}; do
+    INSTALLPATH=`pwd`/installed/${DISTNAME}-${arch}
+    mkdir -p ${INSTALLPATH}
+
+    # Build tendermint binary
+    pushd ${distsrc}
+    GOARCH=${arch} GOROOT_FINAL=${GOROOT} go build -a \
+      -gcflags=all=-trimpath=${GOPATH} \
+      -asmflags=all=-trimpath=${GOPATH} \
+      -mod=readonly -tags "tendermint" \
+      -ldflags="${LDFLAGS}" \
+      -o ${INSTALLPATH}/tendermint ./cmd/tendermint/
+
+    popd # ${distsrc}
+
+    pushd ${INSTALLPATH}
+    find -type f | sort | tar \
+      --no-recursion --mode='u+rw,go+r-w,a+X' \
+      --numeric-owner --sort=name \
+      --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-linux-${arch}.tar.gz
+    popd  # installed
+  done
+
+  rm -rf ${distsrc}
+
+  mkdir -p $OUTDIR/src
+  mv $SOURCEDIST $OUTDIR/src

scripts/gitian-descriptors/gitian-windows.yml

@@ -0,0 +1,111 @@
+---
+name: "tendermint-windows"
+enable_cache: true
+distro: "ubuntu"
+suites:
+- "bionic"
+architectures:
+- "amd64"
+packages:
+- "bsdmainutils"
+- "build-essential"
+- "ca-certificates"
+- "curl"
+- "debhelper"
+- "dpkg-dev"
+- "devscripts"
+- "fakeroot"
+- "git"
+- "golang-any"
+- "xxd"
+- "quilt"
+remotes:
+- "url": "https://github.com/tendermint/tendermint.git"
+  "dir": "tendermint"
+files:
+- "golang-debian-1.12.5-1.tar.gz"
+script: |
+  set -e -o pipefail
+
+  GO_SRC_RELEASE=golang-debian-1.12.5-1
+  GO_SRC_TARBALL="${GO_SRC_RELEASE}.tar.gz"
+  # Compile go and configure the environment
+  export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME""
+  export BUILD_DIR=`pwd`
+  tar xf "${GO_SRC_TARBALL}"
+  rm -f "${GO_SRC_TARBALL}"
+  [ -d "${GO_SRC_RELEASE}/" ]
+  mv "${GO_SRC_RELEASE}/" go/
+  pushd go/
+  QUILT_PATCHES=debian/patches quilt push -a
+  fakeroot debian/rules build RUN_TESTS=false GOCACHE=/tmp/go-cache
+  popd
+
+  export GOOS=windows
+  export GOROOT=${BUILD_DIR}/go
+  export GOPATH=${BUILD_DIR}/gopath
+  mkdir -p ${GOPATH}/bin
+
+  export PATH_orig=${PATH}
+  export PATH=$GOPATH/bin:$GOROOT/bin:$PATH
+
+  export ARCHS='386 amd64'
+  export GO111MODULE=on
+
+  # Make release tarball
+  pushd tendermint
+  VERSION=$(git describe --tags | sed 's/^v//')
+  COMMIT=$(git rev-parse --short=8 HEAD)
+  DISTNAME=tendermint-${VERSION}
+  git archive --format tar.gz --prefix ${DISTNAME}/ -o ${DISTNAME}.tar.gz HEAD
+  SOURCEDIST=`pwd`/`echo tendermint-*.tar.gz`
+  popd
+
+  # Correct tar file order
+  mkdir -p temp
+  pushd temp
+  tar xf $SOURCEDIST
+  rm $SOURCEDIST
+  find tendermint-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > $SOURCEDIST
+  popd
+
+  # Prepare GOPATH and install deps
+  distsrc=${GOPATH}/src/github.com/tendermint/tendermint
+  mkdir -p ${distsrc}
+  pushd ${distsrc}
+  tar --strip-components=1 -xf $SOURCEDIST
+  go mod download
+  popd
+
+  # Configure LDFLAGS for reproducible builds
+  LDFLAGS="-extldflags=-static -buildid=${VERSION} -s -w \
+    -X github.com/tendermint/tendermint/version.GitCommit=${COMMIT}"
+
+  # Extract release tarball and build
+  for arch in ${ARCHS}; do
+    INSTALLPATH=`pwd`/installed/${DISTNAME}-${arch}
+    mkdir -p ${INSTALLPATH}
+
+    # Build tendermint binary
+    pushd ${distsrc}
+    GOARCH=${arch} GOROOT_FINAL=${GOROOT} go build -a \
+      -gcflags=all=-trimpath=${GOPATH} \
+      -asmflags=all=-trimpath=${GOPATH} \
+      -mod=readonly -tags "tendermint" \
+      -ldflags="${LDFLAGS}" \
+      -o ${INSTALLPATH}/tendermint.exe ./cmd/tendermint/
+
+    popd # ${distsrc}
+
+    pushd ${INSTALLPATH}
+    find -type f | sort | tar \
+      --no-recursion --mode='u+rw,go+r-w,a+X' \
+      --numeric-owner --sort=name \
+      --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-windows-${arch}.tar.gz
+    popd  # installed
+  done
+
+  rm -rf ${distsrc}
+
+  mkdir -p $OUTDIR/src
+  mv $SOURCEDIST $OUTDIR/src

scripts/gitian-keys/README.md

@@ -0,0 +1,29 @@
+## PGP keys of Gitian builders and Tendermint Developers
+
+The file `keys.txt` contains fingerprints of the public keys of Gitian builders
+and active developers.
+
+The associated keys are mainly used to sign git commits or the build results
+of Gitian builds.
+
+The most recent version of each pgp key can be found on most PGP key servers.
+
+Fetch the latest version from the key server to see if any key was revoked in
+the meantime.
+To fetch the latest version of all pgp keys in your gpg homedir,
+
+```sh
+gpg --refresh-keys
+```
+
+To fetch keys of Gitian builders and active core developers, feed the list of
+fingerprints of the primary keys into gpg:
+
+```sh
+while read fingerprint keyholder_name; \
+do gpg --keyserver hkp://subset.pool.sks-keyservers.net \
+--recv-keys ${fingerprint}; done < ./keys.txt
+```
+
+Add your key to the list if you are a Tendermint core developer or you have
+provided Gitian signatures for two major or minor releases of Tendermint.

scripts/gitian-keys/keys.txt

@@ -0,0 +1 @@
+04160004A8276E40BB9890FBE8A48AE5311D765A Alessio Treglia