From ddee2d641f864d62212bc3ab318f5ff7b64d9aa3 Mon Sep 17 00:00:00 2001 From: Alessio Treglia Date: Wed, 10 Jul 2019 09:48:31 +0100 Subject: [PATCH] infrastructure for reproducible builds (#3770) * Add deterministic buildsystem * Update CircleCI config * Enable build on all branches for testing purposes * Revert "Enable build on all branches for testing purposes" This reverts commit bf5cf66da94bf9c23787b42995b0409492805f03. * Remove develop from branch filters * Remove dangling reference to develop * Upload binaries too * Build for stable branches too --- .circleci/config.yml | 35 ++- scripts/gitian-build.sh | 201 ++++++++++++++++++ scripts/gitian-descriptors/gitian-darwin.yml | 111 ++++++++++ scripts/gitian-descriptors/gitian-linux.yml | 110 ++++++++++ scripts/gitian-descriptors/gitian-windows.yml | 111 ++++++++++ scripts/gitian-keys/README.md | 29 +++ scripts/gitian-keys/keys.txt | 1 + 7 files changed, 597 insertions(+), 1 deletion(-) create mode 100755 scripts/gitian-build.sh create mode 100644 scripts/gitian-descriptors/gitian-darwin.yml create mode 100644 scripts/gitian-descriptors/gitian-linux.yml create mode 100644 scripts/gitian-descriptors/gitian-windows.yml create mode 100644 scripts/gitian-keys/README.md create mode 100644 scripts/gitian-keys/keys.txt diff --git a/.circleci/config.yml b/.circleci/config.yml index 5836b4546..539dd7ee4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -331,6 +331,34 @@ jobs: docker push "tendermint/tendermint" docker logout + reproducible_builds: + <<: *defaults + steps: + - attach_workspace: + at: /tmp/workspace + - checkout + - setup_remote_docker: + docker_layer_caching: true + - run: + name: Build tendermint + no_output_timeout: 20m + command: | + sudo apt-get install -y ruby + bash -x ./scripts/gitian-build.sh all + for os in darwin linux windows; do + cp gitian-build-${os}/result/tendermint-${os}-res.yml . + cp gitian-build-${os}/build/out/tendermint-*.tar.gz . + rm -rf gitian-build-${os}/ + done + - store_artifacts: + path: /go/src/github.com/tendermint/tendermint/tendermint-darwin-res.yml + - store_artifacts: + path: /go/src/github.com/tendermint/tendermint/tendermint-linux-res.yml + - store_artifacts: + path: /go/src/github.com/tendermint/tendermint/tendermint-windows-res.yml + - store_artifacts: + path: /go/src/github.com/tendermint/tendermint/tendermint-*.tar.gz + workflows: version: 2 test-suite: @@ -340,7 +368,6 @@ workflows: branches: only: - master - - develop - setup_dependencies - test_abci_apps: requires: @@ -364,6 +391,12 @@ workflows: - upload_coverage: requires: - test_cover + - reproducible_builds: + filters: + branches: + only: + - master + - /v[0-9]+\.[0-9]+/ release: jobs: - prepare_build diff --git a/scripts/gitian-build.sh b/scripts/gitian-build.sh new file mode 100755 index 000000000..a7a6acec3 --- /dev/null +++ b/scripts/gitian-build.sh @@ -0,0 +1,201 @@ +#!/bin/bash + +# symbol prefixes: +# g_ -> global +# l_ - local variable +# f_ -> function + +set -euo pipefail + +GITIAN_CACHE_DIRNAME='.gitian-builder-cache' +GO_DEBIAN_RELEASE='1.12.5-1' +GO_TARBALL="golang-debian-${GO_DEBIAN_RELEASE}.tar.gz" +GO_TARBALL_URL="https://salsa.debian.org/go-team/compiler/golang/-/archive/debian/${GO_DEBIAN_RELEASE}/${GO_TARBALL}" + +# Defaults + +DEFAULT_SIGN_COMMAND='gpg --detach-sign' +DEFAULT_TENDERMINT_SIGS=${TENDERMINT_SIGS:-'tendermint.sigs'} +DEFAULT_GITIAN_REPO='https://github.com/devrandom/gitian-builder' +DEFAULT_GBUILD_FLAGS='' +DEFAULT_SIGS_REPO='https://github.com/tendermint/tendermint.sigs' + +# Overrides + +SIGN_COMMAND=${SIGN_COMMAND:-${DEFAULT_SIGN_COMMAND}} +GITIAN_REPO=${GITIAN_REPO:-${DEFAULT_GITIAN_REPO}} +GBUILD_FLAGS=${GBUILD_FLAGS:-${DEFAULT_GBUILD_FLAGS}} + +# Globals + +g_workdir='' +g_gitian_cache='' +g_cached_gitian='' +g_cached_go_tarball='' +g_sign_identity='' +g_sigs_dir='' +g_flag_commit='' + + +f_help() { + cat >&2 <&2 + mkdir "${l_builddir}/inputs/" + cp -v "${g_cached_go_tarball}" "${l_builddir}/inputs/" + done +} + +f_build() { + local l_descriptor + + l_descriptor=$1 + + bin/gbuild --commit tendermint="$g_commit" ${GBUILD_FLAGS} "$l_descriptor" + libexec/stop-target || f_echo_stderr "warning: couldn't stop target" +} + +f_sign_verify() { + local l_descriptor + + l_descriptor=$1 + + bin/gsign -p "${SIGN_COMMAND}" -s "${g_sign_identity}" --destination="${g_sigs_dir}" --release=${g_release} ${l_descriptor} + bin/gverify --destination="${g_sigs_dir}" --release="${g_release}" ${l_descriptor} +} + +f_commit_sig() { + local l_release_name + + l_release_name=$1 + + pushd "${g_sigs_dir}" + git add . || echo "git add failed" >&2 + git commit -m "Add ${l_release_name} reproducible build" || echo "git commit failed" >&2 + popd +} + +f_prep_docker_image() { + pushd $1 + bin/make-base-vm --docker --suite bionic --arch amd64 + popd +} + +f_ensure_cache() { + g_gitian_cache="${g_workdir}/${GITIAN_CACHE_DIRNAME}" + [ -d "${g_gitian_cache}" ] || mkdir "${g_gitian_cache}" + + g_cached_go_tarball="${g_gitian_cache}/${GO_TARBALL}" + if [ ! -f "${g_cached_go_tarball}" ]; then + f_echo_stderr "${g_cached_go_tarball}: cache miss, caching..." + curl -L "${GO_TARBALL_URL}" --output "${g_cached_go_tarball}" + fi + + g_cached_gitian="${g_gitian_cache}/gitian-builder" + if [ ! -d "${g_cached_gitian}" ]; then + f_echo_stderr "${g_cached_gitian}: cache miss, caching..." + git clone ${GITIAN_REPO} "${g_cached_gitian}" + fi +} + +f_demangle_platforms() { + case "${1}" in + all) + printf '%s' 'darwin linux windows' ;; + linux|darwin|windows) + printf '%s' "${1}" ;; + *) + echo "invalid platform -- ${1}" + exit 1 + esac +} + +f_echo_stderr() { + echo $@ >&2 +} + + +while getopts ":cs:h" opt; do + case "${opt}" in + h) f_help ; exit 0 ;; + c) g_flag_commit=y ;; + s) g_sign_identity="${OPTARG}" ;; + esac +done + +shift "$((OPTIND-1))" + +g_platforms=$(f_demangle_platforms "${1}") +g_workdir="$(pwd)" +g_commit="$(git rev-parse HEAD)" +g_sigs_dir=${TENDERMINT_SIGS:-"${g_workdir}/${DEFAULT_TENDERMINT_SIGS}"} + +f_ensure_cache + +f_prep_docker_image "${g_cached_gitian}" + +f_prep_build "${g_platforms}" + +export USE_DOCKER=1 +for g_os in ${g_platforms}; do + g_release="$(git describe --tags --abbrev=9 | sed 's/^v//')-${g_os}" + g_descriptor="${g_workdir}/scripts/gitian-descriptors/gitian-${g_os}.yml" + [ -f ${g_descriptor} ] + g_builddir="$(f_builddir ${g_os})" + + pushd "${g_builddir}" + f_build "${g_descriptor}" + if [ -n "${g_sign_identity}" ]; then + f_sign_verify "${g_descriptor}" + fi + popd + + if [ -n "${g_sign_identity}" -a -n "${g_flag_commit}" ]; then + [ -d "${g_sigs_dir}/.git/" ] && f_commit_sig ${g_release} || f_echo_stderr "couldn't commit, ${g_sigs_dir} is not a git clone" + fi +done + +exit 0 diff --git a/scripts/gitian-descriptors/gitian-darwin.yml b/scripts/gitian-descriptors/gitian-darwin.yml new file mode 100644 index 000000000..03ba1f1a4 --- /dev/null +++ b/scripts/gitian-descriptors/gitian-darwin.yml @@ -0,0 +1,111 @@ +--- +name: "tendermint-darwin" +enable_cache: true +distro: "ubuntu" +suites: +- "bionic" +architectures: +- "amd64" +packages: +- "bsdmainutils" +- "build-essential" +- "ca-certificates" +- "curl" +- "debhelper" +- "dpkg-dev" +- "devscripts" +- "fakeroot" +- "git" +- "golang-any" +- "xxd" +- "quilt" +remotes: +- "url": "https://github.com/tendermint/tendermint.git" + "dir": "tendermint" +files: +- "golang-debian-1.12.5-1.tar.gz" +script: | + set -e -o pipefail + + GO_SRC_RELEASE=golang-debian-1.12.5-1 + GO_SRC_TARBALL="${GO_SRC_RELEASE}.tar.gz" + # Compile go and configure the environment + export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" + export BUILD_DIR=`pwd` + tar xf "${GO_SRC_TARBALL}" + rm -f "${GO_SRC_TARBALL}" + [ -d "${GO_SRC_RELEASE}/" ] + mv "${GO_SRC_RELEASE}/" go/ + pushd go/ + QUILT_PATCHES=debian/patches quilt push -a + fakeroot debian/rules build RUN_TESTS=false GOCACHE=/tmp/go-cache + popd + + export GOOS=darwin + export GOROOT=${BUILD_DIR}/go + export GOPATH=${BUILD_DIR}/gopath + mkdir -p ${GOPATH}/bin + + export PATH_orig=${PATH} + export PATH=$GOPATH/bin:$GOROOT/bin:$PATH + + export ARCHS='386 amd64' + export GO111MODULE=on + + # Make release tarball + pushd tendermint + VERSION=$(git describe --tags | sed 's/^v//') + COMMIT=$(git rev-parse --short=8 HEAD) + DISTNAME=tendermint-${VERSION} + git archive --format tar.gz --prefix ${DISTNAME}/ -o ${DISTNAME}.tar.gz HEAD + SOURCEDIST=`pwd`/`echo tendermint-*.tar.gz` + popd + + # Correct tar file order + mkdir -p temp + pushd temp + tar xf $SOURCEDIST + rm $SOURCEDIST + find tendermint-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > $SOURCEDIST + popd + + # Prepare GOPATH and install deps + distsrc=${GOPATH}/src/github.com/tendermint/tendermint + mkdir -p ${distsrc} + pushd ${distsrc} + tar --strip-components=1 -xf $SOURCEDIST + go mod download + popd + + # Configure LDFLAGS for reproducible builds + LDFLAGS="-extldflags=-static -buildid=${VERSION} -s -w \ + -X github.com/tendermint/tendermint/version.GitCommit=${COMMIT}" + + # Extract release tarball and build + for arch in ${ARCHS}; do + INSTALLPATH=`pwd`/installed/${DISTNAME}-${arch} + mkdir -p ${INSTALLPATH} + + # Build tendermint binary + pushd ${distsrc} + GOARCH=${arch} GOROOT_FINAL=${GOROOT} go build -a \ + -gcflags=all=-trimpath=${GOPATH} \ + -asmflags=all=-trimpath=${GOPATH} \ + -mod=readonly -tags "tendermint" \ + -ldflags="${LDFLAGS}" \ + -o ${INSTALLPATH}/tendermint ./cmd/tendermint/ + + popd # ${distsrc} + + pushd ${INSTALLPATH} + find -type f | sort | tar \ + --no-recursion --mode='u+rw,go+r-w,a+X' \ + --numeric-owner --sort=name \ + --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-darwin-${arch}.tar.gz + popd # installed + done + + rm -rf ${distsrc} + + mkdir -p $OUTDIR/src + mv $SOURCEDIST $OUTDIR/src diff --git a/scripts/gitian-descriptors/gitian-linux.yml b/scripts/gitian-descriptors/gitian-linux.yml new file mode 100644 index 000000000..f1c31c40e --- /dev/null +++ b/scripts/gitian-descriptors/gitian-linux.yml @@ -0,0 +1,110 @@ +--- +name: "tendermint-linux" +enable_cache: true +distro: "ubuntu" +suites: +- "bionic" +architectures: +- "amd64" +packages: +- "bsdmainutils" +- "build-essential" +- "ca-certificates" +- "curl" +- "debhelper" +- "dpkg-dev" +- "devscripts" +- "fakeroot" +- "git" +- "golang-any" +- "xxd" +- "quilt" +remotes: +- "url": "https://github.com/tendermint/tendermint.git" + "dir": "tendermint" +files: +- "golang-debian-1.12.5-1.tar.gz" +script: | + set -e -o pipefail + + GO_SRC_RELEASE=golang-debian-1.12.5-1 + GO_SRC_TARBALL="${GO_SRC_RELEASE}.tar.gz" + # Compile go and configure the environment + export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" + export BUILD_DIR=`pwd` + tar xf "${GO_SRC_TARBALL}" + rm -f "${GO_SRC_TARBALL}" + [ -d "${GO_SRC_RELEASE}/" ] + mv "${GO_SRC_RELEASE}/" go/ + pushd go/ + QUILT_PATCHES=debian/patches quilt push -a + fakeroot debian/rules build RUN_TESTS=false GOCACHE=/tmp/go-cache + popd + + export GOROOT=${BUILD_DIR}/go + export GOPATH=${BUILD_DIR}/gopath + mkdir -p ${GOPATH}/bin + + export PATH_orig=${PATH} + export PATH=$GOPATH/bin:$GOROOT/bin:$PATH + + export ARCHS='386 amd64 arm arm64' + export GO111MODULE=on + + # Make release tarball + pushd tendermint + VERSION=$(git describe --tags | sed 's/^v//') + COMMIT=$(git rev-parse --short=8 HEAD) + DISTNAME=tendermint-${VERSION} + git archive --format tar.gz --prefix ${DISTNAME}/ -o ${DISTNAME}.tar.gz HEAD + SOURCEDIST=`pwd`/`echo tendermint-*.tar.gz` + popd + + # Correct tar file order + mkdir -p temp + pushd temp + tar xf $SOURCEDIST + rm $SOURCEDIST + find tendermint-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > $SOURCEDIST + popd + + # Prepare GOPATH and install deps + distsrc=${GOPATH}/src/github.com/tendermint/tendermint + mkdir -p ${distsrc} + pushd ${distsrc} + tar --strip-components=1 -xf $SOURCEDIST + go mod download + popd + + # Configure LDFLAGS for reproducible builds + LDFLAGS="-extldflags=-static -buildid=${VERSION} -s -w \ + -X github.com/tendermint/tendermint/version.GitCommit=${COMMIT}" + + # Extract release tarball and build + for arch in ${ARCHS}; do + INSTALLPATH=`pwd`/installed/${DISTNAME}-${arch} + mkdir -p ${INSTALLPATH} + + # Build tendermint binary + pushd ${distsrc} + GOARCH=${arch} GOROOT_FINAL=${GOROOT} go build -a \ + -gcflags=all=-trimpath=${GOPATH} \ + -asmflags=all=-trimpath=${GOPATH} \ + -mod=readonly -tags "tendermint" \ + -ldflags="${LDFLAGS}" \ + -o ${INSTALLPATH}/tendermint ./cmd/tendermint/ + + popd # ${distsrc} + + pushd ${INSTALLPATH} + find -type f | sort | tar \ + --no-recursion --mode='u+rw,go+r-w,a+X' \ + --numeric-owner --sort=name \ + --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-linux-${arch}.tar.gz + popd # installed + done + + rm -rf ${distsrc} + + mkdir -p $OUTDIR/src + mv $SOURCEDIST $OUTDIR/src diff --git a/scripts/gitian-descriptors/gitian-windows.yml b/scripts/gitian-descriptors/gitian-windows.yml new file mode 100644 index 000000000..80b2e60d3 --- /dev/null +++ b/scripts/gitian-descriptors/gitian-windows.yml @@ -0,0 +1,111 @@ +--- +name: "tendermint-windows" +enable_cache: true +distro: "ubuntu" +suites: +- "bionic" +architectures: +- "amd64" +packages: +- "bsdmainutils" +- "build-essential" +- "ca-certificates" +- "curl" +- "debhelper" +- "dpkg-dev" +- "devscripts" +- "fakeroot" +- "git" +- "golang-any" +- "xxd" +- "quilt" +remotes: +- "url": "https://github.com/tendermint/tendermint.git" + "dir": "tendermint" +files: +- "golang-debian-1.12.5-1.tar.gz" +script: | + set -e -o pipefail + + GO_SRC_RELEASE=golang-debian-1.12.5-1 + GO_SRC_TARBALL="${GO_SRC_RELEASE}.tar.gz" + # Compile go and configure the environment + export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" + export BUILD_DIR=`pwd` + tar xf "${GO_SRC_TARBALL}" + rm -f "${GO_SRC_TARBALL}" + [ -d "${GO_SRC_RELEASE}/" ] + mv "${GO_SRC_RELEASE}/" go/ + pushd go/ + QUILT_PATCHES=debian/patches quilt push -a + fakeroot debian/rules build RUN_TESTS=false GOCACHE=/tmp/go-cache + popd + + export GOOS=windows + export GOROOT=${BUILD_DIR}/go + export GOPATH=${BUILD_DIR}/gopath + mkdir -p ${GOPATH}/bin + + export PATH_orig=${PATH} + export PATH=$GOPATH/bin:$GOROOT/bin:$PATH + + export ARCHS='386 amd64' + export GO111MODULE=on + + # Make release tarball + pushd tendermint + VERSION=$(git describe --tags | sed 's/^v//') + COMMIT=$(git rev-parse --short=8 HEAD) + DISTNAME=tendermint-${VERSION} + git archive --format tar.gz --prefix ${DISTNAME}/ -o ${DISTNAME}.tar.gz HEAD + SOURCEDIST=`pwd`/`echo tendermint-*.tar.gz` + popd + + # Correct tar file order + mkdir -p temp + pushd temp + tar xf $SOURCEDIST + rm $SOURCEDIST + find tendermint-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > $SOURCEDIST + popd + + # Prepare GOPATH and install deps + distsrc=${GOPATH}/src/github.com/tendermint/tendermint + mkdir -p ${distsrc} + pushd ${distsrc} + tar --strip-components=1 -xf $SOURCEDIST + go mod download + popd + + # Configure LDFLAGS for reproducible builds + LDFLAGS="-extldflags=-static -buildid=${VERSION} -s -w \ + -X github.com/tendermint/tendermint/version.GitCommit=${COMMIT}" + + # Extract release tarball and build + for arch in ${ARCHS}; do + INSTALLPATH=`pwd`/installed/${DISTNAME}-${arch} + mkdir -p ${INSTALLPATH} + + # Build tendermint binary + pushd ${distsrc} + GOARCH=${arch} GOROOT_FINAL=${GOROOT} go build -a \ + -gcflags=all=-trimpath=${GOPATH} \ + -asmflags=all=-trimpath=${GOPATH} \ + -mod=readonly -tags "tendermint" \ + -ldflags="${LDFLAGS}" \ + -o ${INSTALLPATH}/tendermint.exe ./cmd/tendermint/ + + popd # ${distsrc} + + pushd ${INSTALLPATH} + find -type f | sort | tar \ + --no-recursion --mode='u+rw,go+r-w,a+X' \ + --numeric-owner --sort=name \ + --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-windows-${arch}.tar.gz + popd # installed + done + + rm -rf ${distsrc} + + mkdir -p $OUTDIR/src + mv $SOURCEDIST $OUTDIR/src diff --git a/scripts/gitian-keys/README.md b/scripts/gitian-keys/README.md new file mode 100644 index 000000000..f4ad711a9 --- /dev/null +++ b/scripts/gitian-keys/README.md @@ -0,0 +1,29 @@ +## PGP keys of Gitian builders and Tendermint Developers + +The file `keys.txt` contains fingerprints of the public keys of Gitian builders +and active developers. + +The associated keys are mainly used to sign git commits or the build results +of Gitian builds. + +The most recent version of each pgp key can be found on most PGP key servers. + +Fetch the latest version from the key server to see if any key was revoked in +the meantime. +To fetch the latest version of all pgp keys in your gpg homedir, + +```sh +gpg --refresh-keys +``` + +To fetch keys of Gitian builders and active core developers, feed the list of +fingerprints of the primary keys into gpg: + +```sh +while read fingerprint keyholder_name; \ +do gpg --keyserver hkp://subset.pool.sks-keyservers.net \ +--recv-keys ${fingerprint}; done < ./keys.txt +``` + +Add your key to the list if you are a Tendermint core developer or you have +provided Gitian signatures for two major or minor releases of Tendermint. diff --git a/scripts/gitian-keys/keys.txt b/scripts/gitian-keys/keys.txt new file mode 100644 index 000000000..91330ae0b --- /dev/null +++ b/scripts/gitian-keys/keys.txt @@ -0,0 +1 @@ +04160004A8276E40BB9890FBE8A48AE5311D765A Alessio Treglia