From a6b22cfa973c6ff17ec90f7f35b30799409f10a7 Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Mon, 5 Oct 2020 17:16:05 +0200 Subject: [PATCH] circleci: remove Gitian reproducible_builds job (#5462) --- .circleci/config.yml | 34 --- scripts/gitian-build.sh | 201 ------------------ scripts/gitian-descriptors/gitian-darwin.yml | 107 ---------- scripts/gitian-descriptors/gitian-linux.yml | 106 --------- scripts/gitian-descriptors/gitian-windows.yml | 107 ---------- scripts/gitian-keys/README.md | 29 --- scripts/gitian-keys/keys.txt | 1 - 7 files changed, 585 deletions(-) delete mode 100755 scripts/gitian-build.sh delete mode 100644 scripts/gitian-descriptors/gitian-darwin.yml delete mode 100644 scripts/gitian-descriptors/gitian-linux.yml delete mode 100644 scripts/gitian-descriptors/gitian-windows.yml delete mode 100644 scripts/gitian-keys/README.md delete mode 100644 scripts/gitian-keys/keys.txt diff --git a/.circleci/config.yml b/.circleci/config.yml index 340e232d9..779d0a288 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -205,34 +205,6 @@ jobs: export GOOS=linux GOARCH=arm64 && python -u scripts/release_management/github-upload.py --id "${RELEASE_ID}" python -u scripts/release_management/github-upload.py --file "/tmp/workspace/SHA256SUMS" --id "${RELEASE_ID}" python -u scripts/release_management/github-publish.py --id "${RELEASE_ID}" - reproducible_builds: - executor: golang - steps: - - attach_workspace: - at: /tmp/workspace - - checkout - - setup_remote_docker: - docker_layer_caching: true - - run: - name: Build tendermint - no_output_timeout: 20m - command: | - sudo apt-get update - sudo apt-get install -y ruby - bash -x ./scripts/gitian-build.sh all - for os in darwin linux windows; do - cp gitian-build-${os}/result/tendermint-${os}-res.yml . - cp gitian-build-${os}/build/out/tendermint-*.tar.gz . - rm -rf gitian-build-${os}/ - done - - store_artifacts: - path: /go/src/github.com/tendermint/tendermint/tendermint-darwin-res.yml - - store_artifacts: - path: /go/src/github.com/tendermint/tendermint/tendermint-linux-res.yml - - store_artifacts: - path: /go/src/github.com/tendermint/tendermint/tendermint-windows-res.yml - - store_artifacts: - path: /go/src/github.com/tendermint/tendermint/tendermint-*.tar.gz # # Test RPC implementation against the swagger documented specs # contract_tests: @@ -287,12 +259,6 @@ workflows: - test_p2p: name: test_p2p_ipv6 ipv: 6 - - reproducible_builds: - filters: - branches: - only: - - master - - /v[0-9]+\.[0-9]+/ # - contract_tests: # requires: # - setup_dependencies diff --git a/scripts/gitian-build.sh b/scripts/gitian-build.sh deleted file mode 100755 index 7471b472f..000000000 --- a/scripts/gitian-build.sh +++ /dev/null @@ -1,201 +0,0 @@ -#!/bin/bash - -# symbol prefixes: -# g_ -> global -# l_ - local variable -# f_ -> function - -set -euo pipefail - -GITIAN_CACHE_DIRNAME='.gitian-builder-cache' -GO_RELEASE='1.13.3' -GO_TARBALL="go${GO_RELEASE}.linux-amd64.tar.gz" -GO_TARBALL_URL="https://dl.google.com/go/${GO_TARBALL}" - -# Defaults - -DEFAULT_SIGN_COMMAND='gpg --detach-sign' -DEFAULT_TENDERMINT_SIGS=${TENDERMINT_SIGS:-'tendermint.sigs'} -DEFAULT_GITIAN_REPO='https://github.com/tendermint/gitian-builder' -DEFAULT_GBUILD_FLAGS='' -DEFAULT_SIGS_REPO='https://github.com/tendermint/tendermint.sigs' - -# Overrides - -SIGN_COMMAND=${SIGN_COMMAND:-${DEFAULT_SIGN_COMMAND}} -GITIAN_REPO=${GITIAN_REPO:-${DEFAULT_GITIAN_REPO}} -GBUILD_FLAGS=${GBUILD_FLAGS:-${DEFAULT_GBUILD_FLAGS}} - -# Globals - -g_workdir='' -g_gitian_cache='' -g_cached_gitian='' -g_cached_go_tarball='' -g_sign_identity='' -g_sigs_dir='' -g_flag_commit='' - - -f_help() { - cat >&2 <&2 - mkdir "${l_builddir}/inputs/" - cp -v "${g_cached_go_tarball}" "${l_builddir}/inputs/" - done -} - -f_build() { - local l_descriptor - - l_descriptor=$1 - - bin/gbuild --commit tendermint="$g_commit" ${GBUILD_FLAGS} "$l_descriptor" - libexec/stop-target || f_echo_stderr "warning: couldn't stop target" -} - -f_sign_verify() { - local l_descriptor - - l_descriptor=$1 - - bin/gsign -p "${SIGN_COMMAND}" -s "${g_sign_identity}" --destination="${g_sigs_dir}" --release=${g_release} ${l_descriptor} - bin/gverify --destination="${g_sigs_dir}" --release="${g_release}" ${l_descriptor} -} - -f_commit_sig() { - local l_release_name - - l_release_name=$1 - - pushd "${g_sigs_dir}" - git add . || echo "git add failed" >&2 - git commit -m "Add ${l_release_name} reproducible build" || echo "git commit failed" >&2 - popd -} - -f_prep_docker_image() { - pushd $1 - bin/make-base-vm --docker --suite bionic --arch amd64 - popd -} - -f_ensure_cache() { - g_gitian_cache="${g_workdir}/${GITIAN_CACHE_DIRNAME}" - [ -d "${g_gitian_cache}" ] || mkdir "${g_gitian_cache}" - - g_cached_go_tarball="${g_gitian_cache}/${GO_TARBALL}" - if [ ! -f "${g_cached_go_tarball}" ]; then - f_echo_stderr "${g_cached_go_tarball}: cache miss, caching..." - curl -L "${GO_TARBALL_URL}" --output "${g_cached_go_tarball}" - fi - - g_cached_gitian="${g_gitian_cache}/gitian-builder" - if [ ! -d "${g_cached_gitian}" ]; then - f_echo_stderr "${g_cached_gitian}: cache miss, caching..." - git clone ${GITIAN_REPO} "${g_cached_gitian}" - fi -} - -f_demangle_platforms() { - case "${1}" in - all) - printf '%s' 'darwin linux windows' ;; - linux|darwin|windows) - printf '%s' "${1}" ;; - *) - echo "invalid platform -- ${1}" - exit 1 - esac -} - -f_echo_stderr() { - echo $@ >&2 -} - - -while getopts ":cs:h" opt; do - case "${opt}" in - h) f_help ; exit 0 ;; - c) g_flag_commit=y ;; - s) g_sign_identity="${OPTARG}" ;; - esac -done - -shift "$((OPTIND-1))" - -g_platforms=$(f_demangle_platforms "${1}") -g_workdir="$(pwd)" -g_commit="$(git rev-parse HEAD)" -g_sigs_dir=${TENDERMINT_SIGS:-"${g_workdir}/${DEFAULT_TENDERMINT_SIGS}"} - -f_ensure_cache - -f_prep_docker_image "${g_cached_gitian}" - -f_prep_build "${g_platforms}" - -export USE_DOCKER=1 -for g_os in ${g_platforms}; do - g_release="$(git describe --tags --abbrev=9 | sed 's/^v//')-${g_os}" - g_descriptor="${g_workdir}/scripts/gitian-descriptors/gitian-${g_os}.yml" - [ -f ${g_descriptor} ] - g_builddir="$(f_builddir ${g_os})" - - pushd "${g_builddir}" - f_build "${g_descriptor}" - if [ -n "${g_sign_identity}" ]; then - f_sign_verify "${g_descriptor}" - fi - popd - - if [ -n "${g_sign_identity}" -a -n "${g_flag_commit}" ]; then - [ -d "${g_sigs_dir}/.git/" ] && f_commit_sig ${g_release} || f_echo_stderr "couldn't commit, ${g_sigs_dir} is not a git clone" - fi -done - -exit 0 diff --git a/scripts/gitian-descriptors/gitian-darwin.yml b/scripts/gitian-descriptors/gitian-darwin.yml deleted file mode 100644 index 90a9fb9d4..000000000 --- a/scripts/gitian-descriptors/gitian-darwin.yml +++ /dev/null @@ -1,107 +0,0 @@ ---- -name: "tendermint-darwin" -enable_cache: true -distro: "ubuntu" -suites: -- "bionic" -architectures: -- "amd64" -packages: -- "bsdmainutils" -- "build-essential" -- "ca-certificates" -- "curl" -- "debhelper" -- "dpkg-dev" -- "devscripts" -- "fakeroot" -- "git" -- "golang-any" -- "xxd" -- "quilt" -remotes: -- "url": "https://github.com/tendermint/tendermint.git" - "dir": "tendermint" -files: -- "go1.13.3.linux-amd64.tar.gz" -script: | - set -e -o pipefail - - GO_SRC_RELEASE=go1.13.3.linux-amd64 - GO_SRC_TARBALL="${GO_SRC_RELEASE}.tar.gz" - # Compile go and configure the environment - export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" - export BUILD_DIR=`pwd` - tar xf "${GO_SRC_TARBALL}" - rm -f "${GO_SRC_TARBALL}" - [ -d go/ ] - - export GOOS=darwin - export GOROOT=${BUILD_DIR}/go - export GOPATH=${BUILD_DIR}/gopath - mkdir -p ${GOPATH}/bin - - export PATH_orig=${PATH} - export PATH=$GOPATH/bin:$GOROOT/bin:$PATH - - export ARCHS='386 amd64' - export GO111MODULE=on - - # Make release tarball - pushd tendermint - VERSION=$(git describe --tags | sed 's/^v//') - COMMIT=$(git rev-parse --short=8 HEAD) - DISTNAME=tendermint-${VERSION} - git archive --format tar.gz --prefix ${DISTNAME}/ -o ${DISTNAME}.tar.gz HEAD - SOURCEDIST=`pwd`/`echo tendermint-*.tar.gz` - popd - - # Correct tar file order - mkdir -p temp - pushd temp - tar xf $SOURCEDIST - rm $SOURCEDIST - find tendermint-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > $SOURCEDIST - popd - - # Prepare GOPATH and install deps - distsrc=${GOPATH}/src/github.com/tendermint/tendermint - mkdir -p ${distsrc} - pushd ${distsrc} - tar --strip-components=1 -xf $SOURCEDIST - go mod download - popd - - # Configure LDFLAGS for reproducible builds - LDFLAGS="-extldflags=-static -buildid=${VERSION} -s -w \ - -X github.com/tendermint/tendermint/version.GitCommit=${COMMIT}" - - # Extract release tarball and build - for arch in ${ARCHS}; do - INSTALLPATH=`pwd`/installed/${DISTNAME}-${arch} - mkdir -p ${INSTALLPATH} - - # Build tendermint binary - pushd ${distsrc} - GOARCH=${arch} GOROOT_FINAL=${GOROOT} go build -a \ - -trimpath \ - -gcflags=all=-trimpath=${GOPATH} \ - -asmflags=all=-trimpath=${GOPATH} \ - -mod=readonly -tags "tendermint" \ - -ldflags="${LDFLAGS}" \ - -o ${INSTALLPATH}/tendermint ./cmd/tendermint/ - - popd # ${distsrc} - - pushd ${INSTALLPATH} - find -type f | sort | tar \ - --no-recursion --mode='u+rw,go+r-w,a+X' \ - --numeric-owner --sort=name \ - --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-darwin-${arch}.tar.gz - popd # installed - done - - rm -rf ${distsrc} - - mkdir -p $OUTDIR/src - mv $SOURCEDIST $OUTDIR/src diff --git a/scripts/gitian-descriptors/gitian-linux.yml b/scripts/gitian-descriptors/gitian-linux.yml deleted file mode 100644 index 8aab869ee..000000000 --- a/scripts/gitian-descriptors/gitian-linux.yml +++ /dev/null @@ -1,106 +0,0 @@ ---- -name: "tendermint-linux" -enable_cache: true -distro: "ubuntu" -suites: -- "bionic" -architectures: -- "amd64" -packages: -- "bsdmainutils" -- "build-essential" -- "ca-certificates" -- "curl" -- "debhelper" -- "dpkg-dev" -- "devscripts" -- "fakeroot" -- "git" -- "golang-any" -- "xxd" -- "quilt" -remotes: -- "url": "https://github.com/tendermint/tendermint.git" - "dir": "tendermint" -files: -- "go1.13.3.linux-amd64.tar.gz" -script: | - set -e -o pipefail - - GO_SRC_RELEASE=go1.13.3.linux-amd64 - GO_SRC_TARBALL="${GO_SRC_RELEASE}.tar.gz" - # Compile go and configure the environment - export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" - export BUILD_DIR=`pwd` - tar xf "${GO_SRC_TARBALL}" - rm -f "${GO_SRC_TARBALL}" - [ -d go/ ] - - export GOROOT=${BUILD_DIR}/go - export GOPATH=${BUILD_DIR}/gopath - mkdir -p ${GOPATH}/bin - - export PATH_orig=${PATH} - export PATH=$GOPATH/bin:$GOROOT/bin:$PATH - - export ARCHS='386 amd64 arm arm64' - export GO111MODULE=on - - # Make release tarball - pushd tendermint - VERSION=$(git describe --tags | sed 's/^v//') - COMMIT=$(git rev-parse --short=8 HEAD) - DISTNAME=tendermint-${VERSION} - git archive --format tar.gz --prefix ${DISTNAME}/ -o ${DISTNAME}.tar.gz HEAD - SOURCEDIST=`pwd`/`echo tendermint-*.tar.gz` - popd - - # Correct tar file order - mkdir -p temp - pushd temp - tar xf $SOURCEDIST - rm $SOURCEDIST - find tendermint-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > $SOURCEDIST - popd - - # Prepare GOPATH and install deps - distsrc=${GOPATH}/src/github.com/tendermint/tendermint - mkdir -p ${distsrc} - pushd ${distsrc} - tar --strip-components=1 -xf $SOURCEDIST - go mod download - popd - - # Configure LDFLAGS for reproducible builds - LDFLAGS="-extldflags=-static -buildid=${VERSION} -s -w \ - -X github.com/tendermint/tendermint/version.GitCommit=${COMMIT}" - - # Extract release tarball and build - for arch in ${ARCHS}; do - INSTALLPATH=`pwd`/installed/${DISTNAME}-${arch} - mkdir -p ${INSTALLPATH} - - # Build tendermint binary - pushd ${distsrc} - GOARCH=${arch} GOROOT_FINAL=${GOROOT} go build -a \ - -trimpath \ - -gcflags=all=-trimpath=${GOPATH} \ - -asmflags=all=-trimpath=${GOPATH} \ - -mod=readonly -tags "tendermint" \ - -ldflags="${LDFLAGS}" \ - -o ${INSTALLPATH}/tendermint ./cmd/tendermint/ - - popd # ${distsrc} - - pushd ${INSTALLPATH} - find -type f | sort | tar \ - --no-recursion --mode='u+rw,go+r-w,a+X' \ - --numeric-owner --sort=name \ - --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-linux-${arch}.tar.gz - popd # installed - done - - rm -rf ${distsrc} - - mkdir -p $OUTDIR/src - mv $SOURCEDIST $OUTDIR/src diff --git a/scripts/gitian-descriptors/gitian-windows.yml b/scripts/gitian-descriptors/gitian-windows.yml deleted file mode 100644 index 23dbdab2f..000000000 --- a/scripts/gitian-descriptors/gitian-windows.yml +++ /dev/null @@ -1,107 +0,0 @@ ---- -name: "tendermint-windows" -enable_cache: true -distro: "ubuntu" -suites: -- "bionic" -architectures: -- "amd64" -packages: -- "bsdmainutils" -- "build-essential" -- "ca-certificates" -- "curl" -- "debhelper" -- "dpkg-dev" -- "devscripts" -- "fakeroot" -- "git" -- "golang-any" -- "xxd" -- "quilt" -remotes: -- "url": "https://github.com/tendermint/tendermint.git" - "dir": "tendermint" -files: -- "go1.13.3.linux-amd64.tar.gz" -script: | - set -e -o pipefail - - GO_SRC_RELEASE=go1.13.3.linux-amd64 - GO_SRC_TARBALL="${GO_SRC_RELEASE}.tar.gz" - # Compile go and configure the environment - export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" - export BUILD_DIR=`pwd` - tar xf "${GO_SRC_TARBALL}" - rm -f "${GO_SRC_TARBALL}" - [ -d go/ ] - - export GOOS=windows - export GOROOT=${BUILD_DIR}/go - export GOPATH=${BUILD_DIR}/gopath - mkdir -p ${GOPATH}/bin - - export PATH_orig=${PATH} - export PATH=$GOPATH/bin:$GOROOT/bin:$PATH - - export ARCHS='386 amd64' - export GO111MODULE=on - - # Make release tarball - pushd tendermint - VERSION=$(git describe --tags | sed 's/^v//') - COMMIT=$(git rev-parse --short=8 HEAD) - DISTNAME=tendermint-${VERSION} - git archive --format tar.gz --prefix ${DISTNAME}/ -o ${DISTNAME}.tar.gz HEAD - SOURCEDIST=`pwd`/`echo tendermint-*.tar.gz` - popd - - # Correct tar file order - mkdir -p temp - pushd temp - tar xf $SOURCEDIST - rm $SOURCEDIST - find tendermint-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > $SOURCEDIST - popd - - # Prepare GOPATH and install deps - distsrc=${GOPATH}/src/github.com/tendermint/tendermint - mkdir -p ${distsrc} - pushd ${distsrc} - tar --strip-components=1 -xf $SOURCEDIST - go mod download - popd - - # Configure LDFLAGS for reproducible builds - LDFLAGS="-extldflags=-static -buildid=${VERSION} -s -w \ - -X github.com/tendermint/tendermint/version.GitCommit=${COMMIT}" - - # Extract release tarball and build - for arch in ${ARCHS}; do - INSTALLPATH=`pwd`/installed/${DISTNAME}-${arch} - mkdir -p ${INSTALLPATH} - - # Build tendermint binary - pushd ${distsrc} - GOARCH=${arch} GOROOT_FINAL=${GOROOT} go build -a \ - -trimpath \ - -gcflags=all=-trimpath=${GOPATH} \ - -asmflags=all=-trimpath=${GOPATH} \ - -mod=readonly -tags "tendermint" \ - -ldflags="${LDFLAGS}" \ - -o ${INSTALLPATH}/tendermint.exe ./cmd/tendermint/ - - popd # ${distsrc} - - pushd ${INSTALLPATH} - find -type f | sort | tar \ - --no-recursion --mode='u+rw,go+r-w,a+X' \ - --numeric-owner --sort=name \ - --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-windows-${arch}.tar.gz - popd # installed - done - - rm -rf ${distsrc} - - mkdir -p $OUTDIR/src - mv $SOURCEDIST $OUTDIR/src diff --git a/scripts/gitian-keys/README.md b/scripts/gitian-keys/README.md deleted file mode 100644 index 2ed7e02dd..000000000 --- a/scripts/gitian-keys/README.md +++ /dev/null @@ -1,29 +0,0 @@ -## PGP keys of Gitian builders and Tendermint Developers - -The file `keys.txt` contains fingerprints of the public keys of Gitian builders -and active developers. - -The associated keys are mainly used to sign git commits or the build results -of Gitian builds. - -The most recent version of each pgp key can be found on most PGP key servers. - -Fetch the latest version from the key server to see if any key was revoked in -the meantime. -To fetch the latest version of all pgp keys in your gpg homedir, - -```bash -gpg --refresh-keys -``` - -To fetch keys of Gitian builders and active core developers, feed the list of -fingerprints of the primary keys into gpg: - -```bash -while read fingerprint keyholder_name; \ -do gpg --keyserver hkp://subset.pool.sks-keyservers.net \ ---recv-keys ${fingerprint}; done < ./keys.txt -``` - -Add your key to the list if you are a Tendermint core developer or you have -provided Gitian signatures for two major or minor releases of Tendermint. diff --git a/scripts/gitian-keys/keys.txt b/scripts/gitian-keys/keys.txt deleted file mode 100644 index 91330ae0b..000000000 --- a/scripts/gitian-keys/keys.txt +++ /dev/null @@ -1 +0,0 @@ -04160004A8276E40BB9890FBE8A48AE5311D765A Alessio Treglia