proto: update the mechanism for generating protos from spec repo (#7269)

This pull request updates the `protocgen.sh` script to insert the `go_package` option to all of the downloaded proto files. A related pull request into the spec repo removes this options from the .proto files: https://github.com/tendermint/spec/pull/358

This pull requests, along with the related spec PR, aim to move the creation of the `tendermintdev/docker-build-proto` container into the spec repo. This change also relies on several fixes to that container that are made in the PR into the spec repo.

.github/workflows/proto-docker.yml

@@ -1,51 +0,0 @@
-name: Build & Push TM Proto Builder
-on:
-  pull_request:
-    paths:
-      - "tools/proto/*"
-  push:
-    branches:
-      - master
-    paths:
-      - "tools/proto/*"
-  schedule:
-    # run this job once a month to recieve any go or buf updates
-    - cron: "* * 1 * *"
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2.4.0
-      - name: Prepare
-        id: prep
-        run: |
-          DOCKER_IMAGE=tendermintdev/docker-build-proto
-          VERSION=noop
-          if [[ $GITHUB_REF == refs/tags/* ]]; then
-            VERSION=${GITHUB_REF#refs/tags/}
-          elif [[ $GITHUB_REF == refs/heads/* ]]; then
-            VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g')
-            if [ "${{ github.event.repository.default_branch }}" = "$VERSION" ]; then
-              VERSION=latest
-            fi
-          fi
-          TAGS="${DOCKER_IMAGE}:${VERSION}"
-          echo ::set-output name=tags::${TAGS}
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1.6.0
-
-      - name: Login to DockerHub
-        uses: docker/login-action@v1.10.0
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Publish to Docker Hub
-        uses: docker/build-push-action@v2.7.0
-        with:
-          context: ./tools/proto
-          file: ./tools/proto/Dockerfile
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.prep.outputs.tags }}

Makefile

@@ -15,7 +15,7 @@ endif
 LD_FLAGS = -X github.com/tendermint/tendermint/version.TMVersion=$(VERSION)
 BUILD_FLAGS = -mod=readonly -ldflags "$(LD_FLAGS)"
 HTTPS_GIT := https://github.com/tendermint/tendermint.git
-DOCKER_BUF := docker run -v $(shell pwd):/workspace --workdir /workspace bufbuild/buf
+DOCKER_PROTO_BUILDER := docker run -v $(shell pwd):/workspace --workdir /workspace tendermintdev/docker-build-proto
 CGO_ENABLED ?= 0
 
 # handle nostrip
@@ -85,24 +85,24 @@ proto-all: proto-gen proto-lint proto-check-breaking
 proto-gen:
 	@docker pull -q tendermintdev/docker-build-proto
 	@echo "Generating Protobuf files"
-	@docker run -v $(shell pwd):/workspace --workdir /workspace tendermintdev/docker-build-proto sh ./scripts/protocgen.sh
+	@$(DOCKER_PROTO_BUILDER) sh ./scripts/protocgen.sh
 .PHONY: proto-gen
 
 proto-lint:
-	@$(DOCKER_BUF) lint --error-format=json
+	@$(DOCKER_PROTO_BUILDER) buf lint --error-format=json
 .PHONY: proto-lint
 
 proto-format:
 	@echo "Formatting Protobuf files"
-	docker run -v $(shell pwd):/workspace --workdir /workspace tendermintdev/docker-build-proto find ./ -not -path "./third_party/*" -name *.proto -exec clang-format -i {} \;
+	@$(DOCKER_PROTO_BUILDER) find ./ -not -path "./third_party/*" -name *.proto -exec clang-format -i {} \;
 .PHONY: proto-format
 
 proto-check-breaking:
-	@$(DOCKER_BUF) breaking --against .git#branch=master
+	@$(DOCKER_PROTO_BUILDER) buf breaking --against .git#branch=master
 .PHONY: proto-check-breaking
 
 proto-check-breaking-ci:
-	@$(DOCKER_BUF) breaking --against $(HTTPS_GIT)#branch=master
+	@$(DOCKER_PROTO_BUILDER) buf breaking --against $(HTTPS_GIT)#branch=master
 .PHONY: proto-check-breaking-ci
 
 ###############################################################################

buf.gen.yaml

@@ -1,13 +0,0 @@
-# The version of the generation template.
-# Required.
-# The only currently-valid value is v1beta1.
-version: v1beta1
-
-# The plugins to run.
-plugins:
-  # The name of the plugin.
-  - name: gogofaster
-    # The the relative output directory.
-    out: proto
-    # Any options to provide to the plugin.
-    opt: Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types,Mgoogle/protobuf/duration.proto=github.com/golang/protobuf/ptypes/duration,plugins=grpc,paths=source_relative

buf.yaml

@@ -1,16 +0,0 @@
-version: v1beta1
-
-build:
-  roots:
-    - proto
-    - third_party/proto
-lint:
-  use:
-    - BASIC
-    - FILE_LOWER_SNAKE_CASE
-    - UNARY_RPC
-  ignore:
-    - gogoproto
-breaking:
-  use:
-    - FILE

scripts/protocgen.sh

@@ -1,39 +1,44 @@
 #!/usr/bin/env bash
+set -euo pipefail
 
-set -eo pipefail
+# By default, this script runs against the latest commit to the master branch
+# in the Tendermint spec repository. To use this script with a different version
+# of the spec repository, run it with the $VERS environment variable set to the
+# desired branch name or commit hash from the spec repo.
 
 : ${VERS:=master}
-URL_PATH=archive/
-if [[ VERS -ne master ]]; then
-    URL_PATH=archive/refs/tags/v
-fi
 
-# Edit this line to clone your branch, if you are modifying protobuf files
-curl -qL "https://github.com/tendermint/spec/${URL_PATH}${VERS}.tar.gz" | tar -xjf - spec-"$VERS"/proto/
+echo "fetching proto files"
 
-cp -r ./spec-"$VERS"/proto/tendermint/** ./proto/tendermint
+# Get shortened ref of commit
+REF=$(curl -H "Accept: application/vnd.github.v3.sha" -qL \
+  "https://api.github.com/repos/tendermint/spec/commits/${VERS}" \
+  | cut -c -7)
 
-buf generate --path proto/tendermint
+readonly OUTDIR="tendermint-spec-${REF}"
+curl -qL "https://api.github.com/repos/tendermint/spec/tarball/${REF}" | tar -xzf - ${OUTDIR}/
+
+cp -r ${OUTDIR}/proto/tendermint/* ./proto/tendermint
+cp -r ${OUTDIR}/third_party/** ./third_party
+
+MODNAME="$(go list -m)"
+find ./proto/tendermint -name '*.proto' -not -path "./proto/tendermint/abci/types.proto" \
+	-exec sh ./scripts/protopackage.sh {} "$MODNAME" ';'
+
+# For historical compatibility, the abci file needs to get a slightly different import name
+# so that it can be moved into the ./abci/types directory.
+sh ./scripts/protopackage.sh ./proto/tendermint/abci/types.proto $MODNAME "abci/types"
+
+buf generate --path proto/tendermint --template ./${OUTDIR}/buf.gen.yaml --config ./${OUTDIR}/buf.yaml
 
 mv ./proto/tendermint/abci/types.pb.go ./abci/types
 
-echo "proto files have been generated"
+echo "proto files have been compiled"
 
 echo "removing copied files"
 
-rm -rf ./proto/tendermint/abci
-rm -rf ./proto/tendermint/blocksync/types.proto
-rm -rf ./proto/tendermint/consensus/types.proto
-rm -rf ./proto/tendermint/mempool/types.proto
-rm -rf ./proto/tendermint/p2p/types.proto
-rm -rf ./proto/tendermint/p2p/conn.proto
-rm -rf ./proto/tendermint/p2p/pex.proto
-rm -rf ./proto/tendermint/statesync/types.proto
-rm -rf ./proto/tendermint/types/block.proto
-rm -rf ./proto/tendermint/types/evidence.proto
-rm -rf ./proto/tendermint/types/params.proto
-rm -rf ./proto/tendermint/types/types.proto
-rm -rf ./proto/tendermint/types/validator.proto
-rm -rf ./proto/tendermint/version/types.proto
-
-rm -rf ./spec-"$VERS"
+find ${OUTDIR}/proto/tendermint/ -name *.proto \
+	| sed "s/$OUTDIR\/\(.*\)/\1/g" \
+	| xargs -I {} rm {}
+
+rm -rf ${OUTDIR}

scripts/protopackage.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/sh
+set -eo pipefail
+
+# This script appends the "option go_package" proto option to the file located at $FNAME.
+# This option specifies what the package will be named when imported by other packages.
+# This option is not directly included in the proto files to allow the files to more easily
+# be hosted in github.com/tendermint/spec and shared between other repos.
+# If the option is already specified in the file, it will be replaced using the
+# arguments passed to this script.
+
+FNAME="${1:?missing required .proto filename}"
+MODNAME=$(echo $2| sed 's/\//\\\//g')
+PACKAGE="$(dirname $FNAME | sed 's/^\.\/\(.*\)/\1/g' | sed 's/\//\\\//g')"
+if [[ ! -z "$3" ]]; then
+	PACKAGE="$(echo $3 | sed 's/\//\\\//g')"
+fi
+
+
+if ! grep -q 'option\s\+go_package\s\+=\s\+.*;' $FNAME; then 
+	sed -i "s/\(package tendermint.*\)/\1\n\noption go_package = \"$MODNAME\/$PACKAGE\";/g" $FNAME
+else
+	sed -i "s/option\s\+go_package\s\+=\s\+.*;/option go_package = \"$MODNAME\/$PACKAGE\";/g" $FNAME
+fi

tools/proto/Dockerfile

@@ -1,27 +0,0 @@
-FROM bufbuild/buf:latest as buf
-
-FROM golang:1.14-alpine3.11 as builder
-
-RUN apk add --update --no-cache build-base curl git upx && \
-  rm -rf /var/cache/apk/*
-
-ENV GOLANG_PROTOBUF_VERSION=1.3.1 \
-  GOGO_PROTOBUF_VERSION=1.3.2
-
-RUN GO111MODULE=on go get \
-  github.com/golang/protobuf/protoc-gen-go@v${GOLANG_PROTOBUF_VERSION} \
-  github.com/gogo/protobuf/protoc-gen-gogo@v${GOGO_PROTOBUF_VERSION} \
-  github.com/gogo/protobuf/protoc-gen-gogofaster@v${GOGO_PROTOBUF_VERSION} && \
-  mv /go/bin/protoc-gen-go* /usr/local/bin/
-
-
-FROM alpine:edge
-
-WORKDIR /work
-
-RUN echo 'http://dl-cdn.alpinelinux.org/alpine/edge/testing' >> /etc/apk/repositories && \
-  apk add --update --no-cache clang && \
-  rm -rf /var/cache/apk/*
-
-COPY --from=builder /usr/local/bin /usr/local/bin
-COPY --from=buf /usr/local/bin /usr/local/bin