zolfa
/
tendermint
1
0
Fork 0
Code Issues 0 Pull Requests 0 Projects 0 Releases 221 Wiki Activity
Browse Source

proto: update the mechanism for generating protos from spec repo (#7269) 

This pull request updates the `protocgen.sh` script to insert the `go_package` option to all of the downloaded proto files. A related pull request into the spec repo removes this options from the .proto files: https://github.com/tendermint/spec/pull/358

This pull requests, along with the related spec PR, aim to move the creation of the `tendermintdev/docker-build-proto` container into the spec repo. This change also relies on several fixes to that container that are made in the PR into the spec repo.
pull/7289/head
William Banfield 3 years ago
committed by GitHub
parent
2a455be46c
commit
d5df412b26
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 60 additions and 139 deletions
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +0
    -51
      .github/workflows/proto-docker.yml
  2. +6
    -6
      Makefile
  3. +0
    -13
      buf.gen.yaml
  4. +0
    -16
      buf.yaml
  5. +31
    -26
      scripts/protocgen.sh
  6. +23
    -0
      scripts/protopackage.sh
  7. +0
    -27
      tools/proto/Dockerfile

+ 0
- 51
.github/workflows/proto-docker.yml View File

@ -1,51 +0,0 @@
name: Build & Push TM Proto Builder
on:
pull_request:
paths:
- "tools/proto/*"
push:
branches:
- master
paths:
- "tools/proto/*"
schedule:
# run this job once a month to recieve any go or buf updates
- cron: "* * 1 * *"
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2.4.0
- name: Prepare
id: prep
run: |
DOCKER_IMAGE=tendermintdev/docker-build-proto
VERSION=noop
if [[ $GITHUB_REF == refs/tags/* ]]; then
VERSION=${GITHUB_REF#refs/tags/}
elif [[ $GITHUB_REF == refs/heads/* ]]; then
VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g')
if [ "${{ github.event.repository.default_branch }}" = "$VERSION" ]; then
VERSION=latest
fi
fi
TAGS="${DOCKER_IMAGE}:${VERSION}"
echo ::set-output name=tags::${TAGS}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1.6.0
- name: Login to DockerHub
uses: docker/login-action@v1.10.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Publish to Docker Hub
uses: docker/build-push-action@v2.7.0
with:
context: ./tools/proto
file: ./tools/proto/Dockerfile
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.prep.outputs.tags }}

+ 6
- 6
Makefile View File

@ -15,7 +15,7 @@ endif
LD_FLAGS = -X github.com/tendermint/tendermint/version.TMVersion=$(VERSION) LD_FLAGS = -X github.com/tendermint/tendermint/version.TMVersion=$(VERSION)
BUILD_FLAGS = -mod=readonly -ldflags "$(LD_FLAGS)" BUILD_FLAGS = -mod=readonly -ldflags "$(LD_FLAGS)"
HTTPS_GIT := https://github.com/tendermint/tendermint.git HTTPS_GIT := https://github.com/tendermint/tendermint.git
DOCKER_BUF := docker run -v $(shell pwd):/workspace --workdir /workspace bufbuild/buf
DOCKER_PROTO_BUILDER := docker run -v $(shell pwd):/workspace --workdir /workspace tendermintdev/docker-build-proto
CGO_ENABLED ?= 0 CGO_ENABLED ?= 0
# handle nostrip # handle nostrip
@ -85,24 +85,24 @@ proto-all: proto-gen proto-lint proto-check-breaking
proto-gen: proto-gen:
@docker pull -q tendermintdev/docker-build-proto @docker pull -q tendermintdev/docker-build-proto
@echo "Generating Protobuf files" @echo "Generating Protobuf files"
@docker run -v $(shell pwd):/workspace --workdir /workspace tendermintdev/docker-build-proto sh ./scripts/protocgen.sh
@$(DOCKER_PROTO_BUILDER) sh ./scripts/protocgen.sh
.PHONY: proto-gen .PHONY: proto-gen
proto-lint: proto-lint:
@$(DOCKER_BUF) lint --error-format=json
@$(DOCKER_PROTO_BUILDER) buf lint --error-format=json
.PHONY: proto-lint .PHONY: proto-lint
proto-format: proto-format:
@echo "Formatting Protobuf files" @echo "Formatting Protobuf files"
docker run -v $(shell pwd):/workspace --workdir /workspace tendermintdev/docker-build-proto find ./ -not -path "./third_party/*" -name *.proto -exec clang-format -i {} \;
@$(DOCKER_PROTO_BUILDER) find ./ -not -path "./third_party/*" -name *.proto -exec clang-format -i {} \;
.PHONY: proto-format .PHONY: proto-format
proto-check-breaking: proto-check-breaking:
@$(DOCKER_BUF) breaking --against .git#branch=master
@$(DOCKER_PROTO_BUILDER) buf breaking --against .git#branch=master
.PHONY: proto-check-breaking .PHONY: proto-check-breaking
proto-check-breaking-ci: proto-check-breaking-ci:
@$(DOCKER_BUF) breaking --against $(HTTPS_GIT)#branch=master
@$(DOCKER_PROTO_BUILDER) buf breaking --against $(HTTPS_GIT)#branch=master
.PHONY: proto-check-breaking-ci .PHONY: proto-check-breaking-ci
############################################################################### ###############################################################################


+ 0
- 13
buf.gen.yaml View File

@ -1,13 +0,0 @@
# The version of the generation template.
# Required.
# The only currently-valid value is v1beta1.
version: v1beta1
# The plugins to run.
plugins:
# The name of the plugin.
- name: gogofaster
# The the relative output directory.
out: proto
# Any options to provide to the plugin.
opt: Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types,Mgoogle/protobuf/duration.proto=github.com/golang/protobuf/ptypes/duration,plugins=grpc,paths=source_relative

+ 0
- 16
buf.yaml View File

@ -1,16 +0,0 @@
version: v1beta1
build:
roots:
- proto
- third_party/proto
lint:
use:
- BASIC
- FILE_LOWER_SNAKE_CASE
- UNARY_RPC
ignore:
- gogoproto
breaking:
use:
- FILE

+ 31
- 26
scripts/protocgen.sh View File

@ -1,39 +1,44 @@
#!/usr/bin/env bash #!/usr/bin/env bash
set -euo pipefail
set -eo pipefail
# By default, this script runs against the latest commit to the master branch
# in the Tendermint spec repository. To use this script with a different version
# of the spec repository, run it with the $VERS environment variable set to the
# desired branch name or commit hash from the spec repo.
: ${VERS:=master} : ${VERS:=master}
URL_PATH=archive/
if [[ VERS -ne master ]]; then
URL_PATH=archive/refs/tags/v
fi
# Edit this line to clone your branch, if you are modifying protobuf files
curl -qL "https://github.com/tendermint/spec/${URL_PATH}${VERS}.tar.gz" | tar -xjf - spec-"$VERS"/proto/
echo "fetching proto files"
cp -r ./spec-"$VERS"/proto/tendermint/** ./proto/tendermint
# Get shortened ref of commit
REF=$(curl -H "Accept: application/vnd.github.v3.sha" -qL \
"https://api.github.com/repos/tendermint/spec/commits/${VERS}" \
| cut -c -7)
buf generate --path proto/tendermint
readonly OUTDIR="tendermint-spec-${REF}"
curl -qL "https://api.github.com/repos/tendermint/spec/tarball/${REF}" | tar -xzf - ${OUTDIR}/
cp -r ${OUTDIR}/proto/tendermint/* ./proto/tendermint
cp -r ${OUTDIR}/third_party/** ./third_party
MODNAME="$(go list -m)"
find ./proto/tendermint -name '*.proto' -not -path "./proto/tendermint/abci/types.proto" \
-exec sh ./scripts/protopackage.sh {} "$MODNAME" ';'
# For historical compatibility, the abci file needs to get a slightly different import name
# so that it can be moved into the ./abci/types directory.
sh ./scripts/protopackage.sh ./proto/tendermint/abci/types.proto $MODNAME "abci/types"
buf generate --path proto/tendermint --template ./${OUTDIR}/buf.gen.yaml --config ./${OUTDIR}/buf.yaml
mv ./proto/tendermint/abci/types.pb.go ./abci/types mv ./proto/tendermint/abci/types.pb.go ./abci/types
echo "proto files have been generated"
echo "proto files have been compiled"
echo "removing copied files" echo "removing copied files"
rm -rf ./proto/tendermint/abci
rm -rf ./proto/tendermint/blocksync/types.proto
rm -rf ./proto/tendermint/consensus/types.proto
rm -rf ./proto/tendermint/mempool/types.proto
rm -rf ./proto/tendermint/p2p/types.proto
rm -rf ./proto/tendermint/p2p/conn.proto
rm -rf ./proto/tendermint/p2p/pex.proto
rm -rf ./proto/tendermint/statesync/types.proto
rm -rf ./proto/tendermint/types/block.proto
rm -rf ./proto/tendermint/types/evidence.proto
rm -rf ./proto/tendermint/types/params.proto
rm -rf ./proto/tendermint/types/types.proto
rm -rf ./proto/tendermint/types/validator.proto
rm -rf ./proto/tendermint/version/types.proto
rm -rf ./spec-"$VERS"
find ${OUTDIR}/proto/tendermint/ -name *.proto \
| sed "s/$OUTDIR\/\(.*\)/\1/g" \
| xargs -I {} rm {}
rm -rf ${OUTDIR}

+ 23
- 0
scripts/protopackage.sh View File

@ -0,0 +1,23 @@
#!/usr/bin/sh
set -eo pipefail
# This script appends the "option go_package" proto option to the file located at $FNAME.
# This option specifies what the package will be named when imported by other packages.
# This option is not directly included in the proto files to allow the files to more easily
# be hosted in github.com/tendermint/spec and shared between other repos.
# If the option is already specified in the file, it will be replaced using the
# arguments passed to this script.
FNAME="${1:?missing required .proto filename}"
MODNAME=$(echo $2| sed 's/\//\\\//g')
PACKAGE="$(dirname $FNAME | sed 's/^\.\/\(.*\)/\1/g' | sed 's/\//\\\//g')"
if [[ ! -z "$3" ]]; then
PACKAGE="$(echo $3 | sed 's/\//\\\//g')"
fi
if ! grep -q 'option\s\+go_package\s\+=\s\+.*;' $FNAME; then
sed -i "s/\(package tendermint.*\)/\1\n\noption go_package = \"$MODNAME\/$PACKAGE\";/g" $FNAME
else
sed -i "s/option\s\+go_package\s\+=\s\+.*;/option go_package = \"$MODNAME\/$PACKAGE\";/g" $FNAME
fi

+ 0
- 27
tools/proto/Dockerfile View File

@ -1,27 +0,0 @@
FROM bufbuild/buf:latest as buf
FROM golang:1.14-alpine3.11 as builder
RUN apk add --update --no-cache build-base curl git upx && \
rm -rf /var/cache/apk/*
ENV GOLANG_PROTOBUF_VERSION=1.3.1 \
GOGO_PROTOBUF_VERSION=1.3.2
RUN GO111MODULE=on go get \
github.com/golang/protobuf/protoc-gen-go@v${GOLANG_PROTOBUF_VERSION} \
github.com/gogo/protobuf/protoc-gen-gogo@v${GOGO_PROTOBUF_VERSION} \
github.com/gogo/protobuf/protoc-gen-gogofaster@v${GOGO_PROTOBUF_VERSION} && \
mv /go/bin/protoc-gen-go* /usr/local/bin/
FROM alpine:edge
WORKDIR /work
RUN echo 'http://dl-cdn.alpinelinux.org/alpine/edge/testing' >> /etc/apk/repositories && \
apk add --update --no-cache clang && \
rm -rf /var/cache/apk/*
COPY --from=builder /usr/local/bin /usr/local/bin
COPY --from=buf /usr/local/bin /usr/local/bin

Write Preview
Loading…
Cancel
Save