diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7277f36ff..cabaf8b3c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -108,7 +108,24 @@ We use [Protocol Buffers](https://developers.google.com/protocol-buffers) along For linting and checking breaking changes, we use [buf](https://buf.build/). If you would like to run linting and check if the changes you have made are breaking then you will need to have docker running locally. Then the linting cmd will be `make proto-lint` and the breaking changes check will be `make proto-check-breaking`. -We use [Docker](https://www.docker.com/) to generate the protobuf stubs. To generate the stubs yourself, make sure docker is running then run `make proto-gen`. +There are two ways to generate your proto stubs. + +1. Use Docker, pull an image that will generate your proto stubs with no need to install anything. `make proto-gen-docker` +2. Run `make proto-gen` after installing `protoc` and gogoproto, you can do this by running `make protobuf`. + +### Installation Instructions + +To install `protoc`, download an appropriate release () and then move the provided binaries into your PATH (follow instructions in README included with the download). + +To install `gogoproto`, do the following: + +```sh +go get github.com/gogo/protobuf/gogoproto +cd $GOPATH/pkg/mod/github.com/gogo/protobuf@v1.3.1 # or wherever go get installs things +make install +``` + +You should now be able to run `make proto-gen` from inside the root Tendermint directory to generate new files from proto files. ## Vagrant @@ -309,86 +326,12 @@ have distinct names from the tags/release names.) ## Testing -### Unit tests - -Unit tests are located in `_test.go` files as directed by [the Go testing -package](https://golang.org/pkg/testing/). If you're adding or removing a -function, please check there's a `TestType_Method` test for it. - -Run: `make test` - -### Integration tests - -Integration tests are also located in `_test.go` files. What differentiates -them is a more complicated setup, which usually involves setting up two or more -components. - -Run: `make test_integrations` - -### End-to-end tests - -End-to-end tests are used to verify a fully integrated Tendermint network. - -See [README](./test/e2e/README.md) for details. - -Run: - -```sh -cd test/e2e && \ - make && \ - ./build/runner -f networks/ci.toml -``` - -### Maverick - -**If you're changing the code in `consensus` package, please make sure to -replicate all the changes in `./test/maverick/consensus`**. Maverick is a -byzantine node used to assert that the validator gets punished for malicious -behavior. - -See [README](./test/maverick/README.md) for details. - -### Model-based tests (ADVANCED) - -*NOTE: if you're just submitting your first PR, you won't need to touch these -most probably (99.9%)*. - -For components, that have been [formally -verified](https://en.wikipedia.org/wiki/Formal_verification) using -[TLA+](https://en.wikipedia.org/wiki/TLA%2B), it may be possible to generate -tests using a combination of the [Apalache Model -Checker](https://apalache.informal.systems/) and [tendermint-rs testgen -util](https://github.com/informalsystems/tendermint-rs/tree/master/testgen). - -Now, I know there's a lot to take in. If you want to learn more, check out [ -this video](https://www.youtube.com/watch?v=aveoIMphzW8) by Andrey Kupriyanov -& Igor Konnov. +All repos should be hooked up to [CircleCI](https://circleci.com/). -At the moment, we have model-based tests for the light client, located in the -`./light/mbt` directory. - -Run: `cd light/mbt && go test` - -### Fuzz tests (ADVANCED) - -*NOTE: if you're just submitting your first PR, you won't need to touch these -most probably (99.9%)*. - -[Fuzz tests](https://en.wikipedia.org/wiki/Fuzzing) can be found inside the -`./test/fuzz` directory. See [README.md](./test/fuzz/README.md) for details. - -Run: `cd test/fuzz && make fuzz-{PACKAGE-COMPONENT}` - -### Jepsen tests (ADVANCED) - -*NOTE: if you're just submitting your first PR, you won't need to touch these -most probably (99.9%)*. - -[Jepsen](http://jepsen.io/) tests are used to verify the -[linearizability](https://jepsen.io/consistency/models/linearizable) property -of the Tendermint consensus. They are located in a separate repository --> . Please refer to its README for more -information. +If they have `.go` files in the root directory, they will be automatically +tested by circle using `go test -v -race ./...`. If not, they will need a +`circle.yml`. Ideally, every repo has a `Makefile` that defines `make test` and +includes its continuous integration status using a badge in the `README.md`. ### RPC Testing @@ -401,8 +344,4 @@ make build-linux build-contract-tests-hooks make contract-tests ``` -**WARNING: these are currently broken due to -not supporting complete OpenAPI 3**. - -This command will popup a network and check every endpoint against what has -been documented. +This command will popup a network and check every endpoint against what has been documented diff --git a/Makefile b/Makefile index e0b1f3b70..7cb439a1d 100644 --- a/Makefile +++ b/Makefile @@ -49,6 +49,8 @@ LD_FLAGS += $(LDFLAGS) all: check build test install .PHONY: all +# The below include contains the tools. +include tools.mk include tests.mk ############################################################################### @@ -71,10 +73,19 @@ proto-all: proto-gen proto-lint proto-check-breaking .PHONY: proto-all proto-gen: + ## If you get the following error, + ## "error while loading shared libraries: libprotobuf.so.14: cannot open shared object file: No such file or directory" + ## See https://stackoverflow.com/a/25518702 + ## Note the $< here is substituted for the %.proto + ## Note the $@ here is substituted for the %.pb.go + @sh scripts/protocgen.sh +.PHONY: proto-gen + +proto-gen-docker: @docker pull -q tendermintdev/docker-build-proto @echo "Generating Protobuf files" @docker run -v $(shell pwd):/workspace --workdir /workspace tendermintdev/docker-build-proto sh ./scripts/protocgen.sh -.PHONY: proto-gen +.PHONY: proto-gen-docker proto-lint: @$(DOCKER_BUF) check lint --error-format=json diff --git a/docs/app-dev/abci-cli.md b/docs/app-dev/abci-cli.md index 10a5c466d..f19d80abc 100644 --- a/docs/app-dev/abci-cli.md +++ b/docs/app-dev/abci-cli.md @@ -17,6 +17,7 @@ Next, install the `abci-cli` tool and example applications: ```sh git clone https://github.com/tendermint/tendermint.git cd tendermint +make tools make install_abci ``` diff --git a/docs/app-dev/getting-started.md b/docs/app-dev/getting-started.md index 9ba6bab26..132258bf9 100644 --- a/docs/app-dev/getting-started.md +++ b/docs/app-dev/getting-started.md @@ -34,6 +34,7 @@ Then run ```sh go get github.com/tendermint/tendermint cd $GOPATH/src/github.com/tendermint/tendermint +make tools make install_abci ``` diff --git a/docs/introduction/install.md b/docs/introduction/install.md index 72a34306a..7c92c3b05 100644 --- a/docs/introduction/install.md +++ b/docs/introduction/install.md @@ -25,6 +25,12 @@ git clone https://github.com/tendermint/tendermint.git cd tendermint ``` +### Get Tools & Dependencies + +```sh +make tools +``` + ### Compile ```sh diff --git a/tools.mk b/tools.mk new file mode 100644 index 000000000..6e1a61ca0 --- /dev/null +++ b/tools.mk @@ -0,0 +1,111 @@ +### +# Find OS and Go environment +# GO contains the Go binary +# FS contains the OS file separator +### +ifeq ($(OS),Windows_NT) + GO := $(shell where go.exe 2> NUL) + FS := "\\" +else + GO := $(shell command -v go 2> /dev/null) + FS := "/" +endif + +ifeq ($(GO),) + $(error could not find go. Is it in PATH? $(GO)) +endif + +GOPATH ?= $(shell $(GO) env GOPATH) +GITHUBDIR := $(GOPATH)$(FS)src$(FS)github.com + +### +# Functions +### + +go_get = $(if $(findstring Windows_NT,$(OS)),\ +IF NOT EXIST $(GITHUBDIR)$(FS)$(1)$(FS) ( mkdir $(GITHUBDIR)$(FS)$(1) ) else (cd .) &\ +IF NOT EXIST $(GITHUBDIR)$(FS)$(1)$(FS)$(2)$(FS) ( cd $(GITHUBDIR)$(FS)$(1) && git clone https://github.com/$(1)/$(2) ) else (cd .) &\ +,\ +mkdir -p $(GITHUBDIR)$(FS)$(1) &&\ +(test ! -d $(GITHUBDIR)$(FS)$(1)$(FS)$(2) && cd $(GITHUBDIR)$(FS)$(1) && git clone https://github.com/$(1)/$(2)) || true &&\ +)\ +cd $(GITHUBDIR)$(FS)$(1)$(FS)$(2) && git fetch origin && git checkout -q $(3) + +mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST))) +mkfile_dir := $(shell cd $(shell dirname $(mkfile_path)); pwd) + +### +# Go tools +### + +TOOLS_DESTDIR ?= $(GOPATH)/bin + +CERTSTRAP = $(TOOLS_DESTDIR)/certstrap +PROTOBUF = $(TOOLS_DESTDIR)/protoc +GOODMAN = $(TOOLS_DESTDIR)/goodman + +all: tools +.PHONY: all + +tools: certstrap protobuf goodman +.PHONY: tools + +check: check_tools +.PHONY: check + +check_tools: + @# https://stackoverflow.com/a/25668869 + @echo "Found tools: $(foreach tool,$(notdir $(GOTOOLS)),\ + $(if $(shell which $(tool)),$(tool),$(error "No $(tool) in PATH")))" +.PHONY: check_tools + +certstrap: $(CERTSTRAP) +$(CERTSTRAP): + @echo "Get Certstrap" + @go get github.com/square/certstrap@v1.2.0 +.PHONY: certstrap + +protobuf: $(PROTOBUF) +$(PROTOBUF): + @echo "Get GoGo Protobuf" + @go get github.com/gogo/protobuf/protoc-gen-gogofaster@v1.3.1 +.PHONY: protobuf + +goodman: $(GOODMAN) +$(GOODMAN): + @echo "Get Goodman" + @go get github.com/snikch/goodman/cmd/goodman@10e37e294daa3c9a90abded60ff9924bafab3888 +.PHONY: goodman + +tools-clean: + rm -f $(CERTSTRAP) $(PROTOBUF) $(GOX) $(GOODMAN) + rm -f tools-stamp + rm -rf /usr/local/include/google/protobuf + rm -f /usr/local/bin/protoc +.PHONY: tooks-clean + +### +# Non Go tools +### + +# Choose protobuf binary based on OS (only works for 64bit Linux and Mac). +# NOTE: On Mac, installation via brew (brew install protoc) might be favorable. +PROTOC_ZIP="" +ifneq ($(OS),Windows_NT) + UNAME_S := $(shell uname -s) + ifeq ($(UNAME_S),Linux) + PROTOC_ZIP="protoc-3.10.1-linux-x86_64.zip" + endif + ifeq ($(UNAME_S),Darwin) + PROTOC_ZIP="protoc-3.10.1-osx-x86_64.zip" + endif +endif + +protoc: + @echo "Get Protobuf" + @echo "In case of any errors, please install directly from https://github.com/protocolbuffers/protobuf/releases" + @curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.10.1/$(PROTOC_ZIP) + @unzip -o $(PROTOC_ZIP) -d /usr/local bin/protoc + @unzip -o $(PROTOC_ZIP) -d /usr/local 'include/*' + @rm -f $(PROTOC_ZIP) +.PHONY: protoc