From 9d4a480f54b319f4e2704a8685c6755f9aa77cf8 Mon Sep 17 00:00:00 2001 From: Marko Date: Wed, 18 Sep 2019 10:45:03 +0200 Subject: [PATCH 1/4] makefile: minor cleanup (#3994) - goimports is not used as a tool anymore - correct me if wrong - rename devtools folder to merely tools.mk - remove slate_header.txt Signed-off-by: Marko Baricevic --- Makefile | 6 +----- rpc/core/README.md | 10 ---------- rpc/core/slate_header.txt | 13 ------------- scripts/devtools/Makefile => tools.mk | 10 ++-------- 4 files changed, 3 insertions(+), 36 deletions(-) delete mode 100644 rpc/core/slate_header.txt rename scripts/devtools/Makefile => tools.mk (86%) diff --git a/Makefile b/Makefile index 6870258f6..e7937cfc6 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ BUILD_FLAGS = -mod=readonly -ldflags "$(LD_FLAGS)" all: check build test install # The below include contains the tools. -include scripts/devtools/Makefile +include tools.mk include tests.mk ######################################## @@ -153,10 +153,6 @@ lint: DESTINATION = ./index.html.md -rpc-docs: - cat rpc/core/slate_header.txt > $(DESTINATION) - godoc2md -template rpc/core/doc_template.txt github.com/tendermint/tendermint/rpc/core | grep -v -e "pipe.go" -e "routes.go" -e "dev.go" | sed 's,/src/target,https://github.com/tendermint/tendermint/tree/master/rpc/core,' >> $(DESTINATION) - ########################################################### ### Docker image diff --git a/rpc/core/README.md b/rpc/core/README.md index 32c3051e3..d767c5f71 100644 --- a/rpc/core/README.md +++ b/rpc/core/README.md @@ -1,15 +1,5 @@ # Tendermint RPC -We are using [Slate](https://github.com/lord/slate) to power our RPC -documentation. For generating markdown use: - -```shell -go get github.com/davecheney/godoc2md - -# from root of this repo -make rpc-docs -``` - ## Pagination Requests that return multiple items will be paginated to 30 items by default. diff --git a/rpc/core/slate_header.txt b/rpc/core/slate_header.txt deleted file mode 100644 index bb4ca6e03..000000000 --- a/rpc/core/slate_header.txt +++ /dev/null @@ -1,13 +0,0 @@ ---- -title: RPC Reference - -language_tabs: # must be one of https://git.io/vQNgJ - - shell - - go - -toc_footers: - - Tendermint - - Documentation Powered by Slate - -search: true ---- diff --git a/scripts/devtools/Makefile b/tools.mk similarity index 86% rename from scripts/devtools/Makefile rename to tools.mk index 1ca24bf9d..f0a8338a6 100644 --- a/scripts/devtools/Makefile +++ b/tools.mk @@ -40,7 +40,6 @@ mkfile_dir := $(shell cd $(shell dirname $(mkfile_path)); pwd) TOOLS_DESTDIR ?= $(GOPATH)/bin -GOIMPORTS = $(TOOLS_DESTDIR)/goimports CERTSTRAP = $(TOOLS_DESTDIR)/certstrap PROTOBUF = $(TOOLS_DESTDIR)/protoc GOX = $(TOOLS_DESTDIR)/gox @@ -48,7 +47,7 @@ GOODMAN = $(TOOLS_DESTDIR)/goodman all: tools -tools: goimports certstrap protobuf gox goodman +tools: certstrap protobuf gox goodman check: check_tools @@ -57,11 +56,6 @@ check_tools: @echo "Found tools: $(foreach tool,$(notdir $(GOTOOLS)),\ $(if $(shell which $(tool)),$(tool),$(error "No $(tool) in PATH")))" -goimports: $(GOIMPORTS) -$(GOIMPORTS): - @echo "Get goimports@v0.0.0-20190628034336-212fb13d595e" - @go get golang.org/x/tools/cmd/goimports@v0.0.0-20190628034336-212fb13d595e - certstrap: $(CERTSTRAP) $(CERTSTRAP): @echo "Get Certstrap" @@ -86,7 +80,7 @@ $(GOODMAN): @go get github.com/snikch/goodman/cmd/goodman@10e37e294daa3c9a90abded60ff9924bafab3888 tools-clean: - rm -f $(CERTSTRAP) $(GOIMPORTS) $(PROTOBUF) $(GOX) $(GOODMAN) + rm -f $(CERTSTRAP) $(PROTOBUF) $(GOX) $(GOODMAN) rm -f tools-stamp .PHONY: all tools tools-clean From 98b91ce2a2f8a0da4c81cd938ef50fe54181e6fe Mon Sep 17 00:00:00 2001 From: Marko Date: Wed, 18 Sep 2019 11:04:40 +0200 Subject: [PATCH 2/4] tm-bench: add deprecation warning (#3992) - Added a deprecation warining in for deprecation of tm-bench in favor of tm-load-test - With the merging of this pr we can close tm-bench related issues. Signed-off-by: Marko Baricevic --- tools/tm-bench/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/tm-bench/README.md b/tools/tm-bench/README.md index d5ed1231f..b69f693f7 100644 --- a/tools/tm-bench/README.md +++ b/tools/tm-bench/README.md @@ -1,4 +1,8 @@ -# tm-bench +# tm-bench (Deprecated) + +> ## **Deprecation Warning** + +### This tool will be depreacted in favor of [tm-load-test](https://github.com/interchainio/tm-load-test). Tendermint blockchain benchmarking tool: From 1b54369f414d31b729165211d78b35bbc41ce007 Mon Sep 17 00:00:00 2001 From: Marko Date: Wed, 18 Sep 2019 11:32:50 +0200 Subject: [PATCH 3/4] custom marshallers for proto types, which EmitDefaults (#3889) * Remove omitempty from *pb.go - remove omitempty from *pb.go files - added command to makefile for everytime `make protoc_all` is run - open question: - Do we want to further remove omitempty from other places - https://github.com/tendermint/tendermint/blob/master/rpc/lib/types/types.go#L151 - and other places ref #3882 Signed-off-by: Marko Baricevic * bring back omitempty to *pb.go * Update types/tx.go * custom marshlers * undo benchmark `omitepmty` * golangci lint fix * cleanup comments * changelog_pending entry --- CHANGELOG_PENDING.md | 1 + abci/types/messages_test.go | 5 +-- abci/types/result.go | 6 +-- crypto/merkle/result.go | 53 ++++++++++++++++++++++++++ libs/common/result.go | 54 +++++++++++++++++++++++++++ types/proto3/result.go | 74 +++++++++++++++++++++++++++++++++++++ types/tx.go | 6 +-- 7 files changed, 189 insertions(+), 10 deletions(-) create mode 100644 crypto/merkle/result.go create mode 100644 libs/common/result.go create mode 100644 types/proto3/result.go diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 3ab1632e3..405e95696 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -21,6 +21,7 @@ program](https://hackerone.com/tendermint). - [rpc] \#2010 Add NewHTTPWithClient and NewJSONRPCClientWithHTTPClient (note these and NewHTTP, NewJSONRPCClient functions panic if remote is invalid) (@gracenoah) - [rpc] \#3984 Add `MempoolClient` interface to `Client` interface +- [rpc] \#3882 Add custom marshalers to proto messages to disable `omitempty` ### BUG FIXES: diff --git a/abci/types/messages_test.go b/abci/types/messages_test.go index 904b16410..b9ba72dcf 100644 --- a/abci/types/messages_test.go +++ b/abci/types/messages_test.go @@ -15,9 +15,8 @@ import ( func TestMarshalJSON(t *testing.T) { b, err := json.Marshal(&ResponseDeliverTx{}) assert.Nil(t, err) - // Do not include empty fields. - assert.False(t, strings.Contains(string(b), "code")) - + // include empty fields. + assert.True(t, strings.Contains(string(b), "code")) r1 := ResponseCheckTx{ Code: 1, Data: []byte("hello"), diff --git a/abci/types/result.go b/abci/types/result.go index dbf409f4c..321e71f08 100644 --- a/abci/types/result.go +++ b/abci/types/result.go @@ -42,14 +42,12 @@ func (r ResponseQuery) IsErr() bool { } //--------------------------------------------------------------------------- -// override JSON marshalling so we dont emit defaults (ie. disable omitempty) -// note we need Unmarshal functions too because protobuf had the bright idea -// to marshal int64->string. cool. cool, cool, cool: https://developers.google.com/protocol-buffers/docs/proto3#json +// override JSON marshalling so we emit defaults (ie. disable omitempty) var ( jsonpbMarshaller = jsonpb.Marshaler{ EnumsAsInts: true, - EmitDefaults: false, + EmitDefaults: true, } jsonpbUnmarshaller = jsonpb.Unmarshaler{} ) diff --git a/crypto/merkle/result.go b/crypto/merkle/result.go new file mode 100644 index 000000000..c7bbb575f --- /dev/null +++ b/crypto/merkle/result.go @@ -0,0 +1,53 @@ +// nolint: dupl +package merkle + +import ( + "bytes" + "encoding/json" + + "github.com/gogo/protobuf/jsonpb" +) + +//--------------------------------------------------------------------------- +// override JSON marshalling so we emit defaults (ie. disable omitempty) + +var ( + jsonpbMarshaller = jsonpb.Marshaler{ + EnumsAsInts: true, + EmitDefaults: true, + } + jsonpbUnmarshaller = jsonpb.Unmarshaler{} +) + +func (r *ProofOp) MarshalJSON() ([]byte, error) { + s, err := jsonpbMarshaller.MarshalToString(r) + return []byte(s), err +} + +func (r *ProofOp) UnmarshalJSON(b []byte) error { + reader := bytes.NewBuffer(b) + return jsonpbUnmarshaller.Unmarshal(reader, r) +} + +func (r *Proof) MarshalJSON() ([]byte, error) { + s, err := jsonpbMarshaller.MarshalToString(r) + return []byte(s), err +} + +func (r *Proof) UnmarshalJSON(b []byte) error { + reader := bytes.NewBuffer(b) + return jsonpbUnmarshaller.Unmarshal(reader, r) +} + +// Some compile time assertions to ensure we don't +// have accidental runtime surprises later on. +// jsonEncodingRoundTripper ensures that asserted +// interfaces implement both MarshalJSON and UnmarshalJSON + +type jsonRoundTripper interface { + json.Marshaler + json.Unmarshaler +} + +var _ jsonRoundTripper = (*ProofOp)(nil) +var _ jsonRoundTripper = (*Proof)(nil) diff --git a/libs/common/result.go b/libs/common/result.go new file mode 100644 index 000000000..90d149f4b --- /dev/null +++ b/libs/common/result.go @@ -0,0 +1,54 @@ +// nolint: dupl +// dupl is reading this as the same file as crypto/merkle/result.go +package common + +import ( + "bytes" + "encoding/json" + + "github.com/gogo/protobuf/jsonpb" +) + +//--------------------------------------------------------------------------- +// override JSON marshalling so we emit defaults (ie. disable omitempty) + +var ( + jsonpbMarshaller = jsonpb.Marshaler{ + EnumsAsInts: true, + EmitDefaults: true, + } + jsonpbUnmarshaller = jsonpb.Unmarshaler{} +) + +func (r *KVPair) MarshalJSON() ([]byte, error) { + s, err := jsonpbMarshaller.MarshalToString(r) + return []byte(s), err +} + +func (r *KVPair) UnmarshalJSON(b []byte) error { + reader := bytes.NewBuffer(b) + return jsonpbUnmarshaller.Unmarshal(reader, r) +} + +func (r *KI64Pair) MarshalJSON() ([]byte, error) { + s, err := jsonpbMarshaller.MarshalToString(r) + return []byte(s), err +} + +func (r *KI64Pair) UnmarshalJSON(b []byte) error { + reader := bytes.NewBuffer(b) + return jsonpbUnmarshaller.Unmarshal(reader, r) +} + +// Some compile time assertions to ensure we don't +// have accidental runtime surprises later on. +// jsonEncodingRoundTripper ensures that asserted +// interfaces implement both MarshalJSON and UnmarshalJSON + +type jsonRoundTripper interface { + json.Marshaler + json.Unmarshaler +} + +var _ jsonRoundTripper = (*KVPair)(nil) +var _ jsonRoundTripper = (*KI64Pair)(nil) diff --git a/types/proto3/result.go b/types/proto3/result.go new file mode 100644 index 000000000..ee5269bd3 --- /dev/null +++ b/types/proto3/result.go @@ -0,0 +1,74 @@ +package proto3 + +import ( + "bytes" + "encoding/json" + + "github.com/gogo/protobuf/jsonpb" +) + +//--------------------------------------------------------------------------- +// override JSON marshalling so we emit defaults (ie. disable omitempty) + +var ( + jsonpbMarshaller = jsonpb.Marshaler{ + EnumsAsInts: true, + EmitDefaults: true, + } + jsonpbUnmarshaller = jsonpb.Unmarshaler{} +) + +func (r *PartSetHeader) MarshalJSON() ([]byte, error) { + s, err := jsonpbMarshaller.MarshalToString(r) + return []byte(s), err +} + +func (r *PartSetHeader) UnmarshalJSON(b []byte) error { + reader := bytes.NewBuffer(b) + return jsonpbUnmarshaller.Unmarshal(reader, r) +} + +func (r *Header) MarshalJSON() ([]byte, error) { + s, err := jsonpbMarshaller.MarshalToString(r) + return []byte(s), err +} + +func (r *Header) UnmarshalJSON(b []byte) error { + reader := bytes.NewBuffer(b) + return jsonpbUnmarshaller.Unmarshal(reader, r) +} + +func (r *Version) MarshalJSON() ([]byte, error) { + s, err := jsonpbMarshaller.MarshalToString(r) + return []byte(s), err +} + +func (r *Version) UnmarshalJSON(b []byte) error { + reader := bytes.NewBuffer(b) + return jsonpbUnmarshaller.Unmarshal(reader, r) +} + +func (r *Timestamp) MarshalJSON() ([]byte, error) { + s, err := jsonpbMarshaller.MarshalToString(r) + return []byte(s), err +} + +func (r *Timestamp) UnmarshalJSON(b []byte) error { + reader := bytes.NewBuffer(b) + return jsonpbUnmarshaller.Unmarshal(reader, r) +} + +// Some compile time assertions to ensure we don't +// have accidental runtime surprises later on. +// jsonEncodingRoundTripper ensures that asserted +// interfaces implement both MarshalJSON and UnmarshalJSON + +type jsonRoundTripper interface { + json.Marshaler + json.Unmarshaler +} + +var _ jsonRoundTripper = (*PartSetHeader)(nil) +var _ jsonRoundTripper = (*Header)(nil) +var _ jsonRoundTripper = (*Version)(nil) +var _ jsonRoundTripper = (*Timestamp)(nil) diff --git a/types/tx.go b/types/tx.go index b71c70029..54ba6bde8 100644 --- a/types/tx.go +++ b/types/tx.go @@ -83,9 +83,9 @@ func (txs Txs) Proof(i int) TxProof { // TxProof represents a Merkle proof of the presence of a transaction in the Merkle tree. type TxProof struct { - RootHash cmn.HexBytes - Data Tx - Proof merkle.SimpleProof + RootHash cmn.HexBytes `json:"root_hash"` + Data Tx `json:"data"` + Proof merkle.SimpleProof `json:"proof"` } // Leaf returns the hash(tx), which is the leaf in the merkle tree which this proof refers to. From 2c7f42fafabef88e3c43dfd6a04cf6e23efe6797 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gustavo=20Cha=C3=ADn?= Date: Wed, 18 Sep 2019 14:10:00 +0200 Subject: [PATCH 4/4] fix Header misalignment in StringIndent (#3996) --- types/block.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/block.go b/types/block.go index 537aed2e7..2dbab1393 100644 --- a/types/block.go +++ b/types/block.go @@ -432,7 +432,7 @@ func (h *Header) StringIndented(indent string) string { %s Validators: %v %s NextValidators: %v %s App: %v -%s Consensus: %v +%s Consensus: %v %s Results: %v %s Evidence: %v %s Proposer: %v