custom marshallers for proto types, which EmitDefaults (#3889)

* Remove omitempty from *pb.go

- remove omitempty from *pb.go files
- added command to makefile for everytime `make protoc_all` is run

- open question:
  - Do we want to further remove omitempty from other places
	- https://github.com/tendermint/tendermint/blob/master/rpc/lib/types/types.go#L151
	- and other places
ref #3882

Signed-off-by: Marko Baricevic <marbar3778@yahoo.com>

* bring back omitempty to *pb.go

* Update types/tx.go

* custom marshlers

* undo benchmark `omitepmty`

* golangci lint fix

* cleanup comments

* changelog_pending entry

CHANGELOG_PENDING.md

@@ -21,6 +21,7 @@ program](https://hackerone.com/tendermint).
 
 - [rpc] \#2010 Add NewHTTPWithClient and NewJSONRPCClientWithHTTPClient (note these and NewHTTP, NewJSONRPCClient functions panic if remote is invalid) (@gracenoah)
 - [rpc] \#3984 Add `MempoolClient` interface to `Client` interface
+- [rpc] \#3882 Add custom marshalers to proto messages to disable `omitempty`
 
 ### BUG FIXES:
 

abci/types/messages_test.go

@@ -15,9 +15,8 @@ import (
 func TestMarshalJSON(t *testing.T) {
 	b, err := json.Marshal(&ResponseDeliverTx{})
 	assert.Nil(t, err)
-	// Do not include empty fields.
-	assert.False(t, strings.Contains(string(b), "code"))
-
+	// include empty fields.
+	assert.True(t, strings.Contains(string(b), "code"))
 	r1 := ResponseCheckTx{
 		Code:      1,
 		Data:      []byte("hello"),

abci/types/result.go

@@ -42,14 +42,12 @@ func (r ResponseQuery) IsErr() bool {
 }
 
 //---------------------------------------------------------------------------
-// override JSON marshalling so we dont emit defaults (ie. disable omitempty)
-// note we need Unmarshal functions too because protobuf had the bright idea
-// to marshal int64->string. cool. cool, cool, cool: https://developers.google.com/protocol-buffers/docs/proto3#json
+// override JSON marshalling so we emit defaults (ie. disable omitempty)
 
 var (
 	jsonpbMarshaller = jsonpb.Marshaler{
 		EnumsAsInts:  true,
-		EmitDefaults: false,
+		EmitDefaults: true,
 	}
 	jsonpbUnmarshaller = jsonpb.Unmarshaler{}
 )

crypto/merkle/result.go

@@ -0,0 +1,53 @@
+// nolint: dupl
+package merkle
+
+import (
+	"bytes"
+	"encoding/json"
+
+	"github.com/gogo/protobuf/jsonpb"
+)
+
+//---------------------------------------------------------------------------
+// override JSON marshalling so we emit defaults (ie. disable omitempty)
+
+var (
+	jsonpbMarshaller = jsonpb.Marshaler{
+		EnumsAsInts:  true,
+		EmitDefaults: true,
+	}
+	jsonpbUnmarshaller = jsonpb.Unmarshaler{}
+)
+
+func (r *ProofOp) MarshalJSON() ([]byte, error) {
+	s, err := jsonpbMarshaller.MarshalToString(r)
+	return []byte(s), err
+}
+
+func (r *ProofOp) UnmarshalJSON(b []byte) error {
+	reader := bytes.NewBuffer(b)
+	return jsonpbUnmarshaller.Unmarshal(reader, r)
+}
+
+func (r *Proof) MarshalJSON() ([]byte, error) {
+	s, err := jsonpbMarshaller.MarshalToString(r)
+	return []byte(s), err
+}
+
+func (r *Proof) UnmarshalJSON(b []byte) error {
+	reader := bytes.NewBuffer(b)
+	return jsonpbUnmarshaller.Unmarshal(reader, r)
+}
+
+// Some compile time assertions to ensure we don't
+// have accidental runtime surprises later on.
+// jsonEncodingRoundTripper ensures that asserted
+// interfaces implement both MarshalJSON and UnmarshalJSON
+
+type jsonRoundTripper interface {
+	json.Marshaler
+	json.Unmarshaler
+}
+
+var _ jsonRoundTripper = (*ProofOp)(nil)
+var _ jsonRoundTripper = (*Proof)(nil)

libs/common/result.go

@@ -0,0 +1,54 @@
+// nolint: dupl
+// dupl is reading this as the same file as crypto/merkle/result.go
+package common
+
+import (
+	"bytes"
+	"encoding/json"
+
+	"github.com/gogo/protobuf/jsonpb"
+)
+
+//---------------------------------------------------------------------------
+// override JSON marshalling so we emit defaults (ie. disable omitempty)
+
+var (
+	jsonpbMarshaller = jsonpb.Marshaler{
+		EnumsAsInts:  true,
+		EmitDefaults: true,
+	}
+	jsonpbUnmarshaller = jsonpb.Unmarshaler{}
+)
+
+func (r *KVPair) MarshalJSON() ([]byte, error) {
+	s, err := jsonpbMarshaller.MarshalToString(r)
+	return []byte(s), err
+}
+
+func (r *KVPair) UnmarshalJSON(b []byte) error {
+	reader := bytes.NewBuffer(b)
+	return jsonpbUnmarshaller.Unmarshal(reader, r)
+}
+
+func (r *KI64Pair) MarshalJSON() ([]byte, error) {
+	s, err := jsonpbMarshaller.MarshalToString(r)
+	return []byte(s), err
+}
+
+func (r *KI64Pair) UnmarshalJSON(b []byte) error {
+	reader := bytes.NewBuffer(b)
+	return jsonpbUnmarshaller.Unmarshal(reader, r)
+}
+
+// Some compile time assertions to ensure we don't
+// have accidental runtime surprises later on.
+// jsonEncodingRoundTripper ensures that asserted
+// interfaces implement both MarshalJSON and UnmarshalJSON
+
+type jsonRoundTripper interface {
+	json.Marshaler
+	json.Unmarshaler
+}
+
+var _ jsonRoundTripper = (*KVPair)(nil)
+var _ jsonRoundTripper = (*KI64Pair)(nil)

types/proto3/result.go

@@ -0,0 +1,74 @@
+package proto3
+
+import (
+	"bytes"
+	"encoding/json"
+
+	"github.com/gogo/protobuf/jsonpb"
+)
+
+//---------------------------------------------------------------------------
+// override JSON marshalling so we emit defaults (ie. disable omitempty)
+
+var (
+	jsonpbMarshaller = jsonpb.Marshaler{
+		EnumsAsInts:  true,
+		EmitDefaults: true,
+	}
+	jsonpbUnmarshaller = jsonpb.Unmarshaler{}
+)
+
+func (r *PartSetHeader) MarshalJSON() ([]byte, error) {
+	s, err := jsonpbMarshaller.MarshalToString(r)
+	return []byte(s), err
+}
+
+func (r *PartSetHeader) UnmarshalJSON(b []byte) error {
+	reader := bytes.NewBuffer(b)
+	return jsonpbUnmarshaller.Unmarshal(reader, r)
+}
+
+func (r *Header) MarshalJSON() ([]byte, error) {
+	s, err := jsonpbMarshaller.MarshalToString(r)
+	return []byte(s), err
+}
+
+func (r *Header) UnmarshalJSON(b []byte) error {
+	reader := bytes.NewBuffer(b)
+	return jsonpbUnmarshaller.Unmarshal(reader, r)
+}
+
+func (r *Version) MarshalJSON() ([]byte, error) {
+	s, err := jsonpbMarshaller.MarshalToString(r)
+	return []byte(s), err
+}
+
+func (r *Version) UnmarshalJSON(b []byte) error {
+	reader := bytes.NewBuffer(b)
+	return jsonpbUnmarshaller.Unmarshal(reader, r)
+}
+
+func (r *Timestamp) MarshalJSON() ([]byte, error) {
+	s, err := jsonpbMarshaller.MarshalToString(r)
+	return []byte(s), err
+}
+
+func (r *Timestamp) UnmarshalJSON(b []byte) error {
+	reader := bytes.NewBuffer(b)
+	return jsonpbUnmarshaller.Unmarshal(reader, r)
+}
+
+// Some compile time assertions to ensure we don't
+// have accidental runtime surprises later on.
+// jsonEncodingRoundTripper ensures that asserted
+// interfaces implement both MarshalJSON and UnmarshalJSON
+
+type jsonRoundTripper interface {
+	json.Marshaler
+	json.Unmarshaler
+}
+
+var _ jsonRoundTripper = (*PartSetHeader)(nil)
+var _ jsonRoundTripper = (*Header)(nil)
+var _ jsonRoundTripper = (*Version)(nil)
+var _ jsonRoundTripper = (*Timestamp)(nil)

types/tx.go

@@ -83,9 +83,9 @@ func (txs Txs) Proof(i int) TxProof {
 
 // TxProof represents a Merkle proof of the presence of a transaction in the Merkle tree.
 type TxProof struct {
-	RootHash cmn.HexBytes
-	Data     Tx
-	Proof    merkle.SimpleProof
+	RootHash cmn.HexBytes       `json:"root_hash"`
+	Data     Tx                 `json:"data"`
+	Proof    merkle.SimpleProof `json:"proof"`
 }
 
 // Leaf returns the hash(tx), which is the leaf in the merkle tree which this proof refers to.