Remove Bytes, just use []byte; Use protobuf for KVPair/KI64Pair

Makefile

@@ -1,11 +1,12 @@
-.PHONY: all test get_vendor_deps ensure_tools
-
 GOTOOLS = \
 	github.com/Masterminds/glide \
-	github.com/alecthomas/gometalinter.v2
-GOTOOLS_CHECK = glide gometalinter.v2
+	github.com/alecthomas/gometalinter.v2 \
+	github.com/gogo/protobuf/protoc-gen-gogo \
+	github.com/gogo/protobuf/gogoproto
+GOTOOLS_CHECK = glide gometalinter.v2 protoc protoc-gen-gogo
+INCLUDE = -I=. -I=${GOPATH}/src -I=${GOPATH}/src/github.com/gogo/protobuf/protobuf
 
-all: check get_vendor_deps build test install metalinter
+all: check get_vendor_deps protoc build test install metalinter
 
 check: check_tools
 
@@ -13,6 +14,15 @@ check: check_tools
 ########################################
 ###  Build
 
+protoc:
+	## If you get the following error,
+	## "error while loading shared libraries: libprotobuf.so.14: cannot open shared object file: No such file or directory"
+	## See https://stackoverflow.com/a/25518702
+	protoc $(INCLUDE) --gogo_out=plugins=grpc:. common/*.proto
+	@echo "--> adding nolint declarations to protobuf generated files"
+	@awk '/package common/ { print "//nolint: gas"; print; next }1' common/types.pb.go > common/types.pb.go.new
+	@mv common/types.pb.go.new common/types.pb.go
+
 build:
 	# Nothing to build!
 
@@ -33,6 +43,16 @@ get_tools:
 	go get -u -v $(GOTOOLS)
 	@gometalinter.v2 --install
 
+get_protoc:
+	@# https://github.com/google/protobuf/releases
+	curl -L https://github.com/google/protobuf/releases/download/v3.4.1/protobuf-cpp-3.4.1.tar.gz | tar xvz && \
+		cd protobuf-3.4.1 && \
+		DIST_LANG=cpp ./configure && \
+		make && \
+		make install && \
+		cd .. && \
+		rm -rf protobuf-3.4.1
+
 update_tools:
 	@echo "--> Updating tools"
 	@go get -u $(GOTOOLS)
@@ -93,4 +113,4 @@ metalinter_all:
 # To avoid unintended conflicts with file names, always add to .PHONY
 # unless there is a reason not to.
 # https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html
-.PHONY: check build check_tools get_tools update_tools get_vendor_deps test fmt metalinter metalinter_all
+.PHONY: check protoc build check_tools get_tools get_protoc update_tools get_vendor_deps test fmt metalinter metalinter_all

common/bytes.go

@@ -1,53 +0,0 @@
-package common
-
-import (
-	"encoding/hex"
-	"fmt"
-	"strings"
-)
-
-// The main purpose of Bytes is to enable HEX-encoding for json/encoding.
-type Bytes []byte
-
-// Marshal needed for protobuf compatibility
-func (b Bytes) Marshal() ([]byte, error) {
-	return b, nil
-}
-
-// Unmarshal needed for protobuf compatibility
-func (b *Bytes) Unmarshal(data []byte) error {
-	*b = data
-	return nil
-}
-
-// This is the point of Bytes.
-func (b Bytes) MarshalJSON() ([]byte, error) {
-	s := strings.ToUpper(hex.EncodeToString(b))
-	jb := make([]byte, len(s)+2)
-	jb[0] = '"'
-	copy(jb[1:], []byte(s))
-	jb[1] = '"'
-	return jb, nil
-}
-
-// This is the point of Bytes.
-func (b *Bytes) UnmarshalJSON(data []byte) error {
-	if len(data) < 2 || data[0] != '"' || data[len(data)-1] != '"' {
-		return fmt.Errorf("Invalid hex string: %s", data)
-	}
-	bytes, err := hex.DecodeString(string(data[1 : len(data)-1]))
-	if err != nil {
-		return err
-	}
-	*b = bytes
-	return nil
-}
-
-// Allow it to fulfill various interfaces in light-client, etc...
-func (b Bytes) Bytes() []byte {
-	return b
-}
-
-func (b Bytes) String() string {
-	return strings.ToUpper(hex.EncodeToString(b))
-}

common/bytes_test.go

@@ -1,65 +0,0 @@
-package common
-
-import (
-	"encoding/json"
-	"fmt"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-// This is a trivial test for protobuf compatibility.
-func TestMarshal(t *testing.T) {
-	b := []byte("hello world")
-	dataB := Bytes(b)
-	b2, err := dataB.Marshal()
-	assert.Nil(t, err)
-	assert.Equal(t, b, b2)
-
-	var dataB2 Bytes
-	err = (&dataB2).Unmarshal(b)
-	assert.Nil(t, err)
-	assert.Equal(t, dataB, dataB2)
-}
-
-// Test that the hex encoding works.
-func TestJSONMarshal(t *testing.T) {
-
-	type TestStruct struct {
-		B1 []byte
-		B2 Bytes
-	}
-
-	cases := []struct {
-		input    []byte
-		expected string
-	}{
-		{[]byte(``), `{"B1":"","B2":""}`},
-		{[]byte(``), `{"B1":"","B2":""}`},
-		{[]byte(``), `{"B1":"","B2":""}`},
-	}
-
-	for i, tc := range cases {
-		t.Run(fmt.Sprintf("Case %d", i), func(t *testing.T) {
-			ts := TestStruct{B1: tc.input, B2: tc.input}
-
-			// Test that it marshals correctly to JSON.
-			jsonBytes, err := json.Marshal(ts)
-			if err != nil {
-				t.Fatal(err)
-			}
-			assert.Equal(t, string(jsonBytes), tc.expected)
-
-			// TODO do fuzz testing to ensure that unmarshal fails
-
-			// Test that unmarshaling works correctly.
-			ts2 := TestStruct{}
-			err = json.Unmarshal(jsonBytes, &ts2)
-			if err != nil {
-				t.Fatal(err)
-			}
-			assert.Equal(t, ts2.B1, tc.input)
-			assert.Equal(t, ts2.B2, Bytes(tc.input))
-		})
-	}
-}

common/kvpair.go

@@ -8,10 +8,14 @@ import (
 //----------------------------------------
 // KVPair
 
+/*
+Defined in types.proto
+
 type KVPair struct {
-	Key   Bytes
-	Value Bytes
+	Key   []byte
+	Value []byte
 }
+*/
 
 type KVPairs []KVPair
 
@@ -35,10 +39,13 @@ func (kvs KVPairs) Sort()         { sort.Sort(kvs) }
 //----------------------------------------
 // KI64Pair
 
+/*
+Defined in types.proto
 type KI64Pair struct {
-	Key   Bytes
+	Key   []byte
 	Value int64
 }
+*/
 
 type KI64Pairs []KI64Pair
 

common/types.pb.go

@@ -0,0 +1,101 @@
+// Code generated by protoc-gen-gogo. DO NOT EDIT.
+// source: common/types.proto
+
+/*
+Package common is a generated protocol buffer package.
+
+It is generated from these files:
+	common/types.proto
+
+It has these top-level messages:
+	KVPair
+	KI64Pair
+*/
+//nolint: gas
+package common
+
+import proto "github.com/gogo/protobuf/proto"
+import fmt "fmt"
+import math "math"
+import _ "github.com/gogo/protobuf/gogoproto"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
+
+// Define these here for compatibility but use tmlibs/common.KVPair.
+type KVPair struct {
+	Key   []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
+	Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
+}
+
+func (m *KVPair) Reset()                    { *m = KVPair{} }
+func (m *KVPair) String() string            { return proto.CompactTextString(m) }
+func (*KVPair) ProtoMessage()               {}
+func (*KVPair) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{0} }
+
+func (m *KVPair) GetKey() []byte {
+	if m != nil {
+		return m.Key
+	}
+	return nil
+}
+
+func (m *KVPair) GetValue() []byte {
+	if m != nil {
+		return m.Value
+	}
+	return nil
+}
+
+// Define these here for compatibility but use tmlibs/common.KI64Pair.
+type KI64Pair struct {
+	Key   []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
+	Value int64  `protobuf:"varint,2,opt,name=value,proto3" json:"value,omitempty"`
+}
+
+func (m *KI64Pair) Reset()                    { *m = KI64Pair{} }
+func (m *KI64Pair) String() string            { return proto.CompactTextString(m) }
+func (*KI64Pair) ProtoMessage()               {}
+func (*KI64Pair) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{1} }
+
+func (m *KI64Pair) GetKey() []byte {
+	if m != nil {
+		return m.Key
+	}
+	return nil
+}
+
+func (m *KI64Pair) GetValue() int64 {
+	if m != nil {
+		return m.Value
+	}
+	return 0
+}
+
+func init() {
+	proto.RegisterType((*KVPair)(nil), "common.KVPair")
+	proto.RegisterType((*KI64Pair)(nil), "common.KI64Pair")
+}
+
+func init() { proto.RegisterFile("common/types.proto", fileDescriptorTypes) }
+
+var fileDescriptorTypes = []byte{
+	// 137 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0x4a, 0xce, 0xcf, 0xcd,
+	0xcd, 0xcf, 0xd3, 0x2f, 0xa9, 0x2c, 0x48, 0x2d, 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62,
+	0x83, 0x88, 0x49, 0xe9, 0xa6, 0x67, 0x96, 0x64, 0x94, 0x26, 0xe9, 0x25, 0xe7, 0xe7, 0xea, 0xa7,
+	0xe7, 0xa7, 0xe7, 0xeb, 0x83, 0xa5, 0x93, 0x4a, 0xd3, 0xc0, 0x3c, 0x30, 0x07, 0xcc, 0x82, 0x68,
+	0x53, 0x32, 0xe0, 0x62, 0xf3, 0x0e, 0x0b, 0x48, 0xcc, 0x2c, 0x12, 0x12, 0xe0, 0x62, 0xce, 0x4e,
+	0xad, 0x94, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x09, 0x02, 0x31, 0x85, 0x44, 0xb8, 0x58, 0xcb, 0x12,
+	0x73, 0x4a, 0x53, 0x25, 0x98, 0xc0, 0x62, 0x10, 0x8e, 0x92, 0x11, 0x17, 0x87, 0xb7, 0xa7, 0x99,
+	0x09, 0x31, 0x7a, 0x98, 0xa1, 0x7a, 0x92, 0xd8, 0xc0, 0x96, 0x19, 0x03, 0x02, 0x00, 0x00, 0xff,
+	0xff, 0x5c, 0xb8, 0x46, 0xc5, 0xb9, 0x00, 0x00, 0x00,
+}

common/types.proto

@@ -0,0 +1,24 @@
+syntax = "proto3";
+package common;
+
+// For more information on gogo.proto, see:
+// https://github.com/gogo/protobuf/blob/master/extensions.md
+// NOTE: Try really hard not to use custom types,
+// it's often complicated, broken, nor not worth it.
+import "github.com/gogo/protobuf/gogoproto/gogo.proto";
+
+
+//----------------------------------------
+// Abstract types
+
+// Define these here for compatibility but use tmlibs/common.KVPair.
+message KVPair {
+  bytes key = 1;
+  bytes value = 2;
+}
+
+// Define these here for compatibility but use tmlibs/common.KI64Pair.
+message KI64Pair {
+  bytes key = 1;
+  int64 value = 2;
+}