zolfa
/
tendermint
1
0
Fork 0
Code Issues 0 Pull Requests 0 Projects 0 Releases 221 Wiki Activity
Browse Source

json: add Amino-compatible encoder/decoder (#4955) 

Amino-compatible JSON encoder/decoder, including bug compatibility. Interface types must be registered via `json.RegisterType()`. Unlike Amino, this allows floats to be encoded/decoded.

Partial fix for #4828, needs code migration.
pull/4968/head
Erik Grinaker 5 years ago
committed by GitHub
parent
7a8224f8a3
commit
ccc990498d
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 1173 additions and 0 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +278
    -0
      libs/json/decoder.go
  2. +151
    -0
      libs/json/decoder_test.go
  3. +99
    -0
      libs/json/doc.go
  4. +254
    -0
      libs/json/encoder.go
  5. +104
    -0
      libs/json/encoder_test.go
  6. +92
    -0
      libs/json/helpers_test.go
  7. +87
    -0
      libs/json/structs.go
  8. +108
    -0
      libs/json/types.go

+ 278
- 0
libs/json/decoder.go View File

@ -0,0 +1,278 @@
package json
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"reflect"
)
// Unmarshal unmarshals JSON into the given value, using Amino-compatible JSON encoding (strings
// for 64-bit numbers, and type wrappers for registered types).
func Unmarshal(bz []byte, v interface{}) error {
return decode(bz, v)
}
func decode(bz []byte, v interface{}) error {
if len(bz) == 0 {
return errors.New("cannot decode empty bytes")
}
rv := reflect.ValueOf(v)
if rv.Kind() != reflect.Ptr {
return errors.New("must decode into a pointer")
}
rv = rv.Elem()
// If this is a registered type, defer to interface decoder regardless of whether the input is
// an interface or a bare value. This retains Amino's behavior, but is inconsistent with
// behavior in structs where an interface field will get the type wrapper while a bare value
// field will not.
if typeRegistry.name(rv.Type()) != "" {
return decodeReflectInterface(bz, rv)
}
return decodeReflect(bz, rv)
}
func decodeReflect(bz []byte, rv reflect.Value) error {
if !rv.CanAddr() {
return errors.New("value is not addressable")
}
// Handle null for slices, interfaces, and pointers
if bytes.Equal(bz, []byte("null")) {
rv.Set(reflect.Zero(rv.Type()))
return nil
}
// Dereference-and-construct pointers, to handle nested pointers.
for rv.Kind() == reflect.Ptr {
if rv.IsNil() {
rv.Set(reflect.New(rv.Type().Elem()))
}
rv = rv.Elem()
}
// Times must be UTC and end with Z
if rv.Type() == timeType {
switch {
case len(bz) < 2 || bz[0] != '"' || bz[len(bz)-1] != '"':
return fmt.Errorf("JSON time must be an RFC3339 string, but got %q", bz)
case bz[len(bz)-2] != 'Z':
return fmt.Errorf("JSON time must be UTC and end with 'Z', but got %q", bz)
}
}
// If value implements json.Umarshaler, call it.
if rv.Addr().Type().Implements(jsonUnmarshalerType) {
return rv.Addr().Interface().(json.Unmarshaler).UnmarshalJSON(bz)
}
switch rv.Type().Kind() {
// Decode complex types recursively.
case reflect.Slice, reflect.Array:
return decodeReflectList(bz, rv)
case reflect.Map:
return decodeReflectMap(bz, rv)
case reflect.Struct:
return decodeReflectStruct(bz, rv)
case reflect.Interface:
return decodeReflectInterface(bz, rv)
// For 64-bit integers, unwrap expected string and defer to stdlib for integer decoding.
case reflect.Int64, reflect.Int, reflect.Uint64, reflect.Uint:
if bz[0] != '"' || bz[len(bz)-1] != '"' {
return fmt.Errorf("invalid 64-bit integer encoding %q, expected string", string(bz))
}
bz = bz[1 : len(bz)-1]
fallthrough
// Anything else we defer to the stdlib.
default:
return decodeStdlib(bz, rv)
}
}
func decodeReflectList(bz []byte, rv reflect.Value) error {
if !rv.CanAddr() {
return errors.New("list value is not addressable")
}
switch rv.Type().Elem().Kind() {
// Decode base64-encoded bytes using stdlib decoder, via byte slice for arrays.
case reflect.Uint8:
if rv.Type().Kind() == reflect.Array {
var buf []byte
if err := json.Unmarshal(bz, &buf); err != nil {
return err
}
if len(buf) != rv.Len() {
return fmt.Errorf("got %v bytes, expected %v", len(buf), rv.Len())
}
reflect.Copy(rv, reflect.ValueOf(buf))
} else if err := decodeStdlib(bz, rv); err != nil {
return err
}
// Decode anything else into a raw JSON slice, and decode values recursively.
default:
var rawSlice []json.RawMessage
if err := json.Unmarshal(bz, &rawSlice); err != nil {
return err
}
if rv.Type().Kind() == reflect.Slice {
rv.Set(reflect.MakeSlice(reflect.SliceOf(rv.Type().Elem()), len(rawSlice), len(rawSlice)))
}
if rv.Len() != len(rawSlice) { // arrays of wrong size
return fmt.Errorf("got list of %v elements, expected %v", len(rawSlice), rv.Len())
}
for i, bz := range rawSlice {
if err := decodeReflect(bz, rv.Index(i)); err != nil {
return err
}
}
}
// Replace empty slices with nil slices, for Amino compatibility
if rv.Type().Kind() == reflect.Slice && rv.Len() == 0 {
rv.Set(reflect.Zero(rv.Type()))
}
return nil
}
func decodeReflectMap(bz []byte, rv reflect.Value) error {
if !rv.CanAddr() {
return errors.New("map value is not addressable")
}
// Decode into a raw JSON map, using string keys.
rawMap := make(map[string]json.RawMessage)
if err := json.Unmarshal(bz, &rawMap); err != nil {
return err
}
if rv.Type().Key().Kind() != reflect.String {
return fmt.Errorf("map keys must be strings, got %v", rv.Type().Key().String())
}
// Recursively decode values.
rv.Set(reflect.MakeMapWithSize(rv.Type(), len(rawMap)))
for key, bz := range rawMap {
value := reflect.New(rv.Type().Elem()).Elem()
if err := decodeReflect(bz, value); err != nil {
return err
}
rv.SetMapIndex(reflect.ValueOf(key), value)
}
return nil
}
func decodeReflectStruct(bz []byte, rv reflect.Value) error {
if !rv.CanAddr() {
return errors.New("struct value is not addressable")
}
sInfo := makeStructInfo(rv.Type())
// Decode raw JSON values into a string-keyed map.
rawMap := make(map[string]json.RawMessage)
if err := json.Unmarshal(bz, &rawMap); err != nil {
return err
}
for i, fInfo := range sInfo.fields {
if !fInfo.hidden {
frv := rv.Field(i)
bz := rawMap[fInfo.jsonName]
if len(bz) > 0 {
if err := decodeReflect(bz, frv); err != nil {
return err
}
} else if !fInfo.omitEmpty {
frv.Set(reflect.Zero(frv.Type()))
}
}
}
return nil
}
func decodeReflectInterface(bz []byte, rv reflect.Value) error {
if !rv.CanAddr() {
return errors.New("interface value not addressable")
}
// Decode the interface wrapper.
wrapper := interfaceWrapper{}
if err := json.Unmarshal(bz, &wrapper); err != nil {
return err
}
if wrapper.Type == "" {
return errors.New("interface type cannot be empty")
}
if len(wrapper.Value) == 0 {
return errors.New("interface value cannot be empty")
}
// Dereference-and-construct pointers, to handle nested pointers.
for rv.Kind() == reflect.Ptr {
if rv.IsNil() {
rv.Set(reflect.New(rv.Type().Elem()))
}
rv = rv.Elem()
}
// Look up the interface type, and construct a concrete value.
rt, returnPtr := typeRegistry.lookup(wrapper.Type)
if rt == nil {
return fmt.Errorf("unknown type %q", wrapper.Type)
}
cptr := reflect.New(rt)
crv := cptr.Elem()
if err := decodeReflect(wrapper.Value, crv); err != nil {
return err
}
// This makes sure interface implementations with pointer receivers (e.g. func (c *Car)) are
// constructed as pointers behind the interface. The types must be registered as pointers with
// RegisterType().
if rv.Type().Kind() == reflect.Interface && returnPtr {
if !cptr.Type().AssignableTo(rv.Type()) {
return fmt.Errorf("invalid type %q for this value", wrapper.Type)
}
rv.Set(cptr)
} else {
if !crv.Type().AssignableTo(rv.Type()) {
return fmt.Errorf("invalid type %q for this value", wrapper.Type)
}
rv.Set(crv)
}
return nil
}
func decodeStdlib(bz []byte, rv reflect.Value) error {
if !rv.CanAddr() && rv.Kind() != reflect.Ptr {
return errors.New("value must be addressable or pointer")
}
// Make sure we are unmarshaling into a pointer.
target := rv
if rv.Kind() != reflect.Ptr {
target = reflect.New(rv.Type())
}
if err := json.Unmarshal(bz, target.Interface()); err != nil {
return err
}
rv.Set(target.Elem())
return nil
}
type interfaceWrapper struct {
Type string `json:"type"`
Value json.RawMessage `json:"value"`
}

+ 151
- 0
libs/json/decoder_test.go View File

@ -0,0 +1,151 @@
package json_test
import (
"reflect"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/tendermint/tendermint/libs/json"
)
func TestUnmarshal(t *testing.T) {
i64Nil := (*int64)(nil)
str := "string"
strPtr := &str
structNil := (*Struct)(nil)
i32 := int32(32)
i64 := int64(64)
testcases := map[string]struct {
json string
value interface{}
err bool
}{
"bool true": {"true", true, false},
"bool false": {"false", false, false},
"float32": {"3.14", float32(3.14), false},
"float64": {"3.14", float64(3.14), false},
"int32": {`32`, int32(32), false},
"int32 string": {`"32"`, int32(32), true},
"int32 ptr": {`32`, &i32, false},
"int64": {`"64"`, int64(64), false},
"int64 noend": {`"64`, int64(64), true},
"int64 number": {`64`, int64(64), true},
"int64 ptr": {`"64"`, &i64, false},
"int64 ptr nil": {`null`, i64Nil, false},
"string": {`"foo"`, "foo", false},
"string noend": {`"foo`, "foo", true},
"string ptr": {`"string"`, &str, false},
"slice byte": {`"AQID"`, []byte{1, 2, 3}, false},
"slice bytes": {`["AQID"]`, [][]byte{{1, 2, 3}}, false},
"slice int32": {`[1,2,3]`, []int32{1, 2, 3}, false},
"slice int64": {`["1","2","3"]`, []int64{1, 2, 3}, false},
"slice int64 number": {`[1,2,3]`, []int64{1, 2, 3}, true},
"slice int64 ptr": {`["64"]`, []*int64{&i64}, false},
"slice int64 empty": {`[]`, []int64(nil), false},
"slice int64 null": {`null`, []int64(nil), false},
"array byte": {`"AQID"`, [3]byte{1, 2, 3}, false},
"array byte large": {`"AQID"`, [4]byte{1, 2, 3, 4}, true},
"array byte small": {`"AQID"`, [2]byte{1, 2}, true},
"array int32": {`[1,2,3]`, [3]int32{1, 2, 3}, false},
"array int64": {`["1","2","3"]`, [3]int64{1, 2, 3}, false},
"array int64 number": {`[1,2,3]`, [3]int64{1, 2, 3}, true},
"array int64 large": {`["1","2","3"]`, [4]int64{1, 2, 3, 4}, true},
"array int64 small": {`["1","2","3"]`, [2]int64{1, 2}, true},
"map bytes": {`{"b":"AQID"}`, map[string][]byte{"b": {1, 2, 3}}, false},
"map int32": {`{"a":1,"b":2}`, map[string]int32{"a": 1, "b": 2}, false},
"map int64": {`{"a":"1","b":"2"}`, map[string]int64{"a": 1, "b": 2}, false},
"map int64 empty": {`{}`, map[string]int64{}, false},
"map int64 null": {`null`, map[string]int64(nil), false},
"map int key": {`{}`, map[int]int{}, true},
"time": {`"2020-06-03T17:35:30Z"`, time.Date(2020, 6, 3, 17, 35, 30, 0, time.UTC), false},
"time non-utc": {`"2020-06-03T17:35:30+02:00"`, time.Time{}, true},
"time nozone": {`"2020-06-03T17:35:30"`, time.Time{}, true},
"car": {`{"type":"vehicle/car","value":{"Wheels":4}}`, Car{Wheels: 4}, false},
"car ptr": {`{"type":"vehicle/car","value":{"Wheels":4}}`, &Car{Wheels: 4}, false},
"car iface": {`{"type":"vehicle/car","value":{"Wheels":4}}`, Vehicle(&Car{Wheels: 4}), false},
"boat": {`{"type":"vehicle/boat","value":{"Sail":true}}`, Boat{Sail: true}, false},
"boat ptr": {`{"type":"vehicle/boat","value":{"Sail":true}}`, &Boat{Sail: true}, false},
"boat iface": {`{"type":"vehicle/boat","value":{"Sail":true}}`, Vehicle(Boat{Sail: true}), false},
"boat into car": {`{"type":"vehicle/boat","value":{"Sail":true}}`, Car{}, true},
"boat into car iface": {`{"type":"vehicle/boat","value":{"Sail":true}}`, Vehicle(&Car{}), true},
"shoes": {`{"type":"vehicle/shoes","value":{"Soles":"rubber"}}`, Car{}, true},
"shoes ptr": {`{"type":"vehicle/shoes","value":{"Soles":"rubber"}}`, &Car{}, true},
"shoes iface": {`{"type":"vehicle/shoes","value":{"Soles":"rubbes"}}`, Vehicle(&Car{}), true},
"key public": {`{"type":"key/public","value":"AQIDBAUGBwg="}`, PublicKey{1, 2, 3, 4, 5, 6, 7, 8}, false},
"key wrong": {`{"type":"key/public","value":"AQIDBAUGBwg="}`, PrivateKey{1, 2, 3, 4, 5, 6, 7, 8}, true},
"key into car": {`{"type":"key/public","value":"AQIDBAUGBwg="}`, Vehicle(&Car{}), true},
"tags": {
`{"name":"name","OmitEmpty":"foo","Hidden":"bar","tags":{"name":"child"}}`,
Tags{JSONName: "name", OmitEmpty: "foo", Tags: &Tags{JSONName: "child"}},
false,
},
"tags ptr": {
`{"name":"name","OmitEmpty":"foo","tags":null}`,
&Tags{JSONName: "name", OmitEmpty: "foo"},
false,
},
"tags real name": {`{"JSONName":"name"}`, Tags{}, false},
"struct": {
`{
"Bool":true, "Float64":3.14, "Int32":32, "Int64":"64", "Int64Ptr":"64",
"String":"foo", "StringPtrPtr": "string", "Bytes":"AQID",
"Time":"2020-06-02T16:05:13.004346374Z",
"Car":{"Wheels":4},
"Boat":{"Sail":true},
"Vehicles":[
{"type":"vehicle/car","value":{"Wheels":4}},
{"type":"vehicle/boat","value":{"Sail":true}}
],
"Child":{
"Bool":false, "Float64":0, "Int32":0, "Int64":"0", "Int64Ptr":null,
"String":"child", "StringPtrPtr":null, "Bytes":null,
"Time":"0001-01-01T00:00:00Z",
"Car":null, "Boat":{"Sail":false}, "Vehicles":null, "Child":null
},
"private": "foo", "unknown": "bar"
}`,
Struct{
Bool: true, Float64: 3.14, Int32: 32, Int64: 64, Int64Ptr: &i64,
String: "foo", StringPtrPtr: &strPtr, Bytes: []byte{1, 2, 3},
Time: time.Date(2020, 6, 2, 16, 5, 13, 4346374, time.UTC),
Car: &Car{Wheels: 4}, Boat: Boat{Sail: true}, Vehicles: []Vehicle{
Vehicle(&Car{Wheels: 4}),
Vehicle(Boat{Sail: true}),
},
Child: &Struct{Bool: false, String: "child"},
},
false,
},
"struct key into vehicle": {`{"Vehicles":[
{"type":"vehicle/car","value":{"Wheels":4}},
{"type":"key/public","value":"MTIzNDU2Nzg="}
]}`, Struct{}, true},
"struct ptr null": {`null`, structNil, false},
"custom value": {`{"Value":"foo"}`, CustomValue{}, false},
"custom ptr": {`"foo"`, &CustomPtr{Value: "custom"}, false},
"custom ptr value": {`"foo"`, CustomPtr{Value: "custom"}, false},
"invalid type": {`"foo"`, Struct{}, true},
}
for name, tc := range testcases {
tc := tc
t.Run(name, func(t *testing.T) {
// Create a target variable as a pointer to the zero value of the tc.value type,
// and wrap it in an empty interface. Decode into that interface.
target := reflect.New(reflect.TypeOf(tc.value)).Interface()
err := json.Unmarshal([]byte(tc.json), target)
if tc.err {
require.Error(t, err)
return
}
require.NoError(t, err)
// Unwrap the target pointer and get the value behind the interface.
actual := reflect.ValueOf(target).Elem().Interface()
assert.Equal(t, tc.value, actual)
})
}
}

+ 99
- 0
libs/json/doc.go View File

@ -0,0 +1,99 @@
// Package json provides functions for marshaling and unmarshaling JSON in a format that is
// backwards-compatible with Amino JSON encoding. This mostly differs from encoding/json in
// encoding of integers (64-bit integers are encoded as strings, not numbers), and handling
// of interfaces (wrapped in an interface object with type/value keys).
//
// JSON tags (e.g. `json:"name,omitempty"`) are supported in the same way as encoding/json, as is
// custom marshaling overrides via the json.Marshaler and json.Unmarshaler interfaces.
//
// Note that not all JSON emitted by Tendermint is generated by this library; some is generated by
// encoding/json instead, and kept like that for backwards compatibility.
//
// Encoding of numbers uses strings for 64-bit integers (including unspecified ints), to improve
// compatibility with e.g. Javascript (which uses 64-bit floats for numbers, having 53-bit
// precision):
//
// int32(32) // Output: 32
// uint32(32) // Output: 32
// int64(64) // Output: "64"
// uint64(64) // Output: "64"
// int(64) // Output: "64"
// uint(64) // Output: "64"
//
// Encoding of other scalars follows encoding/json:
//
// nil // Output: null
// true // Output: true
// "foo" // Output: "foo"
// "" // Output: ""
//
// Slices and arrays are encoded as encoding/json, including base64-encoding of byte slices
// with additional base64-encoding of byte arrays as well:
//
// []int64(nil) // Output: null
// []int64{} // Output: []
// []int64{1, 2, 3} // Output: ["1", "2", "3"]
// []int32{1, 2, 3} // Output: [1, 2, 3]
// []byte{1, 2, 3} // Output: "AQID"
// [3]int64{1, 2, 3} // Output: ["1", "2", "3"]
// [3]byte{1, 2, 3} // Output: "AQID"
//
// Maps are encoded as encoding/json, but only strings are allowed as map keys (nil maps are not
// emitted as null, to retain Amino backwards-compatibility):
//
// map[string]int64(nil) // Output: {}
// map[string]int64{} // Output: {}
// map[string]int64{"a":1,"b":2} // Output: {"a":"1","b":"2"}
// map[string]int32{"a":1,"b":2} // Output: {"a":1,"b":2}
// map[bool]int{true:1} // Errors
//
// Times are encoded as encoding/json, in RFC3339Nano format, but requiring UTC time zone (with zero
// times emitted as "0001-01-01T00:00:00Z" as with encoding/json):
//
// time.Date(2020, 6, 8, 16, 21, 28, 123, time.FixedZone("UTC+2", 2*60*60))
// // Output: "2020-06-08T14:21:28.000000123Z"
// time.Time{} // Output: "0001-01-01T00:00:00Z"
// (*time.Time)(nil) // Output: null
//
// Structs are encoded as encoding/json, supporting JSON tags and ignoring private fields:
//
// type Struct struct{
// Name string
// Value int32 `json:"value,omitempty"`
// private bool
// }
//
// Struct{Name: "foo", Value: 7, private: true} // Output: {"Name":"foo","value":7}
// Struct{} // Output: {"Name":""}
//
// Registered types are encoded with type wrapper, regardless of whether they are given as interface
// or bare struct, but inside structs they are only emitted with type wrapper for interface fields
// (this follows Amino behavior):
//
// type Vehicle interface {
// Drive() error
// }
//
// type Car struct {
// Wheels int8
// }
//
// func (c *Car) Drive() error { return nil }
//
// RegisterType(&Car{}, "vehicle/car")
//
// Car{Wheels: 4} // Output: {"type":"vehicle/car","value":{"Wheels":4}}
// &Car{Wheels: 4} // Output: {"type":"vehicle/car","value":{"Wheels":4}}
// (*Car)(nil) // Output: null
// Vehicle(Car{Wheels: 4}) // Output: {"type":"vehicle/car","value":{"Wheels":4}}
// Vehicle(nil) // Output: null
//
// type Struct struct {
// Car *Car
// Vehicle Vehicle
// }
//
// Struct{Car: &Car{Wheels: 4}, Vehicle: &Car{Wheels: 4}}
// // Output: {"Car": {"Wheels: 4"}, "Vehicle": {"type":"vehicle/car","value":{"Wheels":4}}}
//
package json

+ 254
- 0
libs/json/encoder.go View File

@ -0,0 +1,254 @@
package json
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"reflect"
"strconv"
"time"
)
var (
timeType = reflect.TypeOf(time.Time{})
jsonMarshalerType = reflect.TypeOf(new(json.Marshaler)).Elem()
jsonUnmarshalerType = reflect.TypeOf(new(json.Unmarshaler)).Elem()
)
// Marshal marshals the value as JSON, using Amino-compatible JSON encoding (strings for
// 64-bit numbers, and type wrappers for registered types).
func Marshal(v interface{}) ([]byte, error) {
buf := new(bytes.Buffer)
err := encode(buf, v)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// MarshalIndent marshals the value as JSON, using the given prefix and indentation.
func MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) {
bz, err := Marshal(v)
if err != nil {
return nil, err
}
buf := new(bytes.Buffer)
err = json.Indent(buf, bz, prefix, indent)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func encode(w io.Writer, v interface{}) error {
// Bare nil values can't be reflected, so we must handle them here.
if v == nil {
return writeStr(w, "null")
}
rv := reflect.ValueOf(v)
// If this is a registered type, defer to interface encoder regardless of whether the input is
// an interface or a bare value. This retains Amino's behavior, but is inconsistent with
// behavior in structs where an interface field will get the type wrapper while a bare value
// field will not.
if typeRegistry.name(rv.Type()) != "" {
return encodeReflectInterface(w, rv)
}
return encodeReflect(w, rv)
}
func encodeReflect(w io.Writer, rv reflect.Value) error {
if !rv.IsValid() {
return errors.New("invalid reflect value")
}
// Recursively dereference if pointer.
for rv.Kind() == reflect.Ptr {
if rv.IsNil() {
return writeStr(w, "null")
}
rv = rv.Elem()
}
// Convert times to UTC.
if rv.Type() == timeType {
rv = reflect.ValueOf(rv.Interface().(time.Time).Round(0).UTC())
}
// If the value implements json.Marshaler, defer to stdlib directly. Since we've already
// dereferenced, we try implementations with both value receiver and pointer receiver. We must
// do this after the time normalization above, and thus after dereferencing.
if rv.Type().Implements(jsonMarshalerType) {
return encodeStdlib(w, rv.Interface())
} else if rv.CanAddr() && rv.Addr().Type().Implements(jsonMarshalerType) {
return encodeStdlib(w, rv.Addr().Interface())
}
switch rv.Type().Kind() {
// Complex types must be recursively encoded.
case reflect.Interface:
return encodeReflectInterface(w, rv)
case reflect.Array, reflect.Slice:
return encodeReflectList(w, rv)
case reflect.Map:
return encodeReflectMap(w, rv)
case reflect.Struct:
return encodeReflectStruct(w, rv)
// 64-bit integers are emitted as strings, to avoid precision problems with e.g.
// Javascript which uses 64-bit floats (having 53-bit precision).
case reflect.Int64, reflect.Int:
return writeStr(w, `"`+strconv.FormatInt(rv.Int(), 10)+`"`)
case reflect.Uint64, reflect.Uint:
return writeStr(w, `"`+strconv.FormatUint(rv.Uint(), 10)+`"`)
// For everything else, defer to the stdlib encoding/json encoder
default:
return encodeStdlib(w, rv.Interface())
}
}
func encodeReflectList(w io.Writer, rv reflect.Value) error {
// Emit nil slices as null.
if rv.Kind() == reflect.Slice && rv.IsNil() {
return writeStr(w, "null")
}
// Encode byte slices as base64 with the stdlib encoder.
if rv.Type().Elem().Kind() == reflect.Uint8 {
// Stdlib does not base64-encode byte arrays, only slices, so we copy to slice.
if rv.Type().Kind() == reflect.Array {
slice := reflect.MakeSlice(reflect.SliceOf(rv.Type().Elem()), rv.Len(), rv.Len())
reflect.Copy(slice, rv)
rv = slice
}
return encodeStdlib(w, rv.Interface())
}
// Anything else we recursively encode ourselves.
length := rv.Len()
if err := writeStr(w, "["); err != nil {
return err
}
for i := 0; i < length; i++ {
if err := encodeReflect(w, rv.Index(i)); err != nil {
return err
}
if i < length-1 {
if err := writeStr(w, ","); err != nil {
return err
}
}
}
return writeStr(w, "]")
}
func encodeReflectMap(w io.Writer, rv reflect.Value) error {
if rv.Type().Key().Kind() != reflect.String {
return errors.New("map key must be string")
}
// nil maps are not emitted as nil, to retain Amino compatibility.
if err := writeStr(w, "{"); err != nil {
return err
}
writeComma := false
for _, keyrv := range rv.MapKeys() {
if writeComma {
if err := writeStr(w, ","); err != nil {
return err
}
}
if err := encodeStdlib(w, keyrv.Interface()); err != nil {
return err
}
if err := writeStr(w, ":"); err != nil {
return err
}
if err := encodeReflect(w, rv.MapIndex(keyrv)); err != nil {
return err
}
writeComma = true
}
return writeStr(w, "}")
}
func encodeReflectStruct(w io.Writer, rv reflect.Value) error {
sInfo := makeStructInfo(rv.Type())
if err := writeStr(w, "{"); err != nil {
return err
}
writeComma := false
for i, fInfo := range sInfo.fields {
frv := rv.Field(i)
if fInfo.hidden || (fInfo.omitEmpty && frv.IsZero()) {
continue
}
if writeComma {
if err := writeStr(w, ","); err != nil {
return err
}
}
if err := encodeStdlib(w, fInfo.jsonName); err != nil {
return err
}
if err := writeStr(w, ":"); err != nil {
return err
}
if err := encodeReflect(w, frv); err != nil {
return err
}
writeComma = true
}
return writeStr(w, "}")
}
func encodeReflectInterface(w io.Writer, rv reflect.Value) error {
// Get concrete value and dereference pointers.
for rv.Kind() == reflect.Ptr || rv.Kind() == reflect.Interface {
if rv.IsNil() {
return writeStr(w, "null")
}
rv = rv.Elem()
}
// Look up the name of the concrete type
name := typeRegistry.name(rv.Type())
if name == "" {
return fmt.Errorf("cannot encode unregistered type %v", rv.Type())
}
// Write value wrapped in interface envelope
if err := writeStr(w, fmt.Sprintf(`{"type":%q,"value":`, name)); err != nil {
return err
}
if err := encodeReflect(w, rv); err != nil {
return err
}
return writeStr(w, "}")
}
func encodeStdlib(w io.Writer, v interface{}) error {
// Doesn't stream the output because that adds a newline, as per:
// https://golang.org/pkg/encoding/json/#Encoder.Encode
blob, err := json.Marshal(v)
if err != nil {
return err
}
_, err = w.Write(blob)
return err
}
func writeStr(w io.Writer, s string) error {
_, err := w.Write([]byte(s))
return err
}

+ 104
- 0
libs/json/encoder_test.go View File

@ -0,0 +1,104 @@
package json_test
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/tendermint/tendermint/libs/json"
)
func TestMarshal(t *testing.T) {
s := "string"
sPtr := &s
i64 := int64(64)
ti := time.Date(2020, 6, 2, 18, 5, 13, 4346374, time.FixedZone("UTC+2", 2*60*60))
car := &Car{Wheels: 4}
boat := Boat{Sail: true}
testcases := map[string]struct {
value interface{}
output string
}{
"nil": {nil, `null`},
"string": {"foo", `"foo"`},
"float32": {float32(3.14), `3.14`},
"float32 neg": {float32(-3.14), `-3.14`},
"float64": {float64(3.14), `3.14`},
"float64 neg": {float64(-3.14), `-3.14`},
"int32": {int32(32), `32`},
"int64": {int64(64), `"64"`},
"int64 neg": {int64(-64), `"-64"`},
"int64 ptr": {&i64, `"64"`},
"uint64": {uint64(64), `"64"`},
"time": {ti, `"2020-06-02T16:05:13.004346374Z"`},
"time empty": {time.Time{}, `"0001-01-01T00:00:00Z"`},
"time ptr": {&ti, `"2020-06-02T16:05:13.004346374Z"`},
"customptr": {CustomPtr{Value: "x"}, `{"Value":"x"}`}, // same as encoding/json
"customptr ptr": {&CustomPtr{Value: "x"}, `"custom"`},
"customvalue": {CustomValue{Value: "x"}, `"custom"`},
"customvalue ptr": {&CustomValue{Value: "x"}, `"custom"`},
"slice nil": {[]int(nil), `null`},
"slice empty": {[]int{}, `[]`},
"slice bytes": {[]byte{1, 2, 3}, `"AQID"`},
"slice int64": {[]int64{1, 2, 3}, `["1","2","3"]`},
"slice int64 ptr": {[]*int64{&i64, nil}, `["64",null]`},
"array bytes": {[3]byte{1, 2, 3}, `"AQID"`},
"array int64": {[3]int64{1, 2, 3}, `["1","2","3"]`},
"map nil": {map[string]int64(nil), `{}`}, // retain Amino compatibility
"map empty": {map[string]int64{}, `{}`},
"map int64": {map[string]int64{"a": 1, "b": 2, "c": 3}, `{"a":"1","b":"2","c":"3"}`},
"car": {car, `{"type":"vehicle/car","value":{"Wheels":4}}`},
"car value": {*car, `{"type":"vehicle/car","value":{"Wheels":4}}`},
"car iface": {Vehicle(car), `{"type":"vehicle/car","value":{"Wheels":4}}`},
"car nil": {(*Car)(nil), `null`},
"boat": {boat, `{"type":"vehicle/boat","value":{"Sail":true}}`},
"boat ptr": {&boat, `{"type":"vehicle/boat","value":{"Sail":true}}`},
"boat iface": {Vehicle(boat), `{"type":"vehicle/boat","value":{"Sail":true}}`},
"key public": {PublicKey{1, 2, 3, 4, 5, 6, 7, 8}, `{"type":"key/public","value":"AQIDBAUGBwg="}`},
"tags": {
Tags{JSONName: "name", OmitEmpty: "foo", Hidden: "bar", Tags: &Tags{JSONName: "child"}},
`{"name":"name","OmitEmpty":"foo","tags":{"name":"child"}}`,
},
"tags empty": {Tags{}, `{"name":""}`},
// The encoding of the Car and Boat fields do not have type wrappers, even though they get
// type wrappers when encoded directly (see "car" and "boat" tests). This is to retain the
// same behavior as Amino. If the field was a Vehicle interface instead, it would get
// type wrappers, as seen in the Vehicles field.
"struct": {
Struct{
Bool: true, Float64: 3.14, Int32: 32, Int64: 64, Int64Ptr: &i64,
String: "foo", StringPtrPtr: &sPtr, Bytes: []byte{1, 2, 3},
Time: ti, Car: car, Boat: boat, Vehicles: []Vehicle{car, boat},
Child: &Struct{Bool: false, String: "child"}, private: "private",
},
`{
"Bool":true, "Float64":3.14, "Int32":32, "Int64":"64", "Int64Ptr":"64",
"String":"foo", "StringPtrPtr": "string", "Bytes":"AQID",
"Time":"2020-06-02T16:05:13.004346374Z",
"Car":{"Wheels":4},
"Boat":{"Sail":true},
"Vehicles":[
{"type":"vehicle/car","value":{"Wheels":4}},
{"type":"vehicle/boat","value":{"Sail":true}}
],
"Child":{
"Bool":false, "Float64":0, "Int32":0, "Int64":"0", "Int64Ptr":null,
"String":"child", "StringPtrPtr":null, "Bytes":null,
"Time":"0001-01-01T00:00:00Z",
"Car":null, "Boat":{"Sail":false}, "Vehicles":null, "Child":null
}
}`,
},
}
for name, tc := range testcases {
tc := tc
t.Run(name, func(t *testing.T) {
bz, err := json.Marshal(tc.value)
require.NoError(t, err)
assert.JSONEq(t, tc.output, string(bz))
})
}
}

+ 92
- 0
libs/json/helpers_test.go View File

@ -0,0 +1,92 @@
package json_test
import (
"time"
"github.com/tendermint/tendermint/libs/json"
)
// Register Car, an instance of the Vehicle interface.
func init() {
json.RegisterType(&Car{}, "vehicle/car")
json.RegisterType(Boat{}, "vehicle/boat")
json.RegisterType(PublicKey{}, "key/public")
json.RegisterType(PrivateKey{}, "key/private")
}
type Vehicle interface {
Drive() error
}
// Car is a pointer implementation of Vehicle.
type Car struct {
Wheels int32
}
func (c *Car) Drive() error { return nil }
// Boat is a value implementation of Vehicle.
type Boat struct {
Sail bool
}
func (b Boat) Drive() error { return nil }
// These are public and private encryption keys.
type PublicKey [8]byte
type PrivateKey [8]byte
// Custom has custom marshalers and unmarshalers, taking pointer receivers.
type CustomPtr struct {
Value string
}
func (c *CustomPtr) MarshalJSON() ([]byte, error) {
return []byte("\"custom\""), nil
}
func (c *CustomPtr) UnmarshalJSON(bz []byte) error {
c.Value = "custom"
return nil
}
// CustomValue has custom marshalers and unmarshalers, taking value receivers (which usually doesn't
// make much sense since the unmarshaler can't change anything).
type CustomValue struct {
Value string
}
func (c CustomValue) MarshalJSON() ([]byte, error) {
return []byte("\"custom\""), nil
}
func (c CustomValue) UnmarshalJSON(bz []byte) error {
c.Value = "custom"
return nil
}
// Tags tests JSON tags.
type Tags struct {
JSONName string `json:"name"`
OmitEmpty string `json:",omitempty"`
Hidden string `json:"-"`
Tags *Tags `json:"tags,omitempty"`
}
// Struct tests structs with lots of contents.
type Struct struct {
Bool bool
Float64 float64
Int32 int32
Int64 int64
Int64Ptr *int64
String string
StringPtrPtr **string
Bytes []byte
Time time.Time
Car *Car
Boat Boat
Vehicles []Vehicle
Child *Struct
private string
}

+ 87
- 0
libs/json/structs.go View File

@ -0,0 +1,87 @@
package json
import (
"fmt"
"reflect"
"strings"
"sync"
"unicode"
)
var (
// cache caches struct info.
cache = newStructInfoCache()
)
// structCache is a cache of struct info.
type structInfoCache struct {
sync.RWMutex
structInfos map[reflect.Type]*structInfo
}
func newStructInfoCache() *structInfoCache {
return &structInfoCache{
structInfos: make(map[reflect.Type]*structInfo),
}
}
func (c *structInfoCache) get(rt reflect.Type) *structInfo {
c.RLock()
defer c.RUnlock()
return c.structInfos[rt]
}
func (c *structInfoCache) set(rt reflect.Type, sInfo *structInfo) {
c.Lock()
defer c.Unlock()
c.structInfos[rt] = sInfo
}
// structInfo contains JSON info for a struct.
type structInfo struct {
fields []*fieldInfo
}
// fieldInfo contains JSON info for a struct field.
type fieldInfo struct {
jsonName string
omitEmpty bool
hidden bool
}
// makeStructInfo generates structInfo for a struct as a reflect.Value.
func makeStructInfo(rt reflect.Type) *structInfo {
if rt.Kind() != reflect.Struct {
panic(fmt.Sprintf("can't make struct info for non-struct value %v", rt))
}
if sInfo := cache.get(rt); sInfo != nil {
return sInfo
}
fields := make([]*fieldInfo, 0, rt.NumField())
for i := 0; i < cap(fields); i++ {
frt := rt.Field(i)
fInfo := &fieldInfo{
jsonName: frt.Name,
omitEmpty: false,
hidden: frt.Name == "" || !unicode.IsUpper(rune(frt.Name[0])),
}
o := frt.Tag.Get("json")
if o == "-" {
fInfo.hidden = true
} else if o != "" {
opts := strings.Split(o, ",")
if opts[0] != "" {
fInfo.jsonName = opts[0]
}
for _, o := range opts[1:] {
if o == "omitempty" {
fInfo.omitEmpty = true
}
}
}
fields = append(fields, fInfo)
}
sInfo := &structInfo{fields: fields}
cache.set(rt, sInfo)
return sInfo
}

+ 108
- 0
libs/json/types.go View File

@ -0,0 +1,108 @@
package json
import (
"errors"
"fmt"
"reflect"
"sync"
)
var (
// typeRegistry contains globally registered types for JSON encoding/decoding.
typeRegistry = newTypes()
)
// RegisterType registers a type for Amino-compatible interface encoding in the global type
// registry. These types will be encoded with a type wrapper `{"type":"<type>","value":<value>}`
// regardless of which interface they are wrapped in (if any). If the type is a pointer, it will
// still be valid both for value and pointer types, but decoding into an interface will generate
// the a value or pointer based on the registered type.
//
// Should only be called in init() functions, as it panics on error.
func RegisterType(_type interface{}, name string) {
if _type == nil {
panic("cannot register nil type")
}
err := typeRegistry.register(name, reflect.ValueOf(_type).Type())
if err != nil {
panic(err)
}
}
// typeInfo contains type information.
type typeInfo struct {
name string
rt reflect.Type
returnPtr bool
}
// types is a type registry. It is safe for concurrent use.
type types struct {
sync.RWMutex
byType map[reflect.Type]*typeInfo
byName map[string]*typeInfo
}
// newTypes creates a new type registry.
func newTypes() types {
return types{
byType: map[reflect.Type]*typeInfo{},
byName: map[string]*typeInfo{},
}
}
// registers the given type with the given name. The name and type must not be registered already.
func (t *types) register(name string, rt reflect.Type) error {
if name == "" {
return errors.New("name cannot be empty")
}
// If this is a pointer type, we recursively resolve until we get a bare type, but register that
// we should return pointers.
returnPtr := false
for rt.Kind() == reflect.Ptr {
returnPtr = true
rt = rt.Elem()
}
tInfo := &typeInfo{
name: name,
rt: rt,
returnPtr: returnPtr,
}
t.Lock()
defer t.Unlock()
if _, ok := t.byName[tInfo.name]; ok {
return fmt.Errorf("a type with name %q is already registered", name)
}
if _, ok := t.byType[tInfo.rt]; ok {
return fmt.Errorf("the type %v is already registered", rt)
}
t.byName[name] = tInfo
t.byType[rt] = tInfo
return nil
}
// lookup looks up a type from a name, or nil if not registered.
func (t *types) lookup(name string) (reflect.Type, bool) {
t.RLock()
defer t.RUnlock()
tInfo := t.byName[name]
if tInfo == nil {
return nil, false
}
return tInfo.rt, tInfo.returnPtr
}
// name looks up the name of a type, or empty if not registered. Unwraps pointers as necessary.
func (t *types) name(rt reflect.Type) string {
for rt.Kind() == reflect.Ptr {
rt = rt.Elem()
}
t.RLock()
defer t.RUnlock()
tInfo := t.byType[rt]
if tInfo == nil {
return ""
}
return tInfo.name
}

Write Preview
Loading…
Cancel
Save