From 939702f5878a8f98fd2d9c54c9e08c3f11d704e0 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Sat, 23 May 2015 14:45:36 -0700 Subject: [PATCH] Support uint64 , but still support int/uint varints. --- binary/binary.go | 6 +-- binary/byteslice.go | 4 +- binary/codec.go | 30 +++++------ binary/int.go | 7 +-- binary/reflect.go | 127 ++++++++++++++++++++++++++++---------------- common/bit_array.go | 4 +- 6 files changed, 107 insertions(+), 71 deletions(-) diff --git a/binary/binary.go b/binary/binary.go index 1573947e9..890ac76aa 100644 --- a/binary/binary.go +++ b/binary/binary.go @@ -9,11 +9,11 @@ import ( func ReadBinary(o interface{}, r io.Reader, n *int64, err *error) interface{} { rv, rt := reflect.ValueOf(o), reflect.TypeOf(o) if rv.Kind() == reflect.Ptr { - readReflect(rv.Elem(), rt.Elem(), r, n, err) + readReflectBinary(rv.Elem(), rt.Elem(), Options{}, r, n, err) return o } else { ptrRv := reflect.New(rt) - readReflect(ptrRv.Elem(), rt, r, n, err) + readReflectBinary(ptrRv.Elem(), rt, Options{}, r, n, err) return ptrRv.Elem().Interface() } } @@ -24,7 +24,7 @@ func WriteBinary(o interface{}, w io.Writer, n *int64, err *error) { if rv.Kind() == reflect.Ptr { rv, rt = rv.Elem(), rt.Elem() } - writeReflect(rv, rt, w, n, err) + writeReflectBinary(rv, rt, Options{}, w, n, err) } func ReadJSON(o interface{}, bytes []byte, err *error) interface{} { diff --git a/binary/byteslice.go b/binary/byteslice.go index d1cd169a9..87a42785f 100644 --- a/binary/byteslice.go +++ b/binary/byteslice.go @@ -46,12 +46,12 @@ func WriteByteSlices(bzz [][]byte, w io.Writer, n *int64, err *error) { } func ReadByteSlices(r io.Reader, n *int64, err *error) [][]byte { - length := ReadUvarint(r, n, err) + length := int(ReadUvarint(r, n, err)) if *err != nil { return nil } bzz := make([][]byte, length) - for i := uint(0); i < length; i++ { + for i := 0; i < length; i++ { bz := ReadByteSlice(r, n, err) if *err != nil { return nil diff --git a/binary/codec.go b/binary/codec.go index 4f1fabe65..85c9a0cfc 100644 --- a/binary/codec.go +++ b/binary/codec.go @@ -36,51 +36,51 @@ const ( ) func BasicCodecEncoder(o interface{}, w io.Writer, n *int64, err *error) { - switch o.(type) { + switch o := o.(type) { case nil: panic("nil type unsupported") case byte: WriteByte(typeByte, w, n, err) - WriteByte(o.(byte), w, n, err) + WriteByte(o, w, n, err) case int8: WriteByte(typeInt8, w, n, err) - WriteInt8(o.(int8), w, n, err) + WriteInt8(o, w, n, err) //case uint8: // WriteByte( typeUint8, w, n, err) - // WriteUint8( o.(uint8), w, n, err) + // WriteUint8( o, w, n, err) case int16: WriteByte(typeInt16, w, n, err) - WriteInt16(o.(int16), w, n, err) + WriteInt16(o, w, n, err) case uint16: WriteByte(typeUint16, w, n, err) - WriteUint16(o.(uint16), w, n, err) + WriteUint16(o, w, n, err) case int32: WriteByte(typeInt32, w, n, err) - WriteInt32(o.(int32), w, n, err) + WriteInt32(o, w, n, err) case uint32: WriteByte(typeUint32, w, n, err) - WriteUint32(o.(uint32), w, n, err) + WriteUint32(o, w, n, err) case int64: WriteByte(typeInt64, w, n, err) - WriteInt64(o.(int64), w, n, err) + WriteInt64(o, w, n, err) case uint64: WriteByte(typeUint64, w, n, err) - WriteUint64(o.(uint64), w, n, err) + WriteUint64(o, w, n, err) case int: WriteByte(typeVarint, w, n, err) - WriteVarint(o.(int), w, n, err) + WriteVarint(o, w, n, err) case uint: WriteByte(typeUvarint, w, n, err) - WriteUvarint(o.(uint), w, n, err) + WriteUvarint(o, w, n, err) case string: WriteByte(typeString, w, n, err) - WriteString(o.(string), w, n, err) + WriteString(o, w, n, err) case []byte: WriteByte(typeByteSlice, w, n, err) - WriteByteSlice(o.([]byte), w, n, err) + WriteByteSlice(o, w, n, err) case time.Time: WriteByte(typeTime, w, n, err) - WriteTime(o.(time.Time), w, n, err) + WriteTime(o, w, n, err) default: panic(fmt.Sprintf("Unsupported type: %v", reflect.TypeOf(o))) } diff --git a/binary/int.go b/binary/int.go index 8a145cb14..8b4a36d24 100644 --- a/binary/int.go +++ b/binary/int.go @@ -158,7 +158,8 @@ func ReadUint64(r io.Reader, n *int64, err *error) uint64 { // Varint -func uvarIntSize(i uint) int { +func uvarintSize(i_ uint) int { + i := uint64(i_) if i < 1<<8 { return 1 } @@ -189,7 +190,7 @@ func WriteVarint(i int, w io.Writer, n *int64, err *error) { negate = true i = -i } - var size = uvarIntSize(uint(i)) + var size = uvarintSize(uint(i)) if negate { // e.g. 0xF1 for a single negative byte WriteUint8(uint8(size+0xF0), w, n, err) @@ -231,7 +232,7 @@ func ReadVarint(r io.Reader, n *int64, err *error) int { // Uvarint func WriteUvarint(i uint, w io.Writer, n *int64, err *error) { - var size = uvarIntSize(i) + var size = uvarintSize(i) WriteUint8(uint8(size), w, n, err) buf := make([]byte, 8) binary.BigEndian.PutUint64(buf, uint64(i)) diff --git a/binary/reflect.go b/binary/reflect.go index 7ec4e2d24..c745b6157 100644 --- a/binary/reflect.go +++ b/binary/reflect.go @@ -31,14 +31,39 @@ type TypeInfo struct { Fields []StructFieldInfo } +type Options struct { + JSONName string // (JSON) Corresponding JSON field name. (override with `json=""`) + Varint bool // (Binary) Use length-prefixed encoding for (u)int* +} + +func getOptionsFromField(field reflect.StructField) (skip bool, opts Options) { + jsonName := field.Tag.Get("json") + if jsonName == "-" { + skip = true + return + } else if jsonName == "" { + jsonName = field.Name + } + varint := false + binTag := field.Tag.Get("binary") + if binTag == "varint" { // TODO: extend + varint = true + } + opts = Options{ + JSONName: jsonName, + Varint: varint, + } + return +} + type StructFieldInfo struct { - Index int // Struct field index - JSONName string // Corresponding JSON field name. (override with `json=""`) - Type reflect.Type // Struct field type + Index int // Struct field index + Type reflect.Type // Struct field type + Options // Encoding options } -func (info StructFieldInfo) unpack() (int, string, reflect.Type) { - return info.Index, info.JSONName, info.Type +func (info StructFieldInfo) unpack() (int, reflect.Type, Options) { + return info.Index, info.Type, info.Options } // e.g. If o is struct{Foo}{}, return is the Foo interface type. @@ -136,16 +161,14 @@ func MakeTypeInfo(rt reflect.Type) *TypeInfo { if field.PkgPath != "" { continue } - jsonName := field.Tag.Get("json") - if jsonName == "-" { + skip, opts := getOptionsFromField(field) + if skip { continue - } else if jsonName == "" { - jsonName = field.Name } structFields = append(structFields, StructFieldInfo{ - Index: i, - JSONName: jsonName, - Type: field.Type, + Index: i, + Type: field.Type, + Options: opts, }) } info.Fields = structFields @@ -154,7 +177,7 @@ func MakeTypeInfo(rt reflect.Type) *TypeInfo { return info } -func readReflect(rv reflect.Value, rt reflect.Type, r io.Reader, n *int64, err *error) { +func readReflectBinary(rv reflect.Value, rt reflect.Type, opts Options, r io.Reader, n *int64, err *error) { // Get typeInfo typeInfo := GetTypeInfo(rt) @@ -179,7 +202,7 @@ func readReflect(rv reflect.Value, rt reflect.Type, r io.Reader, n *int64, err * } crv := reflect.New(crt).Elem() r = NewPrefixedReader([]byte{typeByte}, r) - readReflect(crv, crt, r, n, err) + readReflectBinary(crv, crt, opts, r, n, err) rv.Set(crv) // NOTE: orig rv is ignored. return } @@ -236,7 +259,7 @@ func readReflect(rv reflect.Value, rt reflect.Type, r io.Reader, n *int64, err * tmpSliceRv := reflect.MakeSlice(rt, l, l) for j := 0; j < l; j++ { elemRv := tmpSliceRv.Index(j) - readReflect(elemRv, elemRt, r, n, err) + readReflectBinary(elemRv, elemRt, opts, r, n, err) if *err != nil { return } @@ -254,14 +277,10 @@ func readReflect(rv reflect.Value, rt reflect.Type, r io.Reader, n *int64, err * log.Debug(Fmt("Read time: %v", t)) rv.Set(reflect.ValueOf(t)) } else { - numFields := rt.NumField() - for i := 0; i < numFields; i++ { - field := rt.Field(i) - if field.PkgPath != "" { - continue - } + for _, fieldInfo := range typeInfo.Fields { + i, fieldType, opts := fieldInfo.unpack() fieldRv := rv.Field(i) - readReflect(fieldRv, field.Type, r, n, err) + readReflectBinary(fieldRv, fieldType, opts, r, n, err) } } @@ -271,9 +290,15 @@ func readReflect(rv reflect.Value, rt reflect.Type, r io.Reader, n *int64, err * rv.SetString(str) case reflect.Int64: - num := ReadUint64(r, n, err) - log.Debug(Fmt("Read num: %v", num)) - rv.SetInt(int64(num)) + if opts.Varint { + num := ReadVarint(r, n, err) + log.Debug(Fmt("Read num: %v", num)) + rv.SetInt(int64(num)) + } else { + num := ReadInt64(r, n, err) + log.Debug(Fmt("Read num: %v", num)) + rv.SetInt(int64(num)) + } case reflect.Int32: num := ReadUint32(r, n, err) @@ -291,14 +316,20 @@ func readReflect(rv reflect.Value, rt reflect.Type, r io.Reader, n *int64, err * rv.SetInt(int64(num)) case reflect.Int: - num := ReadUvarint(r, n, err) + num := ReadVarint(r, n, err) log.Debug(Fmt("Read num: %v", num)) rv.SetInt(int64(num)) case reflect.Uint64: - num := ReadUint64(r, n, err) - log.Debug(Fmt("Read num: %v", num)) - rv.SetUint(uint64(num)) + if opts.Varint { + num := ReadUvarint(r, n, err) + log.Debug(Fmt("Read num: %v", num)) + rv.SetUint(uint64(num)) + } else { + num := ReadUint64(r, n, err) + log.Debug(Fmt("Read num: %v", num)) + rv.SetUint(uint64(num)) + } case reflect.Uint32: num := ReadUint32(r, n, err) @@ -332,7 +363,7 @@ func readReflect(rv reflect.Value, rt reflect.Type, r io.Reader, n *int64, err * // rv: the reflection value of the thing to write // rt: the type of rv as declared in the container, not necessarily rv.Type(). -func writeReflect(rv reflect.Value, rt reflect.Type, w io.Writer, n *int64, err *error) { +func writeReflectBinary(rv reflect.Value, rt reflect.Type, opts Options, w io.Writer, n *int64, err *error) { // Get typeInfo typeInfo := GetTypeInfo(rt) @@ -364,8 +395,8 @@ func writeReflect(rv reflect.Value, rt reflect.Type, w io.Writer, n *int64, err // We support writing unsafely for convenience. } // We don't have to write the typeByte here, - // the writeReflect() call below will write it. - writeReflect(crv, crt, w, n, err) + // the writeReflectBinary() call below will write it. + writeReflectBinary(crv, crt, opts, w, n, err) return } @@ -405,7 +436,7 @@ func writeReflect(rv reflect.Value, rt reflect.Type, w io.Writer, n *int64, err // Write elems for i := 0; i < length; i++ { elemRv := rv.Index(i) - writeReflect(elemRv, elemRt, w, n, err) + writeReflectBinary(elemRv, elemRt, opts, w, n, err) } } @@ -414,14 +445,10 @@ func writeReflect(rv reflect.Value, rt reflect.Type, w io.Writer, n *int64, err // Special case: time.Time WriteTime(rv.Interface().(time.Time), w, n, err) } else { - numFields := rt.NumField() - for i := 0; i < numFields; i++ { - field := rt.Field(i) - if field.PkgPath != "" { - continue - } + for _, fieldInfo := range typeInfo.Fields { + i, fieldType, opts := fieldInfo.unpack() fieldRv := rv.Field(i) - writeReflect(fieldRv, field.Type, w, n, err) + writeReflectBinary(fieldRv, fieldType, opts, w, n, err) } } @@ -429,7 +456,11 @@ func writeReflect(rv reflect.Value, rt reflect.Type, w io.Writer, n *int64, err WriteString(rv.String(), w, n, err) case reflect.Int64: - WriteInt64(rv.Int(), w, n, err) + if opts.Varint { + WriteVarint(int(rv.Int()), w, n, err) + } else { + WriteInt64(rv.Int(), w, n, err) + } case reflect.Int32: WriteInt32(int32(rv.Int()), w, n, err) @@ -444,7 +475,11 @@ func writeReflect(rv reflect.Value, rt reflect.Type, w io.Writer, n *int64, err WriteVarint(int(rv.Int()), w, n, err) case reflect.Uint64: - WriteUint64(rv.Uint(), w, n, err) + if opts.Varint { + WriteUvarint(uint(rv.Uint()), w, n, err) + } else { + WriteUint64(rv.Uint(), w, n, err) + } case reflect.Uint32: WriteUint32(uint32(rv.Uint()), w, n, err) @@ -607,8 +642,8 @@ func readReflectJSON(rv reflect.Value, rt reflect.Type, o interface{}, err *erro // TODO: ensure that all fields are set? // TODO: disallow unknown oMap fields? for _, fieldInfo := range typeInfo.Fields { - i, jsonName, fieldType := fieldInfo.unpack() - value, ok := oMap[jsonName] + i, fieldType, opts := fieldInfo.unpack() + value, ok := oMap[opts.JSONName] if !ok { continue // Skip missing fields. } @@ -755,14 +790,14 @@ func writeReflectJSON(rv reflect.Value, rt reflect.Type, w io.Writer, n *int64, WriteTo([]byte("{"), w, n, err) wroteField := false for _, fieldInfo := range typeInfo.Fields { - i, jsonName, fieldType := fieldInfo.unpack() + i, fieldType, opts := fieldInfo.unpack() fieldRv := rv.Field(i) if wroteField { WriteTo([]byte(","), w, n, err) } else { wroteField = true } - WriteTo([]byte(Fmt("\"%v\":", jsonName)), w, n, err) + WriteTo([]byte(Fmt("\"%v\":", opts.JSONName)), w, n, err) writeReflectJSON(fieldRv, fieldType, w, n, err) } WriteTo([]byte("}"), w, n, err) diff --git a/common/bit_array.go b/common/bit_array.go index 2c8e4dada..eb314335b 100644 --- a/common/bit_array.go +++ b/common/bit_array.go @@ -9,8 +9,8 @@ import ( type BitArray struct { mtx sync.Mutex - Bits uint // NOTE: persisted via reflect, must be exported - Elems []uint64 // NOTE: persisted via reflect, must be exported + Bits uint `json:"bits"` // NOTE: persisted via reflect, must be exported + Elems []uint64 `json:"elems"` // NOTE: persisted via reflect, must be exported } func NewBitArray(bits uint) *BitArray {