(U)VarInt binary methods; Txs have sequence numbers

binary/binary.go

@@ -15,6 +15,8 @@ func WriteBinary(w io.Writer, b Binary, n *int64, err *error) {
 	*err = err_
 }
 
+// Write all of bz to w
+// Increment n and set err accordingly.
 func WriteTo(w io.Writer, bz []byte, n *int64, err *error) {
 	if *err != nil {
 		return
@@ -24,6 +26,8 @@ func WriteTo(w io.Writer, bz []byte, n *int64, err *error) {
 	*err = err_
 }
 
+// Read len(buf) from r
+// Increment n and set err accordingly.
 func ReadFull(r io.Reader, buf []byte, n *int64, err *error) {
 	if *err != nil {
 		return

binary/int.go

@@ -2,6 +2,7 @@ package binary
 
 import (
 	"encoding/binary"
+	"errors"
 	"io"
 )
 
@@ -42,6 +43,7 @@ func ReadUInt8(r io.Reader, n *int64, err *error) uint8 {
 func WriteInt16(w io.Writer, i int16, n *int64, err *error) {
 	buf := make([]byte, 2)
 	binary.LittleEndian.PutUint16(buf, uint16(i))
+	*n += 2
 	WriteTo(w, buf, n, err)
 }
 
@@ -56,6 +58,7 @@ func ReadInt16(r io.Reader, n *int64, err *error) int16 {
 func WriteUInt16(w io.Writer, i uint16, n *int64, err *error) {
 	buf := make([]byte, 2)
 	binary.LittleEndian.PutUint16(buf, uint16(i))
+	*n += 2
 	WriteTo(w, buf, n, err)
 }
 
@@ -70,6 +73,7 @@ func ReadUInt16(r io.Reader, n *int64, err *error) uint16 {
 func WriteInt32(w io.Writer, i int32, n *int64, err *error) {
 	buf := make([]byte, 4)
 	binary.LittleEndian.PutUint32(buf, uint32(i))
+	*n += 4
 	WriteTo(w, buf, n, err)
 }
 
@@ -84,6 +88,7 @@ func ReadInt32(r io.Reader, n *int64, err *error) int32 {
 func WriteUInt32(w io.Writer, i uint32, n *int64, err *error) {
 	buf := make([]byte, 4)
 	binary.LittleEndian.PutUint32(buf, uint32(i))
+	*n += 4
 	WriteTo(w, buf, n, err)
 }
 
@@ -98,6 +103,7 @@ func ReadUInt32(r io.Reader, n *int64, err *error) uint32 {
 func WriteInt64(w io.Writer, i int64, n *int64, err *error) {
 	buf := make([]byte, 8)
 	binary.LittleEndian.PutUint64(buf, uint64(i))
+	*n += 8
 	WriteTo(w, buf, n, err)
 }
 
@@ -112,6 +118,7 @@ func ReadInt64(r io.Reader, n *int64, err *error) int64 {
 func WriteUInt64(w io.Writer, i uint64, n *int64, err *error) {
 	buf := make([]byte, 8)
 	binary.LittleEndian.PutUint64(buf, uint64(i))
+	*n += 8
 	WriteTo(w, buf, n, err)
 }
 
@@ -120,3 +127,76 @@ func ReadUInt64(r io.Reader, n *int64, err *error) uint64 {
 	ReadFull(r, buf, n, err)
 	return uint64(binary.LittleEndian.Uint64(buf))
 }
+
+// VarInt
+
+func WriteVarInt(w io.Writer, i int64, n *int64, err *error) {
+	buf := make([]byte, 9)
+	*n += int64(binary.PutVarint(buf, int64(i)))
+	WriteTo(w, buf, n, err)
+}
+
+func ReadVarInt(r io.Reader, n *int64, err *error) int64 {
+	res, n_, err_ := readVarint(r)
+	*n += n_
+	*err = err_
+	return res
+}
+
+// UVarInt
+
+func WriteUVarInt(w io.Writer, i uint64, n *int64, err *error) {
+	buf := make([]byte, 9)
+	*n += int64(binary.PutUvarint(buf, uint64(i)))
+	WriteTo(w, buf, n, err)
+}
+
+func ReadUVarInt(r io.Reader, n *int64, err *error) uint64 {
+	res, n_, err_ := readUvarint(r)
+	*n += n_
+	*err = err_
+	return res
+}
+
+//-----------------------------------------------------------------------------
+
+var overflow = errors.New("binary: varint overflows a 64-bit integer")
+
+// Modified to return number of bytes read, from
+// http://golang.org/src/pkg/encoding/binary/varint.go?s=3652:3699#L116
+func readUvarint(r io.Reader) (uint64, int64, error) {
+	var x uint64
+	var s uint
+	var buf = make([]byte, 1)
+	for i := 0; ; i++ {
+		for {
+			n, err := r.Read(buf)
+			if err != nil {
+				return x, int64(i), err
+			}
+			if n > 0 {
+				break
+			}
+		}
+		b := buf[0]
+		if b < 0x80 {
+			if i > 9 || i == 9 && b > 1 {
+				return x, int64(i), overflow
+			}
+			return x | uint64(b)<<s, int64(i), nil
+		}
+		x |= uint64(b&0x7f) << s
+		s += 7
+	}
+}
+
+// Modified to return number of bytes read, from
+// http://golang.org/src/pkg/encoding/binary/varint.go?s=3652:3699#L116
+func readVarint(r io.Reader) (int64, int64, error) {
+	ux, n, err := readUvarint(r) // ok to continue in presence of error
+	x := int64(ux >> 1)
+	if ux&1 != 0 {
+		x = ^x
+	}
+	return x, n, err
+}

blocks/tx.go

@@ -33,7 +33,9 @@ Consensus Txs:
 
 type Tx interface {
 	Type() byte
-	IsConsensus() bool
+	GetSequence() uint64
+	GetSignature() *Signature
+	//IsConsensus() bool
 	Binary
 }
 
@@ -53,40 +55,42 @@ func ReadTx(r io.Reader, n *int64, err *error) Tx {
 	switch t := ReadByte(r, n, err); t {
 	case TX_TYPE_SEND:
 		return &SendTx{
-			Fee:       ReadUInt64(r, n, err),
-			To:        ReadUInt64(r, n, err),
-			Amount:    ReadUInt64(r, n, err),
-			Signature: ReadSignature(r, n, err),
+			BaseTx: ReadBaseTx(r, n, err),
+			Fee:    ReadUInt64(r, n, err),
+			To:     ReadUInt64(r, n, err),
+			Amount: ReadUInt64(r, n, err),
 		}
 	case TX_TYPE_NAME:
 		return &NameTx{
-			Fee:       ReadUInt64(r, n, err),
-			Name:      ReadString(r, n, err),
-			PubKey:    ReadByteSlice(r, n, err),
-			Signature: ReadSignature(r, n, err),
+			BaseTx: ReadBaseTx(r, n, err),
+			Fee:    ReadUInt64(r, n, err),
+			Name:   ReadString(r, n, err),
+			PubKey: ReadByteSlice(r, n, err),
 		}
 	case TX_TYPE_BOND:
 		return &BondTx{
-			Fee:       ReadUInt64(r, n, err),
-			UnbondTo:  ReadUInt64(r, n, err),
-			Amount:    ReadUInt64(r, n, err),
-			Signature: ReadSignature(r, n, err),
+			BaseTx:   ReadBaseTx(r, n, err),
+			Fee:      ReadUInt64(r, n, err),
+			UnbondTo: ReadUInt64(r, n, err),
+			Amount:   ReadUInt64(r, n, err),
 		}
 	case TX_TYPE_UNBOND:
 		return &UnbondTx{
-			Fee:       ReadUInt64(r, n, err),
-			Amount:    ReadUInt64(r, n, err),
-			Signature: ReadSignature(r, n, err),
+			BaseTx: ReadBaseTx(r, n, err),
+			Fee:    ReadUInt64(r, n, err),
+			Amount: ReadUInt64(r, n, err),
 		}
 	case TX_TYPE_TIMEOUT:
 		return &TimeoutTx{
+			BaseTx:    ReadBaseTx(r, n, err),
 			AccountId: ReadUInt64(r, n, err),
 			Penalty:   ReadUInt64(r, n, err),
 		}
 	case TX_TYPE_DUPEOUT:
 		return &DupeoutTx{
-			VoteA: ReadBlockVote(r, n, err),
-			VoteB: ReadBlockVote(r, n, err),
+			BaseTx: ReadBaseTx(r, n, err),
+			VoteA:  *ReadBlockVote(r, n, err),
+			VoteB:  *ReadBlockVote(r, n, err),
 		}
 	default:
 		Panicf("Unknown Tx type %x", t)
@@ -96,105 +100,135 @@ func ReadTx(r io.Reader, n *int64, err *error) Tx {
 
 //-----------------------------------------------------------------------------
 
+type BaseTx struct {
+	Sequence uint64
+	Signature
+}
+
+func ReadBaseTx(r io.Reader, n *int64, err *error) BaseTx {
+	return BaseTx{
+		Sequence:  ReadUVarInt(r, n, err),
+		Signature: ReadSignature(r, n, err),
+	}
+}
+
+func (tx *BaseTx) GetSequence() uint64 {
+	return tx.Sequence
+}
+
+func (tx *BaseTx) GetSignature() *Signature {
+	return &tx.Signature
+}
+
+func (tx *BaseTx) WriteTo(w io.Writer) (n int64, err error) {
+	WriteUVarInt(w, tx.Sequence, &n, &err)
+	WriteBinary(w, tx.Signature, &n, &err)
+	return
+}
+
+//-----------------------------------------------------------------------------
+
 type SendTx struct {
+	BaseTx
 	Fee    uint64
 	To     uint64
 	Amount uint64
-	Signature
 }
 
-func (self *SendTx) Type() byte {
+func (tx *SendTx) Type() byte {
 	return TX_TYPE_SEND
 }
 
-func (self *SendTx) WriteTo(w io.Writer) (n int64, err error) {
-	WriteByte(w, self.Type(), &n, &err)
-	WriteUInt64(w, self.Fee, &n, &err)
-	WriteUInt64(w, self.To, &n, &err)
-	WriteUInt64(w, self.Amount, &n, &err)
-	WriteBinary(w, self.Signature, &n, &err)
+func (tx *SendTx) WriteTo(w io.Writer) (n int64, err error) {
+	WriteByte(w, tx.Type(), &n, &err)
+	WriteBinary(w, &tx.BaseTx, &n, &err)
+	WriteUInt64(w, tx.Fee, &n, &err)
+	WriteUInt64(w, tx.To, &n, &err)
+	WriteUInt64(w, tx.Amount, &n, &err)
 	return
 }
 
 //-----------------------------------------------------------------------------
 
 type NameTx struct {
+	BaseTx
 	Fee    uint64
 	Name   string
 	PubKey []byte
-	Signature
 }
 
-func (self *NameTx) Type() byte {
+func (tx *NameTx) Type() byte {
 	return TX_TYPE_NAME
 }
 
-func (self *NameTx) WriteTo(w io.Writer) (n int64, err error) {
-	WriteByte(w, self.Type(), &n, &err)
-	WriteUInt64(w, self.Fee, &n, &err)
-	WriteString(w, self.Name, &n, &err)
-	WriteByteSlice(w, self.PubKey, &n, &err)
-	WriteBinary(w, self.Signature, &n, &err)
+func (tx *NameTx) WriteTo(w io.Writer) (n int64, err error) {
+	WriteByte(w, tx.Type(), &n, &err)
+	WriteBinary(w, &tx.BaseTx, &n, &err)
+	WriteUInt64(w, tx.Fee, &n, &err)
+	WriteString(w, tx.Name, &n, &err)
+	WriteByteSlice(w, tx.PubKey, &n, &err)
 	return
 }
 
 //-----------------------------------------------------------------------------
 
 type BondTx struct {
+	BaseTx
 	Fee      uint64
 	UnbondTo uint64
 	Amount   uint64
-	Signature
 }
 
-func (self *BondTx) Type() byte {
+func (tx *BondTx) Type() byte {
 	return TX_TYPE_BOND
 }
 
-func (self *BondTx) WriteTo(w io.Writer) (n int64, err error) {
-	WriteByte(w, self.Type(), &n, &err)
-	WriteUInt64(w, self.Fee, &n, &err)
-	WriteUInt64(w, self.UnbondTo, &n, &err)
-	WriteUInt64(w, self.Amount, &n, &err)
-	WriteBinary(w, self.Signature, &n, &err)
+func (tx *BondTx) WriteTo(w io.Writer) (n int64, err error) {
+	WriteByte(w, tx.Type(), &n, &err)
+	WriteBinary(w, &tx.BaseTx, &n, &err)
+	WriteUInt64(w, tx.Fee, &n, &err)
+	WriteUInt64(w, tx.UnbondTo, &n, &err)
+	WriteUInt64(w, tx.Amount, &n, &err)
 	return
 }
 
 //-----------------------------------------------------------------------------
 
 type UnbondTx struct {
+	BaseTx
 	Fee    uint64
 	Amount uint64
-	Signature
 }
 
-func (self *UnbondTx) Type() byte {
+func (tx *UnbondTx) Type() byte {
 	return TX_TYPE_UNBOND
 }
 
-func (self *UnbondTx) WriteTo(w io.Writer) (n int64, err error) {
-	WriteByte(w, self.Type(), &n, &err)
-	WriteUInt64(w, self.Fee, &n, &err)
-	WriteUInt64(w, self.Amount, &n, &err)
-	WriteBinary(w, self.Signature, &n, &err)
+func (tx *UnbondTx) WriteTo(w io.Writer) (n int64, err error) {
+	WriteByte(w, tx.Type(), &n, &err)
+	WriteBinary(w, &tx.BaseTx, &n, &err)
+	WriteUInt64(w, tx.Fee, &n, &err)
+	WriteUInt64(w, tx.Amount, &n, &err)
 	return
 }
 
 //-----------------------------------------------------------------------------
 
 type TimeoutTx struct {
+	BaseTx
 	AccountId uint64
 	Penalty   uint64
 }
 
-func (self *TimeoutTx) Type() byte {
+func (tx *TimeoutTx) Type() byte {
 	return TX_TYPE_TIMEOUT
 }
 
-func (self *TimeoutTx) WriteTo(w io.Writer) (n int64, err error) {
-	WriteByte(w, self.Type(), &n, &err)
-	WriteUInt64(w, self.AccountId, &n, &err)
-	WriteUInt64(w, self.Penalty, &n, &err)
+func (tx *TimeoutTx) WriteTo(w io.Writer) (n int64, err error) {
+	WriteByte(w, tx.Type(), &n, &err)
+	WriteBinary(w, &tx.BaseTx, &n, &err)
+	WriteUInt64(w, tx.AccountId, &n, &err)
+	WriteUInt64(w, tx.Penalty, &n, &err)
 	return
 }
 
@@ -210,35 +244,37 @@ type BlockVote struct {
 	Signature
 }
 
-func ReadBlockVote(r io.Reader, n *int64, err *error) BlockVote {
-	return BlockVote{
+func ReadBlockVote(r io.Reader, n *int64, err *error) *BlockVote {
+	return &BlockVote{
 		Height:    ReadUInt64(r, n, err),
 		BlockHash: ReadByteSlice(r, n, err),
 		Signature: ReadSignature(r, n, err),
 	}
 }
 
-func (self BlockVote) WriteTo(w io.Writer) (n int64, err error) {
-	WriteUInt64(w, self.Height, &n, &err)
-	WriteByteSlice(w, self.BlockHash, &n, &err)
-	WriteBinary(w, self.Signature, &n, &err)
+func (tx BlockVote) WriteTo(w io.Writer) (n int64, err error) {
+	WriteUInt64(w, tx.Height, &n, &err)
+	WriteByteSlice(w, tx.BlockHash, &n, &err)
+	WriteBinary(w, tx.Signature, &n, &err)
 	return
 }
 
 //-----------------------------------------------------------------------------
 
 type DupeoutTx struct {
+	BaseTx
 	VoteA BlockVote
 	VoteB BlockVote
 }
 
-func (self *DupeoutTx) Type() byte {
+func (tx *DupeoutTx) Type() byte {
 	return TX_TYPE_DUPEOUT
 }
 
-func (self *DupeoutTx) WriteTo(w io.Writer) (n int64, err error) {
-	WriteByte(w, self.Type(), &n, &err)
-	WriteBinary(w, self.VoteA, &n, &err)
-	WriteBinary(w, self.VoteB, &n, &err)
+func (tx *DupeoutTx) WriteTo(w io.Writer) (n int64, err error) {
+	WriteByte(w, tx.Type(), &n, &err)
+	WriteBinary(w, &tx.BaseTx, &n, &err)
+	WriteBinary(w, tx.VoteA, &n, &err)
+	WriteBinary(w, tx.VoteB, &n, &err)
 	return
 }

mempool/mempool.go

@@ -18,13 +18,6 @@ tx fingerprint, the receiver may query the source for the full tx bytes.
 
 When this node happens to be the next proposer, it simply takes the recently
 modified state (and the associated transactions) and use that as the proposal.
-
-There are two types of transactions -- consensus txs (e.g. bonding / unbonding /
-timeout / dupeout txs) and everything else. They are stored separately to allow
-nodes to only request the kind they need.
-TODO: make use of this potential feature when the time comes.
-
-For simplicity we evaluate the consensus transactions after everything else.
 */
 
 //-----------------------------------------------------------------------------
@@ -32,8 +25,7 @@ For simplicity we evaluate the consensus transactions after everything else.
 type Mempool struct {
 	mtx   sync.Mutex
 	state *State
-	txs   []Tx // Regular transactions
-	ctxs  []Tx // Validator related transactions
+	txs   []Tx
 }
 
 func NewMempool(state *State) *Mempool {
@@ -42,18 +34,15 @@ func NewMempool(state *State) *Mempool {
 	}
 }
 
-func (mem *Mempool) AddTx(tx Tx) bool {
+func (mem *Mempool) AddTx(tx Tx) (err error) {
 	mem.mtx.Lock()
 	defer mem.mtx.Unlock()
-	if tx.IsConsensus() {
-		// Remember consensus tx for later staging.
-		// We only keep 1 tx for each validator. TODO what? what about bonding?
-		// TODO talk about prioritization.
-		mem.ctxs = append(mem.ctxs, tx)
+	// Add the tx to the state.
+	err = mem.state.CommitTx(tx)
+	if err != nil {
+		return err
 	} else {
 		mem.txs = append(mem.txs, tx)
+		return nil
 	}
 }
-
-func (mem *Mempool) CollectForState() {
-}

state/state.go

@@ -2,6 +2,7 @@ package state
 
 import (
 	"bytes"
+	"errors"
 	"sync"
 	"time"
 
@@ -12,6 +13,8 @@ import (
 )
 
 var (
+	ErrStateInvalidSequenceNumber = errors.New("Error State invalid sequence number")
+
 	stateKey = []byte("stateKey")
 )
 
@@ -87,9 +90,17 @@ func (s *State) Copy() *State {
 	}
 }
 
-func (s *State) CommitTx(tx *Tx) error {
+// May return ErrStateInvalidSequenceNumber
+func (s *State) CommitTx(tx Tx) error {
 	s.mtx.Lock()
 	defer s.mtx.Unlock()
+	/*
+		// Get the signer's incr
+		signerId := tx.Signature().SignerId
+		if mem.state.AccountSequence(signerId) != tx.Sequence() {
+			return ErrStateInvalidSequenceNumber
+		}
+	*/
 	// TODO commit the tx
 	panic("Implement CommitTx()")
 	return nil