From 4c961bd565ea0bb5574821a97de9c8ce561de7c9 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Wed, 10 Sep 2014 02:43:16 -0700 Subject: [PATCH] updated readme, implementing mempool. --- README.md | 7 +- blocks/adjustment.go | 166 ------------------------------------------- blocks/block.go | 18 ++--- blocks/block_test.go | 8 +-- blocks/tx.go | 151 ++++++++++++++++++++++++++++++++++++++- mempool/mempool.go | 59 +++++++++++++++ p2p/README.md | 91 +----------------------- 7 files changed, 229 insertions(+), 271 deletions(-) delete mode 100644 blocks/adjustment.go create mode 100644 mempool/mempool.go diff --git a/README.md b/README.md index fed2b59eb..281e97dc8 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,16 @@ TenderMint - proof of concept * **[p2p](https://github.com/tendermint/tendermint/blob/master/p2p):** P2P networking stack. Designed to be extensible. * **[merkle](https://github.com/tendermint/tendermint/blob/master/merkle):** Immutable Persistent Merkle-ized AVL+ Tree, used primarily for keeping track of mutable state like account balances. +* **[blocks](https://github.com/tendermint/tendermint/blob/master/blocks):** The blockchain, storage of blocks, and all the associated structures. +* **[state](https://github.com/tendermint/tendermint/blob/master/state):** The application state, which is mutated by blocks in the blockchain. +* **[consensus](https://github.com/tendermint/tendermint/blob/master/consensus):** The core consensus algorithm logic. +* **[mempool](https://github.com/tendermint/tendermint/blob/master/mempool):** Handles the broadcasting of uncommitted transactions. * **[crypto](https://github.com/tendermint/tendermint/blob/master/crypto):** Includes cgo bindings of ed25519. ### Status -* Consensus *now* +* Mempool *now* +* Consensus *complete* * Block propagation *sidelined* * Node & testnet *complete* * PEX peer exchange *complete* diff --git a/blocks/adjustment.go b/blocks/adjustment.go deleted file mode 100644 index 0cc2f19c8..000000000 --- a/blocks/adjustment.go +++ /dev/null @@ -1,166 +0,0 @@ -package blocks - -import ( - . "github.com/tendermint/tendermint/binary" - . "github.com/tendermint/tendermint/common" - "io" -) - -/* Adjustment - -1. Bond New validator posts a bond -2. Unbond Validator leaves -3. Timeout Validator times out -4. Dupeout Validator dupes out (signs twice) - -TODO: signing a bad checkpoint (block) -*/ -type Adjustment interface { - Type() byte - Binary -} - -const ( - ADJ_TYPE_BOND = byte(0x01) - ADJ_TYPE_UNBOND = byte(0x02) - ADJ_TYPE_TIMEOUT = byte(0x03) - ADJ_TYPE_DUPEOUT = byte(0x04) -) - -func ReadAdjustment(r io.Reader, n *int64, err *error) Adjustment { - switch t := ReadByte(r, n, err); t { - case ADJ_TYPE_BOND: - return &Bond{ - Fee: ReadUInt64(r, n, err), - UnbondTo: ReadUInt64(r, n, err), - Amount: ReadUInt64(r, n, err), - Signature: ReadSignature(r, n, err), - } - case ADJ_TYPE_UNBOND: - return &Unbond{ - Fee: ReadUInt64(r, n, err), - Amount: ReadUInt64(r, n, err), - Signature: ReadSignature(r, n, err), - } - case ADJ_TYPE_TIMEOUT: - return &Timeout{ - AccountId: ReadUInt64(r, n, err), - Penalty: ReadUInt64(r, n, err), - } - case ADJ_TYPE_DUPEOUT: - return &Dupeout{ - VoteA: ReadBlockVote(r, n, err), - VoteB: ReadBlockVote(r, n, err), - } - default: - Panicf("Unknown Adjustment type %x", t) - return nil - } -} - -//----------------------------------------------------------------------------- - -/* Bond < Adjustment */ -type Bond struct { - Fee uint64 - UnbondTo uint64 - Amount uint64 - Signature -} - -func (self *Bond) Type() byte { - return ADJ_TYPE_BOND -} - -func (self *Bond) WriteTo(w io.Writer) (n int64, err error) { - WriteByte(w, self.Type(), &n, &err) - WriteUInt64(w, self.Fee, &n, &err) - WriteUInt64(w, self.UnbondTo, &n, &err) - WriteUInt64(w, self.Amount, &n, &err) - WriteBinary(w, self.Signature, &n, &err) - return -} - -//----------------------------------------------------------------------------- - -/* Unbond < Adjustment */ -type Unbond struct { - Fee uint64 - Amount uint64 - Signature -} - -func (self *Unbond) Type() byte { - return ADJ_TYPE_UNBOND -} - -func (self *Unbond) WriteTo(w io.Writer) (n int64, err error) { - WriteByte(w, self.Type(), &n, &err) - WriteUInt64(w, self.Fee, &n, &err) - WriteUInt64(w, self.Amount, &n, &err) - WriteBinary(w, self.Signature, &n, &err) - return -} - -//----------------------------------------------------------------------------- - -/* Timeout < Adjustment */ -type Timeout struct { - AccountId uint64 - Penalty uint64 -} - -func (self *Timeout) Type() byte { - return ADJ_TYPE_TIMEOUT -} - -func (self *Timeout) WriteTo(w io.Writer) (n int64, err error) { - WriteByte(w, self.Type(), &n, &err) - WriteUInt64(w, self.AccountId, &n, &err) - WriteUInt64(w, self.Penalty, &n, &err) - return -} - -//----------------------------------------------------------------------------- - -/* -The full vote structure is only needed when presented as evidence. -Typically only the signature is passed around, as the hash & height are implied. -*/ -type BlockVote struct { - Height uint64 - BlockHash []byte - Signature -} - -func ReadBlockVote(r io.Reader, n *int64, err *error) BlockVote { - return BlockVote{ - Height: ReadUInt64(r, n, err), - BlockHash: ReadByteSlice(r, n, err), - Signature: ReadSignature(r, n, err), - } -} - -func (self BlockVote) WriteTo(w io.Writer) (n int64, err error) { - WriteUInt64(w, self.Height, &n, &err) - WriteByteSlice(w, self.BlockHash, &n, &err) - WriteBinary(w, self.Signature, &n, &err) - return -} - -/* Dupeout < Adjustment */ -type Dupeout struct { - VoteA BlockVote - VoteB BlockVote -} - -func (self *Dupeout) Type() byte { - return ADJ_TYPE_DUPEOUT -} - -func (self *Dupeout) WriteTo(w io.Writer) (n int64, err error) { - WriteByte(w, self.Type(), &n, &err) - WriteBinary(w, self.VoteA, &n, &err) - WriteBinary(w, self.VoteB, &n, &err) - return -} diff --git a/blocks/block.go b/blocks/block.go index 2a849401a..ce88b5c34 100644 --- a/blocks/block.go +++ b/blocks/block.go @@ -189,8 +189,8 @@ func (h *Header) Hash() []byte { /* Validation is part of a block */ type Validation struct { - Signatures []Signature - Adjustments []Adjustment + Signatures []Signature + Txs []Tx // Volatile hash []byte @@ -203,24 +203,24 @@ func ReadValidation(r io.Reader, n *int64, err *error) Validation { for i := uint32(0); i < numSigs; i++ { sigs = append(sigs, ReadSignature(r, n, err)) } - adjs := make([]Adjustment, 0, numAdjs) + tx := make([]Tx, 0, numAdjs) for i := uint32(0); i < numAdjs; i++ { - adjs = append(adjs, ReadAdjustment(r, n, err)) + tx = append(tx, ReadTx(r, n, err)) } return Validation{ - Signatures: sigs, - Adjustments: adjs, + Signatures: sigs, + Txs: tx, } } func (v *Validation) WriteTo(w io.Writer) (n int64, err error) { WriteUInt32(w, uint32(len(v.Signatures)), &n, &err) - WriteUInt32(w, uint32(len(v.Adjustments)), &n, &err) + WriteUInt32(w, uint32(len(v.Txs)), &n, &err) for _, sig := range v.Signatures { WriteBinary(w, sig, &n, &err) } - for _, adj := range v.Adjustments { - WriteBinary(w, adj, &n, &err) + for _, tx := range v.Txs { + WriteBinary(w, tx, &n, &err) } return } diff --git a/blocks/block_test.go b/blocks/block_test.go index ff0d5d342..fb0344312 100644 --- a/blocks/block_test.go +++ b/blocks/block_test.go @@ -47,7 +47,7 @@ func randSig() Signature { func TestBlock(t *testing.T) { - // Txs + // Account Txs sendTx := &SendTx{ Signature: randSig(), @@ -63,7 +63,7 @@ func TestBlock(t *testing.T) { PubKey: randBytes(32), } - // Adjs + // Consensus Txs bond := &Bond{ Signature: randSig(), @@ -109,8 +109,8 @@ func TestBlock(t *testing.T) { TxsHash: randBytes(32), }, Validation: Validation{ - Signatures: []Signature{randSig(), randSig()}, - Adjustments: []Adjustment{bond, unbond, timeout, dupeout}, + Signatures: []Signature{randSig(), randSig()}, + Txs: []Txs{bond, unbond, timeout, dupeout}, }, Txs: Txs{ Txs: []Tx{sendTx, nameTx}, diff --git a/blocks/tx.go b/blocks/tx.go index 1ae439ec2..552595485 100644 --- a/blocks/tx.go +++ b/blocks/tx.go @@ -18,16 +18,35 @@ Tx wire format: A account number, varint encoded (1+ bytes) S signature of all prior bytes (32 bytes) +Account Txs: +1. Send Send coins to account +2. Name Associate account with a name + +Consensus Txs: +3. Bond New validator posts a bond +4. Unbond Validator leaves +5. Timeout Validator times out +6. Dupeout Validator dupes out (signs twice) + + */ type Tx interface { Type() byte + IsConsensus() bool Binary } const ( + // Account transactions TX_TYPE_SEND = byte(0x01) TX_TYPE_NAME = byte(0x02) + + // Consensus transactions + TX_TYPE_BOND = byte(0x11) + TX_TYPE_UNBOND = byte(0x12) + TX_TYPE_TIMEOUT = byte(0x13) + TX_TYPE_DUPEOUT = byte(0x14) ) func ReadTx(r io.Reader, n *int64, err *error) Tx { @@ -46,13 +65,36 @@ func ReadTx(r io.Reader, n *int64, err *error) Tx { PubKey: ReadByteSlice(r, n, err), Signature: ReadSignature(r, n, err), } + case TX_TYPE_BOND: + return &BondTx{ + Fee: ReadUInt64(r, n, err), + UnbondTo: ReadUInt64(r, n, err), + Amount: ReadUInt64(r, n, err), + Signature: ReadSignature(r, n, err), + } + case TX_TYPE_UNBOND: + return &UnbondTx{ + Fee: ReadUInt64(r, n, err), + Amount: ReadUInt64(r, n, err), + Signature: ReadSignature(r, n, err), + } + case TX_TYPE_TIMEOUT: + return &TimeoutTx{ + AccountId: ReadUInt64(r, n, err), + Penalty: ReadUInt64(r, n, err), + } + case TX_TYPE_DUPEOUT: + return &DupeoutTx{ + VoteA: ReadBlockVote(r, n, err), + VoteB: ReadBlockVote(r, n, err), + } default: Panicf("Unknown Tx type %x", t) return nil } } -/* SendTx < Tx */ +//----------------------------------------------------------------------------- type SendTx struct { Fee uint64 @@ -74,7 +116,7 @@ func (self *SendTx) WriteTo(w io.Writer) (n int64, err error) { return } -/* NameTx < Tx */ +//----------------------------------------------------------------------------- type NameTx struct { Fee uint64 @@ -95,3 +137,108 @@ func (self *NameTx) WriteTo(w io.Writer) (n int64, err error) { WriteBinary(w, self.Signature, &n, &err) return } + +//----------------------------------------------------------------------------- + +type BondTx struct { + Fee uint64 + UnbondTo uint64 + Amount uint64 + Signature +} + +func (self *BondTx) Type() byte { + return TX_TYPE_BOND +} + +func (self *BondTx) WriteTo(w io.Writer) (n int64, err error) { + WriteByte(w, self.Type(), &n, &err) + WriteUInt64(w, self.Fee, &n, &err) + WriteUInt64(w, self.UnbondTo, &n, &err) + WriteUInt64(w, self.Amount, &n, &err) + WriteBinary(w, self.Signature, &n, &err) + return +} + +//----------------------------------------------------------------------------- + +type UnbondTx struct { + Fee uint64 + Amount uint64 + Signature +} + +func (self *UnbondTx) Type() byte { + return TX_TYPE_UNBOND +} + +func (self *UnbondTx) WriteTo(w io.Writer) (n int64, err error) { + WriteByte(w, self.Type(), &n, &err) + WriteUInt64(w, self.Fee, &n, &err) + WriteUInt64(w, self.Amount, &n, &err) + WriteBinary(w, self.Signature, &n, &err) + return +} + +//----------------------------------------------------------------------------- + +type TimeoutTx struct { + AccountId uint64 + Penalty uint64 +} + +func (self *TimeoutTx) Type() byte { + return TX_TYPE_TIMEOUT +} + +func (self *TimeoutTx) WriteTo(w io.Writer) (n int64, err error) { + WriteByte(w, self.Type(), &n, &err) + WriteUInt64(w, self.AccountId, &n, &err) + WriteUInt64(w, self.Penalty, &n, &err) + return +} + +//----------------------------------------------------------------------------- + +/* +The full vote structure is only needed when presented as evidence. +Typically only the signature is passed around, as the hash & height are implied. +*/ +type BlockVote struct { + Height uint64 + BlockHash []byte + Signature +} + +func ReadBlockVote(r io.Reader, n *int64, err *error) BlockVote { + return BlockVote{ + Height: ReadUInt64(r, n, err), + BlockHash: ReadByteSlice(r, n, err), + Signature: ReadSignature(r, n, err), + } +} + +func (self BlockVote) WriteTo(w io.Writer) (n int64, err error) { + WriteUInt64(w, self.Height, &n, &err) + WriteByteSlice(w, self.BlockHash, &n, &err) + WriteBinary(w, self.Signature, &n, &err) + return +} + +//----------------------------------------------------------------------------- + +type DupeoutTx struct { + VoteA BlockVote + VoteB BlockVote +} + +func (self *DupeoutTx) Type() byte { + return TX_TYPE_DUPEOUT +} + +func (self *DupeoutTx) WriteTo(w io.Writer) (n int64, err error) { + WriteByte(w, self.Type(), &n, &err) + WriteBinary(w, self.VoteA, &n, &err) + WriteBinary(w, self.VoteB, &n, &err) + return +} diff --git a/mempool/mempool.go b/mempool/mempool.go new file mode 100644 index 000000000..400530d2d --- /dev/null +++ b/mempool/mempool.go @@ -0,0 +1,59 @@ +package mempool + +import ( + "sync" + + . "github.com/tendermint/tendermint/blocks" + . "github.com/tendermint/tendermint/state" +) + +/* +Mempool receives new transactions and applies them to the latest committed state. +If the transaction is acceptable, then it broadcasts a fingerprint to peers. + +The transaction fingerprint is a short sequence of bytes (shorter than a full hash). +Each peer connection uses a different algorithm for turning the tx hash into a +fingerprint in order to prevent transaction blocking attacks. Upon inspecting a +tx fingerprint, the receiver may query the source for the full tx bytes. + +When this node happens to be the next proposer, it simply takes the recently +modified state (and the associated transactions) and use that as the proposal. + +There are two types of transactions -- consensus txs (e.g. bonding / unbonding / +timeout / dupeout txs) and everything else. They are stored separately to allow +nodes to only request the kind they need. +TODO: make use of this potential feature when the time comes. + +For simplicity we evaluate the consensus transactions after everything else. +*/ + +//----------------------------------------------------------------------------- + +type Mempool struct { + mtx sync.Mutex + state *State + txs []Tx // Regular transactions + ctxs []Tx // Validator related transactions +} + +func NewMempool(state *State) *Mempool { + return &Mempool{ + state: state, + } +} + +func (mem *Mempool) AddTx(tx Tx) bool { + mem.mtx.Lock() + defer mem.mtx.Unlock() + if tx.IsConsensus() { + // Remember consensus tx for later staging. + // We only keep 1 tx for each validator. TODO what? what about bonding? + // TODO talk about prioritization. + mem.ctxs = append(mem.ctxs, tx) + } else { + mem.txs = append(mem.txs, tx) + } +} + +func (mem *Mempool) CollectForState() { +} diff --git a/p2p/README.md b/p2p/README.md index 38b0d0158..0fda9fdb8 100644 --- a/p2p/README.md +++ b/p2p/README.md @@ -1,11 +1,8 @@ ## Channels Each peer connection is multiplexed into channels. -
- -### PEX channel - -The PEX channel is used to exchange peer addresses. +The p2p module comes with a channel implementation used for peer +discovery (called PEX, short for "peer exchange"). @@ -24,90 +21,6 @@ The PEX channel is used to exchange peer addresses.
-### Block channel - -The block channel is used to propagate block or header information to new peers or peers catching up with the blockchain. - - - - - - - - - - - - - - -
Channel"block"
Messages -
    -
  • RequestMsg
    • -
  • BlockMsg
    • -
  • HeaderMsg
    • -
-
Notes - Nodes should only advertise having a header or block at height 'h' if it also has all the headers or blocks less than 'h'. Thus for each peer we need only keep track of two integers -- one for the most recent header height 'h_h' and one for the most recent block height 'h_b', where 'h_b' <= 'h_h'. -
-
- -### Mempool channel - -The mempool channel is used for broadcasting new transactions that haven't yet entered the blockchain. It uses a lossy bloom filter on either end, but with sufficient fanout and filter nonce updates every new block, all transactions will eventually reach every node. - - - - - - - - - - - - - - -
Channel"mempool"
Messages -
    -
  • MempoolTxMsg
    • -
-
Notes - Instead of keeping a perfect inventory of what peers have, we use a lossy filter.
- Bloom filter (n:10k, p:0.02 -> k:6, m:10KB)
- Each peer's filter has a random nonce that scrambles the message hashes.
- The filter & nonce refreshes every new block.
-
-
- -### Consensus channel - -The consensus channel broadcasts all information used in the rounds of the Tendermint consensus mechanism. - - - - - - - - - - - - - - -
Channel"consensus"
Messages -
    -
  • ProposalMsg
    • -
  • VoteMsg
    • -
  • NewBlockMsg
    • -
-
Notes - How do optimize/balance propagation speed & bandwidth utilization? -
- ## Resources * http://www.upnp-hacks.org/upnp.html