From 62ff48c02b21ae65e4fbab2b45e50aec81f65ebb Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Tue, 23 Dec 2014 01:35:54 -0800 Subject: [PATCH] Documented Block, some renames --- block/README.md | 61 ++++++++++++++++++++++++++++++++++++++++++ block/block.go | 44 +++++++++++++++++------------- block/part_set.go | 11 ++++---- block/store.go | 52 ++++++++++++++++++++++------------- block/tx.go | 28 ++++++++++--------- block/vote.go | 15 ++++++----- common/bit_array.go | 4 +-- consensus/pol.go | 2 +- consensus/state.go | 27 ++++++++++--------- consensus/vote_set.go | 8 +++--- state/state_test.go | 1 + state/validator_set.go | 4 +-- 12 files changed, 174 insertions(+), 83 deletions(-) create mode 100644 block/README.md diff --git a/block/README.md b/block/README.md new file mode 100644 index 000000000..f99294b00 --- /dev/null +++ b/block/README.md @@ -0,0 +1,61 @@ +# `tendermint/block` + +## Block + +TODO: document + +### Header + +### Validation + +### Data + +## PartSet + +PartSet is used to split a byteslice of data into parts (pieces) for transmission. +By splitting data into smaller parts and computing a Merkle root hash on the list, +you can verify that a part is legitimately part of the complete data, and the +part can be forwarded to other peers before all the parts are known. In short, +it's a fast way to propagate a large file over a gossip network. + +PartSet was inspired by the LibSwift project. + +Usage: + +```Go +data := RandBytes(2 << 20) // Something large + +partSet := NewPartSetFromData(data) +partSet.Total() // Total number of 4KB parts +partSet.Count() // Equal to the Total, since we already have all the parts +partSet.Hash() // The Merkle root hash +partSet.BitArray() // A BitArray of partSet.Total() 1's + +header := partSet.Header() // Send this to the peer +header.Total // Total number of parts +header.Hash // The merkle root hash + +// Now we'll reconstruct the data from the parts +partSet2 := NewPartSetFromHeader(header) +partSet2.Total() // Same total as partSet.Total() +partSet2.Count() // Zero, since this PartSet doesn't have any parts yet. +partSet2.Hash() // Same hash as in partSet.Hash() +partSet2.BitArray() // A BitArray of partSet.Total() 0's + +// In a gossip network the parts would arrive in arbitrary order, perhaps +// in response to explicit requests for parts, or optimistically in response +// to the receiving peer's partSet.BitArray(). +for !partSet2.IsComplete() { + part := receivePartFromGossipNetwork() + added, err := partSet2.AddPart(part) + if err != nil { + // A wrong part, + // the merkle trail does not hash to partSet2.Hash() + } else if !added { + // A duplicate part already received + } +} + +data2, _ := ioutil.ReadAll(partSet2.GetReader()) +bytes.Equal(data, data2) // true +``` diff --git a/block/block.go b/block/block.go index 469b5182e..e3533dca9 100644 --- a/block/block.go +++ b/block/block.go @@ -28,20 +28,25 @@ type Block struct { func (b *Block) ValidateBasic(lastBlockHeight uint, lastBlockHash []byte, lastBlockParts PartSetHeader, lastBlockTime time.Time) error { if b.Network != Config.Network { - return errors.New("Invalid block network") + return errors.New("Wrong Block.Header.Network") } if b.Height != lastBlockHeight+1 { - return errors.New("Invalid block height") + return errors.New("Wrong Block.Header.Height") + } + if b.NumTxs != uint(len(b.Data.Txs)) { + return errors.New("Wrong Block.Header.NumTxs") } if !bytes.Equal(b.LastBlockHash, lastBlockHash) { - return errors.New("Invalid block hash") + return errors.New("Wrong Block.Header.LastBlockHash") } if !b.LastBlockParts.Equals(lastBlockParts) { - return errors.New("Invalid block parts header") - } - if !b.Time.After(lastBlockTime) { - return errors.New("Invalid block time") + return errors.New("Wrong Block.Header.LastBlockParts") } + /* TODO: Determine bounds. + if !b.Time.After(lastBlockTime) { + return errors.New("Invalid Block.Header.Time") + } + */ if b.Header.Height != 1 { if err := b.Validation.ValidateBasic(); err != nil { return err @@ -78,22 +83,22 @@ func (b *Block) HashesTo(hash []byte) bool { } func (b *Block) String() string { - return b.StringWithIndent("") + return b.StringIndented("") } -func (b *Block) StringWithIndent(indent string) string { +func (b *Block) StringIndented(indent string) string { return fmt.Sprintf(`Block{ %s %v %s %v %s %v %s}#%X`, - indent, b.Header.StringWithIndent(indent+" "), - indent, b.Validation.StringWithIndent(indent+" "), - indent, b.Data.StringWithIndent(indent+" "), + indent, b.Header.StringIndented(indent+" "), + indent, b.Validation.StringIndented(indent+" "), + indent, b.Data.StringIndented(indent+" "), indent, b.hash) } -func (b *Block) Description() string { +func (b *Block) StringShort() string { if b == nil { return "nil-Block" } else { @@ -108,6 +113,7 @@ type Header struct { Height uint Time time.Time Fees uint64 + NumTxs uint LastBlockHash []byte LastBlockParts PartSetHeader StateHash []byte @@ -128,12 +134,13 @@ func (h *Header) Hash() []byte { return h.hash } -func (h *Header) StringWithIndent(indent string) string { +func (h *Header) StringIndented(indent string) string { return fmt.Sprintf(`Header{ %s Network: %v %s Height: %v %s Time: %v %s Fees: %v +%s NumTxs: %v %s LastBlockHash: %X %s LastBlockParts: %v %s StateHash: %X @@ -142,6 +149,7 @@ func (h *Header) StringWithIndent(indent string) string { indent, h.Height, indent, h.Time, indent, h.Fees, + indent, h.NumTxs, indent, h.LastBlockHash, indent, h.LastBlockParts, indent, h.StateHash, @@ -166,8 +174,8 @@ func (commit Commit) String() string { //------------------------------------- -// NOTE: The Commits are in order of address to preserve the active ValidatorSet order. -// Any peer with a block can gossip commits by index with a peer catching up without recalculating the +// NOTE: The Commits are in order of address to preserve the bonded ValidatorSet order. +// Any peer with a block can gossip commits by index with a peer without recalculating the // active ValidatorSet. type Validation struct { Commits []Commit // Commits (or nil) of all active validators in address order. @@ -212,7 +220,7 @@ func (v *Validation) Hash() []byte { return v.hash } -func (v *Validation) StringWithIndent(indent string) string { +func (v *Validation) StringIndented(indent string) string { commitStrings := make([]string, len(v.Commits)) for i, commit := range v.Commits { commitStrings[i] = commit.String() @@ -254,7 +262,7 @@ func (data *Data) Hash() []byte { return data.hash } -func (data *Data) StringWithIndent(indent string) string { +func (data *Data) StringIndented(indent string) string { txStrings := make([]string, len(data.Txs)) for i, tx := range data.Txs { txStrings[i] = fmt.Sprintf("Tx:%v", tx) diff --git a/block/part_set.go b/block/part_set.go index 71d090f2c..046b95430 100644 --- a/block/part_set.go +++ b/block/part_set.go @@ -46,10 +46,10 @@ func (part *Part) Hash() []byte { } func (part *Part) String() string { - return part.StringWithIndent("") + return part.StringIndented("") } -func (part *Part) StringWithIndent(indent string) string { +func (part *Part) StringIndented(indent string) string { trailStrings := make([]string, len(part.Trail)) for i, hash := range part.Trail { trailStrings[i] = fmt.Sprintf("%X", hash) @@ -96,8 +96,8 @@ type PartSet struct { count uint } -// Returns an immutable, full PartSet. -// TODO Name is confusing, Data/Header clash with Block.Data/Header +// Returns an immutable, full PartSet from the data bytes. +// The data bytes are split into "partSize" chunks, and merkle tree computed. func NewPartSetFromData(data []byte) *PartSet { // divide data into 4kb parts. total := (len(data) + partSize - 1) / partSize @@ -128,7 +128,6 @@ func NewPartSetFromData(data []byte) *PartSet { } // Returns an empty PartSet ready to be populated. -// TODO Name is confusing, Data/Header clash with Block.Data/Header func NewPartSetFromHeader(header PartSetHeader) *PartSet { return &PartSet{ total: header.Total, @@ -239,7 +238,7 @@ func (ps *PartSet) GetReader() io.Reader { return bytes.NewReader(buf) } -func (ps *PartSet) Description() string { +func (ps *PartSet) StringShort() string { if ps == nil { return "nil-PartSet" } else { diff --git a/block/store.go b/block/store.go index b7c5aac72..fd4654610 100644 --- a/block/store.go +++ b/block/store.go @@ -11,10 +11,17 @@ import ( db_ "github.com/tendermint/tendermint/db" ) -//----------------------------------------------------------------------------- - /* -Simple low level store for blocks, which is actually stored as separte parts (wire format). +Simple low level store for blocks. + +There are three types of information stored: + - BlockMeta: Meta information about each block + - Block part: Parts of each block, aggregated w/ PartSet + - Validation: The Validation part of each block, for gossiping commit votes + +Currently the commit signatures are duplicated in the Block parts as +well as the Validation. In the future this may change, perhaps by moving +the Validation data outside the Block. */ type BlockStore struct { height uint @@ -22,7 +29,7 @@ type BlockStore struct { } func NewBlockStore(db db_.DB) *BlockStore { - bsjson := LoadBlockStoreJSON(db) + bsjson := LoadBlockStoreStateJSON(db) return &BlockStore{ height: bsjson.Height, db: db, @@ -99,23 +106,24 @@ func (bs *BlockStore) SaveBlock(block *Block, blockParts *PartSet) { if !blockParts.IsComplete() { Panicf("BlockStore can only save complete block part sets") } - meta := &BlockMeta{ - Hash: block.Hash(), - Parts: blockParts.Header(), - Header: block.Header, - } + // Save block meta + meta := makeBlockMeta(block, blockParts) metaBytes := BinaryBytes(meta) bs.db.Set(calcBlockMetaKey(height), metaBytes) + // Save block parts for i := uint(0); i < blockParts.Total(); i++ { bs.saveBlockPart(height, i, blockParts.GetPart(i)) } + // Save block validation (duplicate and separate) validationBytes := BinaryBytes(block.Validation) bs.db.Set(calcBlockValidationKey(height), validationBytes) - // Save new BlockStoreJSON descriptor - BlockStoreJSON{Height: height}.Save(bs.db) + + // Save new BlockStoreStateJSON descriptor + BlockStoreStateJSON{Height: height}.Save(bs.db) + // Done! bs.height = height } @@ -131,9 +139,17 @@ func (bs *BlockStore) saveBlockPart(height uint, index uint, part *Part) { //----------------------------------------------------------------------------- type BlockMeta struct { - Hash []byte // The BlockHash - Parts PartSetHeader // The PartSetHeader, for transfer + Hash []byte // The block hash Header *Header // The block's Header + Parts PartSetHeader // The PartSetHeader, for transfer +} + +func makeBlockMeta(block *Block, blockParts *PartSet) *BlockMeta { + return &BlockMeta{ + Hash: block.Hash(), + Header: block.Header, + Parts: blockParts.Header(), + } } //----------------------------------------------------------------------------- @@ -154,11 +170,11 @@ func calcBlockValidationKey(height uint) []byte { var blockStoreKey = []byte("blockStore") -type BlockStoreJSON struct { +type BlockStoreStateJSON struct { Height uint } -func (bsj BlockStoreJSON) Save(db db_.DB) { +func (bsj BlockStoreStateJSON) Save(db db_.DB) { bytes, err := json.Marshal(bsj) if err != nil { Panicf("Could not marshal state bytes: %v", err) @@ -166,14 +182,14 @@ func (bsj BlockStoreJSON) Save(db db_.DB) { db.Set(blockStoreKey, bytes) } -func LoadBlockStoreJSON(db db_.DB) BlockStoreJSON { +func LoadBlockStoreStateJSON(db db_.DB) BlockStoreStateJSON { bytes := db.Get(blockStoreKey) if bytes == nil { - return BlockStoreJSON{ + return BlockStoreStateJSON{ Height: 0, } } - bsj := BlockStoreJSON{} + bsj := BlockStoreStateJSON{} err := json.Unmarshal(bytes, &bsj) if err != nil { Panicf("Could not unmarshal bytes: %X", bytes) diff --git a/block/tx.go b/block/tx.go index 1fad2aab3..b483eb9a5 100644 --- a/block/tx.go +++ b/block/tx.go @@ -10,21 +10,31 @@ import ( . "github.com/tendermint/tendermint/common" ) +var ( + ErrTxInvalidAddress = errors.New("Error invalid address") + ErrTxDuplicateAddress = errors.New("Error duplicate address") + ErrTxInvalidAmount = errors.New("Error invalid amount") + ErrTxInsufficientFunds = errors.New("Error insufficient funds") + ErrTxInvalidSignature = errors.New("Error invalid signature") + ErrTxInvalidSequence = errors.New("Error invalid sequence") +) + /* Tx (Transaction) is an atomic operation on the ledger state. Account Txs: -1. SendTx Send coins to address + - SendTx Send coins to address Validation Txs: -1. BondTx New validator posts a bond -2. UnbondTx Validator leaves -3. DupeoutTx Validator dupes out (equivocates) + - BondTx New validator posts a bond + - UnbondTx Validator leaves + - DupeoutTx Validator dupes out (equivocates) */ type Tx interface { WriteSignBytes(w io.Writer, n *int64, err *error) } +// Types of Tx implementations const ( // Account transactions TxTypeSend = byte(0x01) @@ -36,14 +46,8 @@ const ( TxTypeDupeout = byte(0x14) ) -var ( - ErrTxInvalidAddress = errors.New("Error invalid address") - ErrTxDuplicateAddress = errors.New("Error duplicate address") - ErrTxInvalidAmount = errors.New("Error invalid amount") - ErrTxInsufficientFunds = errors.New("Error insufficient funds") - ErrTxInvalidSignature = errors.New("Error invalid signature") - ErrTxInvalidSequence = errors.New("Error invalid sequence") -) +//------------------------------------- +// for binary.readReflect func TxDecoder(r io.Reader, n *int64, err *error) interface{} { switch t := ReadByte(r, n, err); t { diff --git a/block/vote.go b/block/vote.go index 2d4e460c8..0c0dafc72 100644 --- a/block/vote.go +++ b/block/vote.go @@ -10,12 +10,6 @@ import ( . "github.com/tendermint/tendermint/common" ) -const ( - VoteTypePrevote = byte(0x00) - VoteTypePrecommit = byte(0x01) - VoteTypeCommit = byte(0x02) -) - var ( ErrVoteUnexpectedStep = errors.New("Unexpected step") ErrVoteInvalidAccount = errors.New("Invalid round vote account") @@ -24,7 +18,7 @@ var ( ErrVoteConflictingSignature = errors.New("Conflicting round vote signature") ) -// Represents a prevote, precommit, or commit vote for proposals from validators. +// Represents a prevote, precommit, or commit vote from validators for consensus. // Commit votes get aggregated into the next block's Validaiton. // See the whitepaper for details. type Vote struct { @@ -36,6 +30,13 @@ type Vote struct { Signature SignatureEd25519 } +// Types of votes +const ( + VoteTypePrevote = byte(0x00) + VoteTypePrecommit = byte(0x01) + VoteTypeCommit = byte(0x02) +) + func (vote *Vote) WriteSignBytes(w io.Writer, n *int64, err *error) { WriteUvarint(vote.Height, w, n, err) WriteUvarint(vote.Round, w, n, err) diff --git a/common/bit_array.go b/common/bit_array.go index 07b72ea98..82fe8f95f 100644 --- a/common/bit_array.go +++ b/common/bit_array.go @@ -139,10 +139,10 @@ func (bA BitArray) PickRandom() (uint, bool) { } func (bA BitArray) String() string { - return bA.StringWithIndent("") + return bA.StringIndented("") } -func (bA BitArray) StringWithIndent(indent string) string { +func (bA BitArray) StringIndented(indent string) string { lines := []string{} bits := "" for i := uint(0); i < bA.Bits; i++ { diff --git a/consensus/pol.go b/consensus/pol.go index d83baef76..20cb07897 100644 --- a/consensus/pol.go +++ b/consensus/pol.go @@ -86,7 +86,7 @@ func (pol *POL) Verify(valSet *state.ValidatorSet) error { } -func (pol *POL) Description() string { +func (pol *POL) StringShort() string { if pol == nil { return "nil-POL" } else { diff --git a/consensus/state.go b/consensus/state.go index ca9edb3ec..100fe0d87 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -131,10 +131,10 @@ type RoundState struct { } func (rs *RoundState) String() string { - return rs.StringWithIndent("") + return rs.StringIndented("") } -func (rs *RoundState) StringWithIndent(indent string) string { +func (rs *RoundState) StringIndented(indent string) string { return fmt.Sprintf(`RoundState{ %s H:%v R:%v S:%v %s StartTime: %v @@ -153,20 +153,20 @@ func (rs *RoundState) StringWithIndent(indent string) string { indent, rs.Height, rs.Round, rs.Step, indent, rs.StartTime, indent, rs.CommitTime, - indent, rs.Validators.StringWithIndent(indent+" "), + indent, rs.Validators.StringIndented(indent+" "), indent, rs.Proposal, - indent, rs.ProposalBlockParts.Description(), rs.ProposalBlock.Description(), - indent, rs.ProposalPOLParts.Description(), rs.ProposalPOL.Description(), - indent, rs.LockedBlockParts.Description(), rs.LockedBlock.Description(), - indent, rs.LockedPOL.Description(), - indent, rs.Prevotes.StringWithIndent(indent+" "), - indent, rs.Precommits.StringWithIndent(indent+" "), - indent, rs.Commits.StringWithIndent(indent+" "), - indent, rs.LastCommits.Description(), + indent, rs.ProposalBlockParts.StringShort(), rs.ProposalBlock.StringShort(), + indent, rs.ProposalPOLParts.StringShort(), rs.ProposalPOL.StringShort(), + indent, rs.LockedBlockParts.StringShort(), rs.LockedBlock.StringShort(), + indent, rs.LockedPOL.StringShort(), + indent, rs.Prevotes.StringIndented(indent+" "), + indent, rs.Precommits.StringIndented(indent+" "), + indent, rs.Commits.StringIndented(indent+" "), + indent, rs.LastCommits.StringShort(), indent) } -func (rs *RoundState) Description() string { +func (rs *RoundState) StringShort() string { return fmt.Sprintf(`RS{%v/%v/%X %v}`, rs.Height, rs.Round, rs.Step, rs.StartTime) } @@ -306,7 +306,7 @@ ACTION_LOOP: height, round, action := roundAction.Height, roundAction.Round, roundAction.Action rs := cs.GetRoundState() - log.Info("Running round action A:%X %v", action, rs.Description()) + log.Info("Running round action A:%X %v", action, rs.StringShort()) // Continue if action is not relevant if height != rs.Height { @@ -545,6 +545,7 @@ func (cs *ConsensusState) RunActionPropose(height uint, round uint) { Height: cs.Height, Time: time.Now(), Fees: 0, // TODO fees + NumTxs: uint(len(txs)), LastBlockHash: cs.state.LastBlockHash, LastBlockParts: cs.state.LastBlockParts, StateHash: nil, // Will set afterwards. diff --git a/consensus/vote_set.go b/consensus/vote_set.go index 666c97698..dbe865a55 100644 --- a/consensus/vote_set.go +++ b/consensus/vote_set.go @@ -247,14 +247,14 @@ func (voteSet *VoteSet) MakeValidation() *Validation { } func (voteSet *VoteSet) String() string { - return voteSet.StringWithIndent("") + return voteSet.StringIndented("") } -func (voteSet *VoteSet) StringWithIndent(indent string) string { +func (voteSet *VoteSet) StringIndented(indent string) string { voteStrings := make([]string, len(voteSet.votes)) for i, vote := range voteSet.votes { if vote == nil { - voteStrings[i] = "nil" + voteStrings[i] = "nil-Vote" } else { voteStrings[i] = vote.String() } @@ -270,7 +270,7 @@ func (voteSet *VoteSet) StringWithIndent(indent string) string { indent) } -func (voteSet *VoteSet) Description() string { +func (voteSet *VoteSet) StringShort() string { if voteSet == nil { return "nil-VoteSet" } diff --git a/state/state_test.go b/state/state_test.go index a7b8e0f71..c2c7ee2f5 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -59,6 +59,7 @@ func TestGenesisSaveLoad(t *testing.T) { Height: 1, Time: s0.LastBlockTime.Add(time.Minute), Fees: 0, + NumTxs: 0, LastBlockHash: s0.LastBlockHash, LastBlockParts: s0.LastBlockParts, StateHash: nil, diff --git a/state/validator_set.go b/state/validator_set.go index c5087b395..ad8b2dedb 100644 --- a/state/validator_set.go +++ b/state/validator_set.go @@ -197,10 +197,10 @@ func (valSet *ValidatorSet) Iterate(fn func(index uint, val *Validator) bool) { } func (valSet *ValidatorSet) String() string { - return valSet.StringWithIndent("") + return valSet.StringIndented("") } -func (valSet *ValidatorSet) StringWithIndent(indent string) string { +func (valSet *ValidatorSet) StringIndented(indent string) string { valStrings := []string{} valSet.Iterate(func(index uint, val *Validator) bool { valStrings = append(valStrings, val.String())