move handshake to state, use Handshaker, more tests

consensus/race.test

@@ -1 +0,0 @@
-ok  	github.com/tendermint/tendermint/consensus	5.928s

consensus/state.go

@@ -932,25 +932,8 @@ func (cs *ConsensusState) createProposalBlock() (block *types.Block, blockParts
 	// Mempool validated transactions
 	txs := cs.mempool.Reap(cs.config.GetInt("block_size"))
 
-	block = &types.Block{
-		Header: &types.Header{
-			ChainID:        cs.state.ChainID,
-			Height:         cs.Height,
-			Time:           time.Now(),
-			NumTxs:         len(txs),
-			LastBlockID:    cs.state.LastBlockID,
-			ValidatorsHash: cs.state.Validators.Hash(),
-			AppHash:        cs.state.AppHash, // state merkle root of txs from the previous block.
-		},
-		LastCommit: commit,
-		Data: &types.Data{
-			Txs: txs,
-		},
-	}
-	block.FillHeader()
-	blockParts = block.MakePartSet(cs.config.GetInt("block_part_size"))
-
-	return block, blockParts
+	return types.MakeBlock(cs.Height, cs.state.ChainID, txs, commit,
+		cs.state.LastBlockID, cs.state.Validators.Hash(), cs.state.AppHash)
 }
 
 // Enter: `timeoutPropose` after entering Propose.
@@ -1273,8 +1256,6 @@ func (cs *ConsensusState) finalizeCommit(height int) {
 	// Create a copy of the state for staging
 	// and an event cache for txs
 	stateCopy := cs.state.Copy()
-
-	// event cache for txs
 	eventCache := types.NewEventCache(cs.evsw)
 
 	// Execute and commit the block, and update the mempool.

node/node.go

@@ -63,7 +63,7 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato
 	state := sm.GetState(config, stateDB)
 
 	// Create the proxyApp, which manages connections (consensus, mempool, query)
-	proxyApp := proxy.NewAppConns(config, clientCreator, state, blockStore)
+	proxyApp := proxy.NewAppConns(config, clientCreator, sm.NewHandshaker(state, blockStore))
 	if _, err := proxyApp.Start(); err != nil {
 		Exit(Fmt("Error starting proxy app connections: %v", err))
 	}
@@ -380,7 +380,7 @@ func newConsensusState(config cfg.Config) *consensus.ConsensusState {
 	state := sm.MakeGenesisStateFromFile(stateDB, config.GetString("genesis_file"))
 
 	// Create proxyAppConn connection (consensus, mempool, query)
-	proxyApp := proxy.NewAppConns(config, proxy.DefaultClientCreator(config), state, blockStore)
+	proxyApp := proxy.NewAppConns(config, proxy.DefaultClientCreator(config), sm.NewHandshaker(state, blockStore))
 	_, err := proxyApp.Start()
 	if err != nil {
 		Exit(Fmt("Error starting proxy app conns: %v", err))

proxy/multi_app_conn.go

@@ -1,12 +1,8 @@
 package proxy
 
 import (
-	"bytes"
-	"fmt"
-
 	. "github.com/tendermint/go-common"
 	cfg "github.com/tendermint/go-config"
-	"github.com/tendermint/tendermint/types"
 )
 
 //-----------------------------
@@ -20,13 +16,17 @@ type AppConns interface {
 	Query() AppConnQuery
 }
 
-func NewAppConns(config cfg.Config, clientCreator ClientCreator, state State, blockStore BlockStore) AppConns {
-	return NewMultiAppConn(config, clientCreator, state, blockStore)
+func NewAppConns(config cfg.Config, clientCreator ClientCreator, handshaker Handshaker) AppConns {
+	return NewMultiAppConn(config, clientCreator, handshaker)
 }
 
 //-----------------------------
 // multiAppConn implements AppConns
 
+type Handshaker interface {
+	Handshake(AppConns) error
+}
+
 // a multiAppConn is made of a few appConns (mempool, consensus, query)
 // and manages their underlying tmsp clients, including the handshake
 // which ensures the app and tendermint are synced.
@@ -36,8 +36,7 @@ type multiAppConn struct {
 
 	config cfg.Config
 
-	state      State
-	blockStore BlockStore
+	handshaker Handshaker
 
 	mempoolConn   *appConnMempool
 	consensusConn *appConnConsensus
@@ -47,11 +46,10 @@ type multiAppConn struct {
 }
 
 // Make all necessary tmsp connections to the application
-func NewMultiAppConn(config cfg.Config, clientCreator ClientCreator, state State, blockStore BlockStore) *multiAppConn {
+func NewMultiAppConn(config cfg.Config, clientCreator ClientCreator, handshaker Handshaker) *multiAppConn {
 	multiAppConn := &multiAppConn{
 		config:        config,
-		state:         state,
-		blockStore:    blockStore,
+		handshaker:    handshaker,
 		clientCreator: clientCreator,
 	}
 	multiAppConn.BaseService = *NewBaseService(log, "multiAppConn", multiAppConn)
@@ -98,74 +96,8 @@ func (app *multiAppConn) OnStart() error {
 	app.consensusConn = NewAppConnConsensus(concli)
 
 	// ensure app is synced to the latest state
-	return app.Handshake()
-}
-
-// TODO: retry the handshake once if it fails the first time
-// ... let Info take an argument determining its behaviour
-func (app *multiAppConn) Handshake() error {
-	// handshake is done via info request on the query conn
-	res, tmspInfo, blockInfo, configInfo := app.queryConn.InfoSync()
-	if res.IsErr() {
-		return fmt.Errorf("Error calling Info. Code: %v; Data: %X; Log: %s", res.Code, res.Data, res.Log)
-	}
-
-	if blockInfo == nil {
-		log.Warn("blockInfo is nil, aborting handshake")
-		return nil
-	}
-
-	log.Notice("TMSP Handshake", "height", blockInfo.BlockHeight, "block_hash", blockInfo.BlockHash, "app_hash", blockInfo.AppHash)
-
-	// TODO: check overflow or change pb to int32
-	blockHeight := int(blockInfo.BlockHeight)
-	blockHash := blockInfo.BlockHash
-	appHash := blockInfo.AppHash
-
-	if tmspInfo != nil {
-		// TODO: check tmsp version (or do this in the tmspcli?)
-		_ = tmspInfo
-	}
-
-	// last block (nil if we starting from 0)
-	var header *types.Header
-	var partsHeader types.PartSetHeader
-
-	// replay all blocks after blockHeight
-	// if blockHeight == 0, we will replay everything
-	if blockHeight != 0 {
-		blockMeta := app.blockStore.LoadBlockMeta(blockHeight)
-		if blockMeta == nil {
-			return fmt.Errorf("Handshake error. Could not find block #%d", blockHeight)
-		}
-
-		// check block hash
-		if !bytes.Equal(blockMeta.Hash, blockHash) {
-			return fmt.Errorf("Handshake error. Block hash at height %d does not match. Got %X, expected %X", blockHeight, blockHash, blockMeta.Hash)
-		}
-
-		// NOTE: app hash should be in the next block ...
-		// check app hash
-		/*if !bytes.Equal(blockMeta.Header.AppHash, appHash) {
-			return fmt.Errorf("Handshake error. App hash at height %d does not match. Got %X, expected %X", blockHeight, appHash, blockMeta.Header.AppHash)
-		}*/
-
-		header = blockMeta.Header
-		partsHeader = blockMeta.PartsHeader
-	}
-
-	if configInfo != nil {
-		// TODO: set config info
-		_ = configInfo
+	if app.handshaker != nil {
+		return app.handshaker.Handshake(app)
 	}
-
-	// replay blocks up to the latest in the blockstore
-	err := app.state.ReplayBlocks(appHash, header, partsHeader, app.consensusConn, app.blockStore)
-	if err != nil {
-		return fmt.Errorf("Error on replay: %v", err)
-	}
-
-	// TODO: (on restart) replay mempool
-
 	return nil
 }

proxy/multi_app_conn_test.go

@@ -1,22 +0,0 @@
-package proxy
-
-import (
-	"testing"
-	"time"
-
-	"github.com/tendermint/go-p2p"
-	"github.com/tendermint/tendermint/config/tendermint_test"
-	"github.com/tendermint/tendermint/node"
-	"github.com/tendermint/tendermint/types"
-)
-
-func TestPersistence(t *testing.T) {
-
-	// create persistent dummy app
-	// set state on dummy app
-	// proxy handshake
-
-	config := tendermint_test.ResetConfig("proxy_test_")
-	multiApp := NewMultiAppConn(config, state, blockStore)
-
-}

proxy/state.go

@@ -1,15 +0,0 @@
-package proxy
-
-import (
-	"github.com/tendermint/tendermint/types"
-)
-
-type State interface {
-	ReplayBlocks([]byte, *types.Header, types.PartSetHeader, AppConnConsensus, BlockStore) error
-}
-
-type BlockStore interface {
-	Height() int
-	LoadBlockMeta(height int) *types.BlockMeta
-	LoadBlock(height int) *types.Block
-}

state/errors.go

@@ -0,0 +1,55 @@
+package state
+
+import (
+	. "github.com/tendermint/go-common"
+)
+
+type (
+	ErrInvalidBlock error
+	ErrProxyAppConn error
+
+	ErrUnknownBlock struct {
+		height int
+	}
+
+	ErrBlockHashMismatch struct {
+		coreHash []byte
+		appHash  []byte
+		height   int
+	}
+
+	ErrAppBlockHeightTooHigh struct {
+		coreHeight int
+		appHeight  int
+	}
+
+	ErrLastStateMismatch struct {
+		height int
+		core   []byte
+		app    []byte
+	}
+
+	ErrStateMismatch struct {
+		got      *State
+		expected *State
+	}
+)
+
+func (e ErrUnknownBlock) Error() string {
+	return Fmt("Could not find block #%d", e.height)
+}
+
+func (e ErrBlockHashMismatch) Error() string {
+	return Fmt("App block hash (%X) does not match core block hash (%X) for height %d", e.appHash, e.coreHash, e.height)
+}
+
+func (e ErrAppBlockHeightTooHigh) Error() string {
+	return Fmt("App block height (%d) is higher than core (%d)", e.appHeight, e.coreHeight)
+}
+func (e ErrLastStateMismatch) Error() string {
+	return Fmt("Latest tendermint block (%d) LastAppHash (%X) does not match app's AppHash (%X)", e.height, e.core, e.app)
+}
+
+func (e ErrStateMismatch) Error() string {
+	return Fmt("State after replay does not match saved state. Got ----\n%v\nExpected ----\n%v\n", e.got, e.expected)
+}

state/execution.go

@@ -15,11 +15,6 @@ import (
 //--------------------------------------------------
 // Execute the block
 
-type (
-	ErrInvalidBlock error
-	ErrProxyAppConn error
-)
-
 // Execute the block to mutate State.
 // Validates block and then executes Data.Txs in the block.
 func (s *State) ExecBlock(eventCache types.Fireable, proxyAppConn proxy.AppConnConsensus, block *types.Block, blockPartsHeader types.PartSetHeader) error {
@@ -257,42 +252,96 @@ func (m mockMempool) Unlock()                           {}
 func (m mockMempool) Update(height int, txs []types.Tx) {}
 
 //----------------------------------------------------------------
-// Replay blocks to sync app to latest state of core
+// Handshake with app to sync to latest state of core by replaying blocks
 
-type ErrReplay error
-
-type ErrAppBlockHeightTooHigh struct {
-	coreHeight int
-	appHeight  int
+// TODO: Should we move blockchain/store.go to its own package?
+type BlockStore interface {
+	Height() int
+	LoadBlock(height int) *types.Block
 }
 
-func (e ErrAppBlockHeightTooHigh) Error() string {
-	return Fmt("App block height (%d) is higher than core (%d)", e.appHeight, e.coreHeight)
-}
+type Handshaker struct {
+	state *State
+	store BlockStore
 
-type ErrLastStateMismatch struct {
-	height int
-	core   []byte
-	app    []byte
+	nBlocks int // number of blocks applied to the state
 }
 
-func (e ErrLastStateMismatch) Error() string {
-	return Fmt("Latest tendermint block (%d) LastAppHash (%X) does not match app's AppHash (%X)", e.height, e.core, e.app)
+func NewHandshaker(state *State, store BlockStore) *Handshaker {
+	return &Handshaker{state, store, 0}
 }
 
-type ErrStateMismatch struct {
-	got      *State
-	expected *State
-}
+// TODO: retry the handshake once if it fails the first time
+// ... let Info take an argument determining its behaviour
+func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error {
+	// handshake is done via info request on the query conn
+	res, tmspInfo, blockInfo, configInfo := proxyApp.Query().InfoSync()
+	if res.IsErr() {
+		return errors.New(Fmt("Error calling Info. Code: %v; Data: %X; Log: %s", res.Code, res.Data, res.Log))
+	}
+
+	if blockInfo == nil {
+		log.Warn("blockInfo is nil, aborting handshake")
+		return nil
+	}
+
+	log.Notice("TMSP Handshake", "height", blockInfo.BlockHeight, "block_hash", blockInfo.BlockHash, "app_hash", blockInfo.AppHash)
+
+	blockHeight := int(blockInfo.BlockHeight) // safe, should be an int32
+	blockHash := blockInfo.BlockHash
+	appHash := blockInfo.AppHash
+
+	if tmspInfo != nil {
+		// TODO: check tmsp version (or do this in the tmspcli?)
+		_ = tmspInfo
+	}
+
+	// last block (nil if we starting from 0)
+	var header *types.Header
+	var partsHeader types.PartSetHeader
+
+	// replay all blocks after blockHeight
+	// if blockHeight == 0, we will replay everything
+	if blockHeight != 0 {
+		block := h.store.LoadBlock(blockHeight)
+		if block == nil {
+			return ErrUnknownBlock{blockHeight}
+		}
+
+		// check block hash
+		if !bytes.Equal(block.Hash(), blockHash) {
+			return ErrBlockHashMismatch{block.Hash(), blockHash, blockHeight}
+		}
+
+		// NOTE: app hash should be in the next block ...
+		// check app hash
+		/*if !bytes.Equal(block.Header.AppHash, appHash) {
+			return fmt.Errorf("Handshake error. App hash at height %d does not match. Got %X, expected %X", blockHeight, appHash, block.Header.AppHash)
+		}*/
+
+		header = block.Header
+		partsHeader = block.MakePartSet().Header()
+	}
+
+	if configInfo != nil {
+		// TODO: set config info
+		_ = configInfo
+	}
 
-func (e ErrStateMismatch) Error() string {
-	return Fmt("State after replay does not match saved state. Got ----\n%v\nExpected ----\n%v\n", e.got, e.expected)
+	// replay blocks up to the latest in the blockstore
+	err := h.ReplayBlocks(appHash, header, partsHeader, proxyApp.Consensus())
+	if err != nil {
+		return errors.New(Fmt("Error on replay: %v", err))
+	}
+
+	// TODO: (on restart) replay mempool
+
+	return nil
 }
 
 // Replay all blocks after blockHeight and ensure the result matches the current state.
-// XXX: blockStore must guarantee to have blocks for height <= blockStore.Height()
-func (s *State) ReplayBlocks(appHash []byte, header *types.Header, partsHeader types.PartSetHeader,
-	appConnConsensus proxy.AppConnConsensus, blockStore proxy.BlockStore) error {
+func (h *Handshaker) ReplayBlocks(appHash []byte, header *types.Header, partsHeader types.PartSetHeader,
+	appConnConsensus proxy.AppConnConsensus) error {
 
 	// NOTE/TODO: tendermint may crash after the app commits
 	// but before it can save the new state root.
@@ -300,17 +349,25 @@ func (s *State) ReplayBlocks(appHash []byte, header *types.Header, partsHeader t
 	// then, if tm state is behind app state, the only thing missing can be app hash
 
 	// get a fresh state and reset to the apps latest
-	stateCopy := s.Copy()
-	if header != nil {
-		// TODO: put validators in iavl tree so we can set the state with an older validator set
-		lastVals, nextVals := stateCopy.GetValidators()
+	stateCopy := h.state.Copy()
+
+	// TODO: put validators in iavl tree so we can set the state with an older validator set
+	lastVals, nextVals := stateCopy.GetValidators()
+	if header == nil {
+		stateCopy.LastBlockHeight = 0
+		stateCopy.LastBlockHash = nil
+		stateCopy.LastBlockParts = types.PartSetHeader{}
+		// stateCopy.LastBlockTime = ... doesnt matter
+		stateCopy.Validators = nextVals
+		stateCopy.LastValidators = lastVals
+	} else {
 		stateCopy.SetBlockAndValidators(header, partsHeader, lastVals, nextVals)
-		stateCopy.Stale = false
-		stateCopy.AppHash = appHash
 	}
+	stateCopy.Stale = false
+	stateCopy.AppHash = appHash
 
 	appBlockHeight := stateCopy.LastBlockHeight
-	coreBlockHeight := blockStore.Height()
+	coreBlockHeight := h.store.Height()
 	if coreBlockHeight < appBlockHeight {
 		// if the app is ahead, there's nothing we can do
 		return ErrAppBlockHeightTooHigh{coreBlockHeight, appBlockHeight}
@@ -319,13 +376,13 @@ func (s *State) ReplayBlocks(appHash []byte, header *types.Header, partsHeader t
 		// if we crashed between Commit and SaveState,
 		// the state's app hash is stale.
 		// otherwise we're synced
-		if s.Stale {
-			s.Stale = false
-			s.AppHash = appHash
+		if h.state.Stale {
+			h.state.Stale = false
+			h.state.AppHash = appHash
 		}
-		return checkState(s, stateCopy)
+		return checkState(h.state, stateCopy)
 
-	} else if s.LastBlockHeight == appBlockHeight {
+	} else if h.state.LastBlockHeight == appBlockHeight {
 		// core is ahead of app but core's state height is at apps height
 		// this happens if we crashed after saving the block,
 		// but before committing it. We should be 1 ahead
@@ -334,21 +391,21 @@ func (s *State) ReplayBlocks(appHash []byte, header *types.Header, partsHeader t
 		}
 
 		// check that the blocks last apphash is the states apphash
-		blockMeta := blockStore.LoadBlockMeta(coreBlockHeight)
-		if !bytes.Equal(blockMeta.Header.AppHash, appHash) {
-			return ErrLastStateMismatch{coreBlockHeight, blockMeta.Header.AppHash, appHash}
+		block := h.store.LoadBlock(coreBlockHeight)
+		if !bytes.Equal(block.Header.AppHash, appHash) {
+			return ErrLastStateMismatch{coreBlockHeight, block.Header.AppHash, appHash}
 		}
 
 		// replay the block against the actual tendermint state (not the copy)
-		return loadApplyBlock(coreBlockHeight, s, blockStore, appConnConsensus)
+		return h.loadApplyBlock(coreBlockHeight, h.state, appConnConsensus)
 
 	} else {
 		// either we're caught up or there's blocks to replay
 		// replay all blocks starting with appBlockHeight+1
 		for i := appBlockHeight + 1; i <= coreBlockHeight; i++ {
-			loadApplyBlock(i, stateCopy, blockStore, appConnConsensus)
+			h.loadApplyBlock(i, stateCopy, appConnConsensus)
 		}
-		return checkState(s, stateCopy)
+		return checkState(h.state, stateCopy)
 	}
 }
 
@@ -360,23 +417,21 @@ func checkState(s, stateCopy *State) error {
 	return nil
 }
 
-func loadApplyBlock(blockIndex int, s *State, blockStore proxy.BlockStore, appConnConsensus proxy.AppConnConsensus) error {
-	blockMeta := blockStore.LoadBlockMeta(blockIndex)
-	block := blockStore.LoadBlock(blockIndex)
-	panicOnNilBlock(blockIndex, blockStore.Height(), block, blockMeta) // XXX
-
-	var eventCache types.Fireable // nil
-	return s.ApplyBlock(eventCache, appConnConsensus, block, blockMeta.PartsHeader, mockMempool{})
+func (h *Handshaker) loadApplyBlock(blockIndex int, state *State, appConnConsensus proxy.AppConnConsensus) error {
+	h.nBlocks += 1
+	block := h.store.LoadBlock(blockIndex)
+	panicOnNilBlock(blockIndex, h.store.Height(), block) // XXX
+	var eventCache types.Fireable                        // nil
+	return state.ApplyBlock(eventCache, appConnConsensus, block, block.MakePartSet().Header(), mockMempool{})
 }
 
-func panicOnNilBlock(height, bsHeight int, block *types.Block, blockMeta *types.BlockMeta) {
-	if block == nil || blockMeta == nil {
+func panicOnNilBlock(height, bsHeight int, block *types.Block) {
+	if block == nil {
 		// Sanity?
 		PanicCrisis(Fmt(`
-block/blockMeta is nil for height <= blockStore.Height() (%d <= %d).
+block is nil for height <= blockStore.Height() (%d <= %d).
 Block: %v,
-BlockMeta: %v
-`, height, bsHeight, block, blockMeta))
+`, height, bsHeight, block))
 
 	}
 }

state/execution_test.go

@@ -0,0 +1,186 @@
+package state
+
+import (
+	"bytes"
+	//"fmt"
+	"path"
+	"testing"
+
+	"github.com/tendermint/tendermint/config/tendermint_test"
+	//	. "github.com/tendermint/go-common"
+	"github.com/tendermint/go-crypto"
+	dbm "github.com/tendermint/go-db"
+	"github.com/tendermint/tendermint/proxy"
+	"github.com/tendermint/tendermint/types"
+	"github.com/tendermint/tmsp/example/dummy"
+)
+
+var (
+	privKey = crypto.GenPrivKeyEd25519FromSecret([]byte("handshake_test"))
+	chainID = "handshake_chain"
+	nBlocks = 5
+	mempool = mockMempool{}
+)
+
+func TestExecBlock(t *testing.T) {
+	// TODO
+}
+
+// Sync from scratch
+func TestHandshakeReplayAll(t *testing.T) {
+	testHandshakeReplay(t, 0)
+}
+
+// Sync many, not from scratch
+func TestHandshakeReplaySome(t *testing.T) {
+	testHandshakeReplay(t, 1)
+}
+
+// Sync from lagging by one
+func TestHandshakeReplayOne(t *testing.T) {
+	testHandshakeReplay(t, nBlocks-1)
+}
+
+// Sync from caught up
+func TestHandshakeReplayNone(t *testing.T) {
+	testHandshakeReplay(t, nBlocks)
+}
+
+// Make some blocks. Start a fresh app and apply n blocks. Then restart the app and sync it up with the remaining blocks
+func testHandshakeReplay(t *testing.T, n int) {
+	config := tendermint_test.ResetConfig("proxy_test_")
+
+	state, store := stateAndStore()
+	clientCreator := proxy.NewLocalClientCreator(dummy.NewPersistentDummyApplication(path.Join(config.GetString("db_dir"), "1")))
+	clientCreator2 := proxy.NewLocalClientCreator(dummy.NewPersistentDummyApplication(path.Join(config.GetString("db_dir"), "2")))
+	proxyApp := proxy.NewAppConns(config, clientCreator, NewHandshaker(state, store))
+	if _, err := proxyApp.Start(); err != nil {
+		t.Fatalf("Error starting proxy app connections: %v", err)
+	}
+	chain := makeBlockchain(t, proxyApp, state)
+	store.chain = chain //
+	latestAppHash := state.AppHash
+	proxyApp.Stop()
+
+	if n > 0 {
+		// start a new app without handshake, play n blocks
+		proxyApp = proxy.NewAppConns(config, clientCreator2, nil)
+		if _, err := proxyApp.Start(); err != nil {
+			t.Fatalf("Error starting proxy app connections: %v", err)
+		}
+		state2, _ := stateAndStore()
+		for i := 0; i < n; i++ {
+			block := chain[i]
+			err := state2.ApplyBlock(nil, proxyApp.Consensus(), block, block.MakePartSet().Header(), mempool)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+		proxyApp.Stop()
+	}
+
+	// now start it with the handshake
+	handshaker := NewHandshaker(state, store)
+	proxyApp = proxy.NewAppConns(config, clientCreator2, handshaker)
+	if _, err := proxyApp.Start(); err != nil {
+		t.Fatalf("Error starting proxy app connections: %v", err)
+	}
+
+	// get the latest app hash from the app
+	r, _, blockInfo, _ := proxyApp.Query().InfoSync()
+	if r.IsErr() {
+		t.Fatal(r)
+	}
+
+	// the app hash should be synced up
+	if !bytes.Equal(latestAppHash, blockInfo.AppHash) {
+		t.Fatalf("Expected app hashes to match after handshake/replay. got %X, expected %X", blockInfo.AppHash, latestAppHash)
+	}
+
+	if handshaker.nBlocks != nBlocks-n {
+		t.Fatalf("Expected handshake to sync %d blocks, got %d", nBlocks-n, handshaker.nBlocks)
+	}
+
+}
+
+//--------------------------
+
+// make some bogus txs
+func txsFunc(blockNum int) (txs []types.Tx) {
+	for i := 0; i < 10; i++ {
+		txs = append(txs, types.Tx([]byte{byte(blockNum), byte(i)}))
+	}
+	return txs
+}
+
+// sign a commit vote
+func signCommit(height, round int, hash []byte, header types.PartSetHeader) *types.Vote {
+	vote := &types.Vote{
+		Height:           height,
+		Round:            round,
+		Type:             types.VoteTypePrecommit,
+		BlockHash:        hash,
+		BlockPartsHeader: header,
+	}
+
+	sig := privKey.Sign(types.SignBytes(chainID, vote))
+	vote.Signature = sig.(crypto.SignatureEd25519)
+	return vote
+}
+
+// make a blockchain with one validator
+func makeBlockchain(t *testing.T, proxyApp proxy.AppConns, state *State) (blockchain []*types.Block) {
+
+	prevHash := state.LastBlockHash
+	lastCommit := new(types.Commit)
+	prevParts := types.PartSetHeader{}
+	valHash := state.Validators.Hash()
+
+	for i := 1; i < nBlocks+1; i++ {
+		block, parts := types.MakeBlock(i, chainID, txsFunc(i), lastCommit,
+			prevParts, prevHash, valHash, state.AppHash)
+		err := state.ApplyBlock(nil, proxyApp.Consensus(), block, block.MakePartSet().Header(), mempool)
+		if err != nil {
+			t.Fatal(i, err)
+		}
+
+		voteSet := types.NewVoteSet(chainID, i, 0, types.VoteTypePrecommit, state.Validators)
+		vote := signCommit(i, 0, block.Hash(), parts.Header())
+		_, _, err = voteSet.AddByIndex(0, vote)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		blockchain = append(blockchain, block)
+		prevHash = block.Hash()
+		prevParts = parts.Header()
+		lastCommit = voteSet.MakeCommit()
+	}
+	return blockchain
+}
+
+// fresh state and mock store
+func stateAndStore() (*State, *mockBlockStore) {
+	stateDB := dbm.NewMemDB()
+	return MakeGenesisState(stateDB, &types.GenesisDoc{
+		ChainID: chainID,
+		Validators: []types.GenesisValidator{
+			types.GenesisValidator{privKey.PubKey(), 10000, "test"},
+		},
+		AppHash: nil,
+	}), NewMockBlockStore(nil)
+}
+
+//----------------------------------
+// mock block store
+
+type mockBlockStore struct {
+	chain []*types.Block
+}
+
+func NewMockBlockStore(chain []*types.Block) *mockBlockStore {
+	return &mockBlockStore{chain}
+}
+
+func (bs *mockBlockStore) Height() int                       { return len(bs.chain) }
+func (bs *mockBlockStore) LoadBlock(height int) *types.Block { return bs.chain[height-1] }

state/state.go

@@ -69,7 +69,7 @@ func (s *State) Copy() *State {
 		LastBlockTime:   s.LastBlockTime,
 		Validators:      s.Validators.Copy(),
 		LastValidators:  s.LastValidators.Copy(),
-		Stale:           s.Stale, // but really state shouldnt be copied while its stale
+		Stale:           s.Stale, // XXX: but really state shouldnt be copied while its stale
 		AppHash:         s.AppHash,
 	}
 }
@@ -94,7 +94,7 @@ func (s *State) Bytes() []byte {
 }
 
 // Mutate state variables to match block and validators
-// Since we don't have the AppHash yet, it becomes stale
+// Since we don't have the new AppHash yet, we set s.Stale=true
 func (s *State) SetBlockAndValidators(header *types.Header, blockPartsHeader types.PartSetHeader, prevValSet, nextValSet *types.ValidatorSet) {
 	s.LastBlockHeight = header.Height
 	s.LastBlockID = types.BlockID{block.Hash(), blockPartsHeader}

state/state_test.go

@@ -0,0 +1,42 @@
+package state
+
+import (
+	"testing"
+
+	dbm "github.com/tendermint/go-db"
+	"github.com/tendermint/tendermint/config/tendermint_test"
+)
+
+func TestStateCopyEquals(t *testing.T) {
+	config := tendermint_test.ResetConfig("state_")
+	// Get State db
+	stateDB := dbm.NewDB("state", config.GetString("db_backend"), config.GetString("db_dir"))
+	state := GetState(config, stateDB)
+
+	stateCopy := state.Copy()
+
+	if !state.Equals(stateCopy) {
+		t.Fatal("expected state and its copy to be identical. got %v\n expected %v\n", stateCopy, state)
+	}
+
+	stateCopy.LastBlockHeight += 1
+
+	if state.Equals(stateCopy) {
+		t.Fatal("expected states to be different. got same %v", state)
+	}
+}
+
+func TestStateSaveLoad(t *testing.T) {
+	config := tendermint_test.ResetConfig("state_")
+	// Get State db
+	stateDB := dbm.NewDB("state", config.GetString("db_backend"), config.GetString("db_dir"))
+	state := GetState(config, stateDB)
+
+	state.LastBlockHeight += 1
+	state.Save()
+
+	loadedState := LoadState(stateDB)
+	if !state.Equals(loadedState) {
+		t.Fatal("expected state and its copy to be identical. got %v\n expected %v\n", loadedState, state)
+	}
+}

types/block.go

@@ -21,6 +21,27 @@ type Block struct {
 	LastCommit *Commit `json:"last_commit"`
 }
 
+func MakeBlock(height int, chainID string, txs []Tx, commit *Commit,
+	prevBlockID BlockID, valHash, appHash []byte) (*Block, *PartSet) {
+	block := &Block{
+		Header: &Header{
+			ChainID:        chainID,
+			Height:         height,
+			Time:           time.Now(),
+			NumTxs:         len(txs),
+			LastBlockID:    prevBlockID,
+			ValidatorsHash: valHash,
+			AppHash:        appHash, // state merkle root of txs from the previous block.
+		},
+		LastCommit: commit,
+		Data: &Data{
+			Txs: txs,
+		},
+	}
+	block.FillHeader()
+	return block, block.MakePartSet()
+}
+
 // Basic validation that doesn't involve state data.
 func (b *Block) ValidateBasic(chainID string, lastBlockHeight int, lastBlockID BlockID,
 	lastBlockTime time.Time, appHash []byte) error {

types/validator.go

@@ -11,8 +11,9 @@ import (
 )
 
 // Volatile state for each Validator
-// Also persisted with the state, but fields change
-// every height|round so they don't go in merkle.Tree
+// TODO: make non-volatile identity
+// 	- Remove LastCommitHeight, send bitarray of vals that signed in BeginBlock
+// 	- Remove Accum - it can be computed, and now valset becomes identifying
 type Validator struct {
 	Address          []byte        `json:"address"`
 	PubKey           crypto.PubKey `json:"pub_key"`

types/validator_set.go

@@ -20,7 +20,7 @@ import (
 // NOTE: Not goroutine-safe.
 // NOTE: All get/set to validators should copy the value for safety.
 // TODO: consider validator Accum overflow
-// TODO: replace validators []*Validator with github.com/jaekwon/go-ibbs?
+// TODO: move valset into an iavl tree where key is 'blockbonded|pubkey'
 type ValidatorSet struct {
 	Validators []*Validator // NOTE: persisted via reflect, must be exported.