diff --git a/consensus/replay.go b/consensus/replay.go index 0c7c27e90..71dc81b5a 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -324,8 +324,11 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp p func (h *Handshaker) replayBlocks(proxyApp proxy.AppConns, appBlockHeight, storeBlockHeight int, mutateState bool) ([]byte, error) { // App is further behind than it should be, so we need to replay blocks. // We replay all blocks from appBlockHeight+1. + // // Note that we don't have an old version of the state, // so we by-pass state validation/mutation using sm.ExecCommitBlock. + // This also means we won't be saving validator sets if they change during this period. + // // If mutateState == true, the final block is replayed with h.replayBlock() var appHash []byte diff --git a/state/errors.go b/state/errors.go index 50c5a2c04..4a87384a0 100644 --- a/state/errors.go +++ b/state/errors.go @@ -33,6 +33,10 @@ type ( Got *State Expected *State } + + ErrNoValSetForHeight struct { + Height int + } ) func (e ErrUnknownBlock) Error() string { @@ -53,3 +57,7 @@ func (e ErrLastStateMismatch) Error() string { func (e ErrStateMismatch) Error() string { return cmn.Fmt("State after replay does not match saved state. Got ----\n%v\nExpected ----\n%v\n", e.Got, e.Expected) } + +func (e ErrNoValSetForHeight) Error() string { + return cmn.Fmt("Could not find validator set for height #%d", e.Height) +} diff --git a/state/execution.go b/state/execution.go index 9e4a5fb29..869d28d48 100644 --- a/state/execution.go +++ b/state/execution.go @@ -231,6 +231,9 @@ func (s *State) ApplyBlock(eventCache types.Fireable, proxyAppConn proxy.AppConn // now update the block and validators s.SetBlockAndValidators(block.Header, partsHeader, abciResponses) + // save the validators for the next block now that we know them + s.SaveValidators() + // lock mempool, commit state, update mempoool err = s.CommitStateUpdateMempool(proxyAppConn, block, mempool) if err != nil { diff --git a/state/state.go b/state/state.go index 21ad92776..6106891b5 100644 --- a/state/state.go +++ b/state/state.go @@ -22,6 +22,10 @@ var ( abciResponsesKey = []byte("abciResponsesKey") ) +func calcValidatorsKey(height int) []byte { + return []byte(cmn.Fmt("validatorsKey:%v", height)) +} + //----------------------------------------------------------------------------- // State represents the latest committed state of the Tendermint consensus, @@ -47,8 +51,11 @@ type State struct { // AppHash is updated after Commit AppHash []byte + // XXX: do we need this json tag ? TxIndexer txindex.TxIndexer `json:"-"` // Transaction indexer. + lastHeightValidatorsChanged int + logger log.Logger } @@ -71,6 +78,14 @@ func loadState(db dbm.DB, key []byte) *State { } // TODO: ensure that buf is completely read. } + + v := s.loadValidators(s.LastBlockHeight) + if v != nil { + s.lastHeightValidatorsChanged = v.LastHeightChanged + } else { + s.lastHeightValidatorsChanged = 1 + } + return s } @@ -93,6 +108,8 @@ func (s *State) Copy() *State { AppHash: s.AppHash, TxIndexer: s.TxIndexer, // pointer here, not value logger: s.logger, + + lastHeightValidatorsChanged: s.lastHeightValidatorsChanged, } } @@ -126,6 +143,56 @@ func (s *State) LoadABCIResponses() *ABCIResponses { return abciResponses } +// SaveValidators persists the validator set for the next block to disk. +// It should be called after the validator set is updated with the results of EndBlock. +// If the validator set did not change after processing the latest block, +// only the last height for which the validators changed is persisted. +func (s *State) SaveValidators() { + lastHeight := s.lastHeightValidatorsChanged + nextHeight := s.LastBlockHeight + 1 + v := &Validators{ + LastHeightChanged: lastHeight, + } + if lastHeight == nextHeight { + v.ValidatorSet = s.Validators + } + s.db.SetSync(calcValidatorsKey(nextHeight), v.Bytes()) +} + +// LoadValidators loads the ValidatorSet for a given height. +func (s *State) LoadValidators(height int) (*types.ValidatorSet, error) { + v := s.loadValidators(height) + if v == nil { + return nil, ErrNoValSetForHeight{height} + } + + if v.ValidatorSet == nil { + v = s.loadValidators(v.LastHeightChanged) + if v == nil { + return nil, ErrNoValSetForHeight{height} + } + } + + return v.ValidatorSet, nil +} + +func (s *State) loadValidators(height int) *Validators { + buf := s.db.Get(calcValidatorsKey(height)) + if len(buf) == 0 { + return nil + } + + v := new(Validators) + r, n, err := bytes.NewReader(buf), new(int), new(error) + wire.ReadBinaryPtr(v, r, 0, n, err) + if *err != nil { + // DATA HAS BEEN CORRUPTED OR THE SPEC HAS CHANGED + cmn.Exit(cmn.Fmt("LoadValidators: Data has been corrupted or its spec has changed: %v\n", *err)) + } + // TODO: ensure that buf is completely read. + return v +} + // Equals returns true if the States are identical. func (s *State) Equals(s2 *State) bool { return bytes.Equal(s.Bytes(), s2.Bytes()) @@ -144,10 +211,15 @@ func (s *State) SetBlockAndValidators(header *types.Header, blockPartsHeader typ prevValSet := s.Validators.Copy() nextValSet := prevValSet.Copy() - err := updateValidators(nextValSet, abciResponses.EndBlock.Diffs) - if err != nil { - s.logger.Error("Error changing validator set", "err", err) - // TODO: err or carry on? + // update the validator set with the latest abciResponses + if len(abciResponses.EndBlock.Diffs) > 0 { + err := updateValidators(nextValSet, abciResponses.EndBlock.Diffs) + if err != nil { + s.logger.Error("Error changing validator set", "err", err) + // TODO: err or carry on? + } + // change results from this height but only applies to the next height + s.lastHeightValidatorsChanged = header.Height + 1 } // Update validator accums and set state variables @@ -182,6 +254,7 @@ func GetState(stateDB dbm.DB, genesisFile string) *State { state := LoadState(stateDB) if state == nil { state = MakeGenesisStateFromFile(stateDB, genesisFile) + state.SaveValidators() // save the validators right away for height 1 state.Save() } @@ -215,6 +288,19 @@ func (a *ABCIResponses) Bytes() []byte { return wire.BinaryBytes(*a) } +//----------------------------------------------------------------------------- + +// Validators represents the latest validator set, or the last time it changed +type Validators struct { + ValidatorSet *types.ValidatorSet + LastHeightChanged int +} + +// Bytes serializes the Validators using go-wire +func (v *Validators) Bytes() []byte { + return wire.BinaryBytes(*v) +} + //----------------------------------------------------------------------------- // Genesis @@ -260,15 +346,16 @@ func MakeGenesisState(db dbm.DB, genDoc *types.GenesisDoc) *State { } return &State{ - db: db, - GenesisDoc: genDoc, - ChainID: genDoc.ChainID, - LastBlockHeight: 0, - LastBlockID: types.BlockID{}, - LastBlockTime: genDoc.GenesisTime, - Validators: types.NewValidatorSet(validators), - LastValidators: types.NewValidatorSet(nil), - AppHash: genDoc.AppHash, - TxIndexer: &null.TxIndex{}, // we do not need indexer during replay and in tests + db: db, + GenesisDoc: genDoc, + ChainID: genDoc.ChainID, + LastBlockHeight: 0, + LastBlockID: types.BlockID{}, + LastBlockTime: genDoc.GenesisTime, + Validators: types.NewValidatorSet(validators), + LastValidators: types.NewValidatorSet(nil), + AppHash: genDoc.AppHash, + TxIndexer: &null.TxIndex{}, // we do not need indexer during replay and in tests + lastHeightValidatorsChanged: 1, } } diff --git a/state/state_test.go b/state/state_test.go index e97c3289a..22543cc17 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -1,18 +1,21 @@ package state import ( - "fmt" "testing" "github.com/stretchr/testify/assert" + abci "github.com/tendermint/abci/types" crypto "github.com/tendermint/go-crypto" - cfg "github.com/tendermint/tendermint/config" + cmn "github.com/tendermint/tmlibs/common" dbm "github.com/tendermint/tmlibs/db" "github.com/tendermint/tmlibs/log" + + cfg "github.com/tendermint/tendermint/config" ) func TestStateCopyEquals(t *testing.T) { + assert := assert.New(t) config := cfg.ResetTestRoot("state_") // Get State db @@ -22,18 +25,13 @@ func TestStateCopyEquals(t *testing.T) { stateCopy := state.Copy() - if !state.Equals(stateCopy) { - t.Fatal("expected state and its copy to be identical. got %v\n expected %v\n", stateCopy, state) - } - + assert.True(state.Equals(stateCopy), cmn.Fmt("expected state and its copy to be identical. got %v\n expected %v\n", stateCopy, state)) stateCopy.LastBlockHeight += 1 - - if state.Equals(stateCopy) { - t.Fatal("expected states to be different. got same %v", state) - } + assert.False(state.Equals(stateCopy), cmn.Fmt("expected states to be different. got same %v", state)) } func TestStateSaveLoad(t *testing.T) { + assert := assert.New(t) config := cfg.ResetTestRoot("state_") // Get State db stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir()) @@ -44,9 +42,7 @@ func TestStateSaveLoad(t *testing.T) { state.Save() loadedState := LoadState(stateDB) - if !state.Equals(loadedState) { - t.Fatal("expected state and its copy to be identical. got %v\n expected %v\n", loadedState, state) - } + assert.True(state.Equals(loadedState), cmn.Fmt("expected state and its copy to be identical. got %v\n expected %v\n", loadedState, state)) } func TestABCIResponsesSaveLoad(t *testing.T) { @@ -74,5 +70,41 @@ func TestABCIResponsesSaveLoad(t *testing.T) { state.SaveABCIResponses(abciResponses) abciResponses2 := state.LoadABCIResponses() - assert.Equal(abciResponses, abciResponses2, fmt.Sprintf("ABCIResponses don't match: Got %v, Expected %v", abciResponses2, abciResponses)) + assert.Equal(abciResponses, abciResponses2, cmn.Fmt("ABCIResponses don't match: Got %v, Expected %v", abciResponses2, abciResponses)) +} + +func TestValidatorsSaveLoad(t *testing.T) { + assert := assert.New(t) + config := cfg.ResetTestRoot("state_") + // Get State db + stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir()) + state := GetState(stateDB, config.GenesisFile()) + state.SetLogger(log.TestingLogger()) + + // cant load anything for height 0 + v, err := state.LoadValidators(0) + assert.NotNil(err, "expected err at height 0") + + // should be able to load for height 1 + v, err = state.LoadValidators(1) + assert.Nil(err, "expected no err at height 1") + assert.Equal(v.Hash(), state.Validators.Hash(), "expected validator hashes to match") + + // increment height, save; should be able to load for next height + state.LastBlockHeight += 1 + state.SaveValidators() + v, err = state.LoadValidators(state.LastBlockHeight + 1) + assert.Nil(err, "expected no err") + assert.Equal(v.Hash(), state.Validators.Hash(), "expected validator hashes to match") + + // increment height, save; should be able to load for next height + state.LastBlockHeight += 10 + state.SaveValidators() + v, err = state.LoadValidators(state.LastBlockHeight + 1) + assert.Nil(err, "expected no err") + assert.Equal(v.Hash(), state.Validators.Hash(), "expected validator hashes to match") + + // should be able to load for next next height + _, err = state.LoadValidators(state.LastBlockHeight + 2) + assert.NotNil(err, "expected err") }