From 16ba782fa6549ea1e50a7fe7addb538ff15bf619 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Fri, 8 Oct 2021 14:35:13 +0200 Subject: [PATCH] cli: allow node operator to rollback last state (backport #7033) (#7080) --- CHANGELOG_PENDING.md | 3 +- cmd/tendermint/commands/rollback.go | 69 ++++++++++ cmd/tendermint/main.go | 1 + state/mocks/block_store.go | 194 ++++++++++++++++++++++++++++ state/mocks/evidence_pool.go | 2 +- state/mocks/store.go | 2 +- state/rollback.go | 89 +++++++++++++ state/rollback_test.go | 165 +++++++++++++++++++++++ state/services.go | 2 + 9 files changed, 524 insertions(+), 3 deletions(-) create mode 100644 cmd/tendermint/commands/rollback.go create mode 100644 state/mocks/block_store.go create mode 100644 state/rollback.go create mode 100644 state/rollback_test.go diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 044f5fb6f..9b333b83c 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -21,9 +21,10 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi ### FEATURES - [\#6982](https://github.com/tendermint/tendermint/pull/6982) tendermint binary has built-in suppport for running the end to end application (with state sync support) (@cmwaters). +- [cli] [#7033](https://github.com/tendermint/tendermint/pull/7033) Add a `rollback` command to rollback to the previous tendermint state in the event of non-determinstic app hash or reverting an upgrade. ### IMPROVEMENTS ### BUG FIXES -- [\#7057](https://github.com/tendermint/tendermint/pull/7057) Import Postgres driver support for the psql indexer (@creachadair). +- [\#7057](https://github.com/tendermint/tendermint/pull/7057) Import Postgres driver support for the psql indexer (@creachadair). \ No newline at end of file diff --git a/cmd/tendermint/commands/rollback.go b/cmd/tendermint/commands/rollback.go new file mode 100644 index 000000000..a91e72102 --- /dev/null +++ b/cmd/tendermint/commands/rollback.go @@ -0,0 +1,69 @@ +package commands + +import ( + "fmt" + + "github.com/spf13/cobra" + + dbm "github.com/tendermint/tm-db" + + cfg "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/state" + "github.com/tendermint/tendermint/store" +) + +var RollbackStateCmd = &cobra.Command{ + Use: "rollback", + Short: "rollback tendermint state by one height", + Long: ` +A state rollback is performed to recover from an incorrect application state transition, +when Tendermint has persisted an incorrect app hash and is thus unable to make +progress. Rollback overwrites a state at height n with the state at height n - 1. +The application should also roll back to height n - 1. No blocks are removed, so upon +restarting Tendermint the transactions in block n will be re-executed against the +application. +`, + RunE: func(cmd *cobra.Command, args []string) error { + height, hash, err := RollbackState(config) + if err != nil { + return fmt.Errorf("failed to rollback state: %w", err) + } + + fmt.Printf("Rolled back state to height %d and hash %v", height, hash) + return nil + }, +} + +// RollbackState takes the state at the current height n and overwrites it with the state +// at height n - 1. Note state here refers to tendermint state not application state. +// Returns the latest state height and app hash alongside an error if there was one. +func RollbackState(config *cfg.Config) (int64, []byte, error) { + // use the parsed config to load the block and state store + blockStore, stateStore, err := loadStateAndBlockStore(config) + if err != nil { + return -1, nil, err + } + + // rollback the last state + return state.Rollback(blockStore, stateStore) +} + +func loadStateAndBlockStore(config *cfg.Config) (*store.BlockStore, state.Store, error) { + dbType := dbm.BackendType(config.DBBackend) + + // Get BlockStore + blockStoreDB, err := dbm.NewDB("blockstore", dbType, config.DBDir()) + if err != nil { + return nil, nil, err + } + blockStore := store.NewBlockStore(blockStoreDB) + + // Get StateStore + stateDB, err := dbm.NewDB("state", dbType, config.DBDir()) + if err != nil { + return nil, nil, err + } + stateStore := state.NewStore(stateDB) + + return blockStore, stateStore, nil +} diff --git a/cmd/tendermint/main.go b/cmd/tendermint/main.go index 311a59a65..730946aa9 100644 --- a/cmd/tendermint/main.go +++ b/cmd/tendermint/main.go @@ -27,6 +27,7 @@ func main() { cmd.ShowNodeIDCmd, cmd.GenNodeKeyCmd, cmd.VersionCmd, + cmd.RollbackStateCmd, debug.DebugCmd, cli.NewCompletionCmd(rootCmd, true), ) diff --git a/state/mocks/block_store.go b/state/mocks/block_store.go new file mode 100644 index 000000000..91bfd0f5b --- /dev/null +++ b/state/mocks/block_store.go @@ -0,0 +1,194 @@ +// Code generated by mockery 2.9.0. DO NOT EDIT. + +package mocks + +import ( + mock "github.com/stretchr/testify/mock" + + types "github.com/tendermint/tendermint/types" +) + +// BlockStore is an autogenerated mock type for the BlockStore type +type BlockStore struct { + mock.Mock +} + +// Base provides a mock function with given fields: +func (_m *BlockStore) Base() int64 { + ret := _m.Called() + + var r0 int64 + if rf, ok := ret.Get(0).(func() int64); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(int64) + } + + return r0 +} + +// Height provides a mock function with given fields: +func (_m *BlockStore) Height() int64 { + ret := _m.Called() + + var r0 int64 + if rf, ok := ret.Get(0).(func() int64); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(int64) + } + + return r0 +} + +// LoadBaseMeta provides a mock function with given fields: +func (_m *BlockStore) LoadBaseMeta() *types.BlockMeta { + ret := _m.Called() + + var r0 *types.BlockMeta + if rf, ok := ret.Get(0).(func() *types.BlockMeta); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.BlockMeta) + } + } + + return r0 +} + +// LoadBlock provides a mock function with given fields: height +func (_m *BlockStore) LoadBlock(height int64) *types.Block { + ret := _m.Called(height) + + var r0 *types.Block + if rf, ok := ret.Get(0).(func(int64) *types.Block); ok { + r0 = rf(height) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.Block) + } + } + + return r0 +} + +// LoadBlockByHash provides a mock function with given fields: hash +func (_m *BlockStore) LoadBlockByHash(hash []byte) *types.Block { + ret := _m.Called(hash) + + var r0 *types.Block + if rf, ok := ret.Get(0).(func([]byte) *types.Block); ok { + r0 = rf(hash) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.Block) + } + } + + return r0 +} + +// LoadBlockCommit provides a mock function with given fields: height +func (_m *BlockStore) LoadBlockCommit(height int64) *types.Commit { + ret := _m.Called(height) + + var r0 *types.Commit + if rf, ok := ret.Get(0).(func(int64) *types.Commit); ok { + r0 = rf(height) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.Commit) + } + } + + return r0 +} + +// LoadBlockMeta provides a mock function with given fields: height +func (_m *BlockStore) LoadBlockMeta(height int64) *types.BlockMeta { + ret := _m.Called(height) + + var r0 *types.BlockMeta + if rf, ok := ret.Get(0).(func(int64) *types.BlockMeta); ok { + r0 = rf(height) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.BlockMeta) + } + } + + return r0 +} + +// LoadBlockPart provides a mock function with given fields: height, index +func (_m *BlockStore) LoadBlockPart(height int64, index int) *types.Part { + ret := _m.Called(height, index) + + var r0 *types.Part + if rf, ok := ret.Get(0).(func(int64, int) *types.Part); ok { + r0 = rf(height, index) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.Part) + } + } + + return r0 +} + +// LoadSeenCommit provides a mock function with given fields: height +func (_m *BlockStore) LoadSeenCommit(height int64) *types.Commit { + ret := _m.Called(height) + + var r0 *types.Commit + if rf, ok := ret.Get(0).(func(int64) *types.Commit); ok { + r0 = rf(height) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.Commit) + } + } + + return r0 +} + +// PruneBlocks provides a mock function with given fields: height +func (_m *BlockStore) PruneBlocks(height int64) (uint64, error) { + ret := _m.Called(height) + + var r0 uint64 + if rf, ok := ret.Get(0).(func(int64) uint64); ok { + r0 = rf(height) + } else { + r0 = ret.Get(0).(uint64) + } + + var r1 error + if rf, ok := ret.Get(1).(func(int64) error); ok { + r1 = rf(height) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// SaveBlock provides a mock function with given fields: block, blockParts, seenCommit +func (_m *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) { + _m.Called(block, blockParts, seenCommit) +} + +// Size provides a mock function with given fields: +func (_m *BlockStore) Size() int64 { + ret := _m.Called() + + var r0 int64 + if rf, ok := ret.Get(0).(func() int64); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(int64) + } + + return r0 +} diff --git a/state/mocks/evidence_pool.go b/state/mocks/evidence_pool.go index 7292991ca..9d6091cde 100644 --- a/state/mocks/evidence_pool.go +++ b/state/mocks/evidence_pool.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.1.0. DO NOT EDIT. +// Code generated by mockery 2.9.0. DO NOT EDIT. package mocks diff --git a/state/mocks/store.go b/state/mocks/store.go index 17e1ef7b9..91525b223 100644 --- a/state/mocks/store.go +++ b/state/mocks/store.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.1.0. DO NOT EDIT. +// Code generated by mockery 2.9.0. DO NOT EDIT. package mocks diff --git a/state/rollback.go b/state/rollback.go new file mode 100644 index 000000000..211f0ba9d --- /dev/null +++ b/state/rollback.go @@ -0,0 +1,89 @@ +package state + +import ( + "errors" + "fmt" + + tmstate "github.com/tendermint/tendermint/proto/tendermint/state" + tmversion "github.com/tendermint/tendermint/proto/tendermint/version" + "github.com/tendermint/tendermint/version" +) + +// Rollback overwrites the current Tendermint state (height n) with the most +// recent previous state (height n - 1). +// Note that this function does not affect application state. +func Rollback(bs BlockStore, ss Store) (int64, []byte, error) { + invalidState, err := ss.Load() + if err != nil { + return -1, nil, err + } + if invalidState.IsEmpty() { + return -1, nil, errors.New("no state found") + } + + rollbackHeight := invalidState.LastBlockHeight + rollbackBlock := bs.LoadBlockMeta(rollbackHeight) + if rollbackBlock == nil { + return -1, nil, fmt.Errorf("block at height %d not found", rollbackHeight) + } + + previousValidatorSet, err := ss.LoadValidators(rollbackHeight - 1) + if err != nil { + return -1, nil, err + } + + previousParams, err := ss.LoadConsensusParams(rollbackHeight) + if err != nil { + return -1, nil, err + } + + valChangeHeight := invalidState.LastHeightValidatorsChanged + // this can only happen if the validator set changed since the last block + if valChangeHeight > rollbackHeight { + valChangeHeight = rollbackHeight + } + + paramsChangeHeight := invalidState.LastHeightConsensusParamsChanged + // this can only happen if params changed from the last block + if paramsChangeHeight > rollbackHeight { + paramsChangeHeight = rollbackHeight + } + + // build the new state from the old state and the prior block + rolledBackState := State{ + Version: tmstate.Version{ + Consensus: tmversion.Consensus{ + Block: version.BlockProtocol, + App: previousParams.Version.AppVersion, + }, + Software: version.TMCoreSemVer, + }, + // immutable fields + ChainID: invalidState.ChainID, + InitialHeight: invalidState.InitialHeight, + + LastBlockHeight: invalidState.LastBlockHeight - 1, + LastBlockID: rollbackBlock.Header.LastBlockID, + LastBlockTime: rollbackBlock.Header.Time, + + NextValidators: invalidState.Validators, + Validators: invalidState.LastValidators, + LastValidators: previousValidatorSet, + LastHeightValidatorsChanged: valChangeHeight, + + ConsensusParams: previousParams, + LastHeightConsensusParamsChanged: paramsChangeHeight, + + LastResultsHash: rollbackBlock.Header.LastResultsHash, + AppHash: rollbackBlock.Header.AppHash, + } + + // persist the new state. This overrides the invalid one. NOTE: this will also + // persist the validator set and consensus params over the existing structures, + // but both should be the same + if err := ss.Save(rolledBackState); err != nil { + return -1, nil, fmt.Errorf("failed to save rolled back state: %w", err) + } + + return rolledBackState.LastBlockHeight, rolledBackState.AppHash, nil +} diff --git a/state/rollback_test.go b/state/rollback_test.go new file mode 100644 index 000000000..06927d37c --- /dev/null +++ b/state/rollback_test.go @@ -0,0 +1,165 @@ +package state_test + +import ( + "crypto/rand" + "testing" + + "github.com/stretchr/testify/require" + dbm "github.com/tendermint/tm-db" + + "github.com/tendermint/tendermint/crypto/tmhash" + tmstate "github.com/tendermint/tendermint/proto/tendermint/state" + tmversion "github.com/tendermint/tendermint/proto/tendermint/version" + "github.com/tendermint/tendermint/state" + "github.com/tendermint/tendermint/state/mocks" + "github.com/tendermint/tendermint/types" + "github.com/tendermint/tendermint/version" +) + +func TestRollback(t *testing.T) { + stateStore := state.NewStore(dbm.NewMemDB()) + blockStore := &mocks.BlockStore{} + var ( + height int64 = 100 + appVersion uint64 = 10 + ) + + valSet, _ := types.RandValidatorSet(5, 10) + + params := types.DefaultConsensusParams() + params.Version.AppVersion = appVersion + newParams := types.DefaultConsensusParams() + newParams.Block.MaxBytes = 10000 + + initialState := state.State{ + Version: tmstate.Version{ + Consensus: tmversion.Consensus{ + Block: version.BlockProtocol, + App: 10, + }, + Software: version.TMCoreSemVer, + }, + ChainID: "test-chain", + InitialHeight: 10, + LastBlockID: makeBlockIDRandom(), + AppHash: tmhash.Sum([]byte("app_hash")), + LastResultsHash: tmhash.Sum([]byte("last_results_hash")), + LastBlockHeight: height, + LastValidators: valSet, + Validators: valSet.CopyIncrementProposerPriority(1), + NextValidators: valSet.CopyIncrementProposerPriority(2), + LastHeightValidatorsChanged: height + 1, + ConsensusParams: *params, + LastHeightConsensusParamsChanged: height + 1, + } + require.NoError(t, stateStore.Bootstrap(initialState)) + + height++ + block := &types.BlockMeta{ + Header: types.Header{ + Height: height, + AppHash: initialState.AppHash, + LastBlockID: initialState.LastBlockID, + LastResultsHash: initialState.LastResultsHash, + }, + } + blockStore.On("LoadBlockMeta", height).Return(block) + + appVersion++ + newParams.Version.AppVersion = appVersion + nextState := initialState.Copy() + nextState.LastBlockHeight = height + nextState.Version.Consensus.App = appVersion + nextState.LastBlockID = makeBlockIDRandom() + nextState.AppHash = tmhash.Sum([]byte("next_app_hash")) + nextState.LastValidators = initialState.Validators + nextState.Validators = initialState.NextValidators + nextState.NextValidators = initialState.NextValidators.CopyIncrementProposerPriority(1) + nextState.ConsensusParams = *newParams + nextState.LastHeightConsensusParamsChanged = height + 1 + nextState.LastHeightValidatorsChanged = height + 1 + + // update the state + require.NoError(t, stateStore.Save(nextState)) + + // rollback the state + rollbackHeight, rollbackHash, err := state.Rollback(blockStore, stateStore) + require.NoError(t, err) + require.EqualValues(t, int64(100), rollbackHeight) + require.EqualValues(t, initialState.AppHash, rollbackHash) + blockStore.AssertExpectations(t) + + // assert that we've recovered the prior state + loadedState, err := stateStore.Load() + require.NoError(t, err) + require.EqualValues(t, initialState, loadedState) +} + +func TestRollbackNoState(t *testing.T) { + stateStore := state.NewStore(dbm.NewMemDB()) + blockStore := &mocks.BlockStore{} + + _, _, err := state.Rollback(blockStore, stateStore) + require.Error(t, err) + require.Contains(t, err.Error(), "no state found") +} + +func TestRollbackNoBlocks(t *testing.T) { + stateStore := state.NewStore(dbm.NewMemDB()) + blockStore := &mocks.BlockStore{} + var ( + height int64 = 100 + appVersion uint64 = 10 + ) + + valSet, _ := types.RandValidatorSet(5, 10) + + params := types.DefaultConsensusParams() + params.Version.AppVersion = appVersion + newParams := types.DefaultConsensusParams() + newParams.Block.MaxBytes = 10000 + + initialState := state.State{ + Version: tmstate.Version{ + Consensus: tmversion.Consensus{ + Block: version.BlockProtocol, + App: 10, + }, + Software: version.TMCoreSemVer, + }, + ChainID: "test-chain", + InitialHeight: 10, + LastBlockID: makeBlockIDRandom(), + AppHash: tmhash.Sum([]byte("app_hash")), + LastResultsHash: tmhash.Sum([]byte("last_results_hash")), + LastBlockHeight: height, + LastValidators: valSet, + Validators: valSet.CopyIncrementProposerPriority(1), + NextValidators: valSet.CopyIncrementProposerPriority(2), + LastHeightValidatorsChanged: height + 1, + ConsensusParams: *params, + LastHeightConsensusParamsChanged: height + 1, + } + require.NoError(t, stateStore.Save(initialState)) + blockStore.On("LoadBlockMeta", height).Return(nil) + + _, _, err := state.Rollback(blockStore, stateStore) + require.Error(t, err) + require.Contains(t, err.Error(), "block at height 100 not found") +} + +func makeBlockIDRandom() types.BlockID { + var ( + blockHash = make([]byte, tmhash.Size) + partSetHash = make([]byte, tmhash.Size) + ) + rand.Read(blockHash) //nolint: errcheck // ignore errcheck for read + rand.Read(partSetHash) //nolint: errcheck // ignore errcheck for read + return types.BlockID{ + Hash: blockHash, + PartSetHeader: types.PartSetHeader{ + Total: 123, + Hash: partSetHash, + }, + } +} diff --git a/state/services.go b/state/services.go index a46863904..2cc376e85 100644 --- a/state/services.go +++ b/state/services.go @@ -12,6 +12,8 @@ import ( //------------------------------------------------------ // blockstore +//go:generate mockery --case underscore --name BlockStore + // BlockStore defines the interface used by the ConsensusState. type BlockStore interface { Base() int64