From f0cd54825f6b32279932104e3ce182f60dc68ab4 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Fri, 8 Oct 2021 09:56:18 +0200 Subject: [PATCH] cli: allow node operator to rollback last state (backport #7033) (#7081) --- CHANGELOG_PENDING.md | 1 + cmd/tendermint/commands/reindex_event.go | 11 +- cmd/tendermint/commands/rollback.go | 46 +++++++ cmd/tendermint/main.go | 1 + internal/state/rollback.go | 87 ++++++++++++++ internal/state/rollback_test.go | 146 +++++++++++++++++++++++ internal/state/state.go | 2 + test/e2e/app/app.go | 6 + 8 files changed, 295 insertions(+), 5 deletions(-) create mode 100644 cmd/tendermint/commands/rollback.go create mode 100644 internal/state/rollback.go create mode 100644 internal/state/rollback_test.go diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 56efd763b..12da9e705 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -22,6 +22,7 @@ Special thanks to external contributors on this release: ### FEATURES +- [cli] [#7033](https://github.com/tendermint/tendermint/pull/7033) Add a `rollback` command to rollback to the previous tendermint state in the event of an incorrect app hash. (@cmwaters) ### IMPROVEMENTS ### BUG FIXES diff --git a/cmd/tendermint/commands/reindex_event.go b/cmd/tendermint/commands/reindex_event.go index 9545e32d7..58f11657b 100644 --- a/cmd/tendermint/commands/reindex_event.go +++ b/cmd/tendermint/commands/reindex_event.go @@ -29,11 +29,12 @@ var ReIndexEventCmd = &cobra.Command{ Use: "reindex-event", Short: "reindex events to the event store backends", Long: ` - reindex-event is an offline tooling to re-index block and tx events to the eventsinks, - you can run this command when the event store backend dropped/disconnected or you want to replace the backend. - The default start-height is 0, meaning the tooling will start reindex from the base block height(inclusive); and the - default end-height is 0, meaning the tooling will reindex until the latest block height(inclusive). User can omits - either or both arguments. +reindex-event is an offline tooling to re-index block and tx events to the eventsinks, +you can run this command when the event store backend dropped/disconnected or you want to +replace the backend. The default start-height is 0, meaning the tooling will start +reindex from the base block height(inclusive); and the default end-height is 0, meaning +the tooling will reindex until the latest block height(inclusive). User can omit +either or both arguments. `, Example: ` tendermint reindex-event diff --git a/cmd/tendermint/commands/rollback.go b/cmd/tendermint/commands/rollback.go new file mode 100644 index 000000000..5aff232be --- /dev/null +++ b/cmd/tendermint/commands/rollback.go @@ -0,0 +1,46 @@ +package commands + +import ( + "fmt" + + "github.com/spf13/cobra" + + cfg "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/internal/state" +) + +var RollbackStateCmd = &cobra.Command{ + Use: "rollback", + Short: "rollback tendermint state by one height", + Long: ` +A state rollback is performed to recover from an incorrect application state transition, +when Tendermint has persisted an incorrect app hash and is thus unable to make +progress. Rollback overwrites a state at height n with the state at height n - 1. +The application should also roll back to height n - 1. No blocks are removed, so upon +restarting Tendermint the transactions in block n will be re-executed against the +application. +`, + RunE: func(cmd *cobra.Command, args []string) error { + height, hash, err := RollbackState(config) + if err != nil { + return fmt.Errorf("failed to rollback state: %w", err) + } + + fmt.Printf("Rolled back state to height %d and hash %v", height, hash) + return nil + }, +} + +// RollbackState takes the state at the current height n and overwrites it with the state +// at height n - 1. Note state here refers to tendermint state not application state. +// Returns the latest state height and app hash alongside an error if there was one. +func RollbackState(config *cfg.Config) (int64, []byte, error) { + // use the parsed config to load the block and state store + blockStore, stateStore, err := loadStateAndBlockStore(config) + if err != nil { + return -1, nil, err + } + + // rollback the last state + return state.Rollback(blockStore, stateStore) +} diff --git a/cmd/tendermint/main.go b/cmd/tendermint/main.go index 9f908a1ed..52a00e4c0 100644 --- a/cmd/tendermint/main.go +++ b/cmd/tendermint/main.go @@ -29,6 +29,7 @@ func main() { cmd.GenNodeKeyCmd, cmd.VersionCmd, cmd.InspectCmd, + cmd.RollbackStateCmd, cmd.MakeKeyMigrateCommand(), debug.DebugCmd, cli.NewCompletionCmd(rootCmd, true), diff --git a/internal/state/rollback.go b/internal/state/rollback.go new file mode 100644 index 000000000..6e13da0e2 --- /dev/null +++ b/internal/state/rollback.go @@ -0,0 +1,87 @@ +package state + +import ( + "errors" + "fmt" + + "github.com/tendermint/tendermint/version" +) + +// Rollback overwrites the current Tendermint state (height n) with the most +// recent previous state (height n - 1). +// Note that this function does not affect application state. +func Rollback(bs BlockStore, ss Store) (int64, []byte, error) { + invalidState, err := ss.Load() + if err != nil { + return -1, nil, err + } + if invalidState.IsEmpty() { + return -1, nil, errors.New("no state found") + } + + rollbackHeight := invalidState.LastBlockHeight + rollbackBlock := bs.LoadBlockMeta(rollbackHeight) + if rollbackBlock == nil { + return -1, nil, fmt.Errorf("block at height %d not found", rollbackHeight) + } + + previousValidatorSet, err := ss.LoadValidators(rollbackHeight - 1) + if err != nil { + return -1, nil, err + } + + previousParams, err := ss.LoadConsensusParams(rollbackHeight) + if err != nil { + return -1, nil, err + } + + valChangeHeight := invalidState.LastHeightValidatorsChanged + // this can only happen if the validator set changed since the last block + if valChangeHeight > rollbackHeight { + valChangeHeight = rollbackHeight + } + + paramsChangeHeight := invalidState.LastHeightConsensusParamsChanged + // this can only happen if params changed from the last block + if paramsChangeHeight > rollbackHeight { + paramsChangeHeight = rollbackHeight + } + + // build the new state from the old state and the prior block + rolledBackState := State{ + Version: Version{ + Consensus: version.Consensus{ + Block: version.BlockProtocol, + App: previousParams.Version.AppVersion, + }, + Software: version.TMVersion, + }, + // immutable fields + ChainID: invalidState.ChainID, + InitialHeight: invalidState.InitialHeight, + + LastBlockHeight: invalidState.LastBlockHeight - 1, + LastBlockID: rollbackBlock.Header.LastBlockID, + LastBlockTime: rollbackBlock.Header.Time, + + NextValidators: invalidState.Validators, + Validators: invalidState.LastValidators, + LastValidators: previousValidatorSet, + LastHeightValidatorsChanged: valChangeHeight, + + ConsensusParams: previousParams, + LastHeightConsensusParamsChanged: paramsChangeHeight, + + LastResultsHash: rollbackBlock.Header.LastResultsHash, + AppHash: rollbackBlock.Header.AppHash, + } + + // persist the new state. This overrides the invalid one. NOTE: this will also + // persist the validator set and consensus params over the existing structures, + // but both should be the same + if err := ss.Save(rolledBackState); err != nil { + return -1, nil, fmt.Errorf("failed to save rolled back state: %w", err) + } + + return rolledBackState.LastBlockHeight, rolledBackState.AppHash, nil +} diff --git a/internal/state/rollback_test.go b/internal/state/rollback_test.go new file mode 100644 index 000000000..ae5c8ee84 --- /dev/null +++ b/internal/state/rollback_test.go @@ -0,0 +1,146 @@ +package state_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + dbm "github.com/tendermint/tm-db" + + "github.com/tendermint/tendermint/internal/state" + "github.com/tendermint/tendermint/internal/state/mocks" + "github.com/tendermint/tendermint/internal/test/factory" + "github.com/tendermint/tendermint/types" + "github.com/tendermint/tendermint/version" +) + +func TestRollback(t *testing.T) { + stateStore := state.NewStore(dbm.NewMemDB()) + blockStore := &mocks.BlockStore{} + var ( + height int64 = 100 + appVersion uint64 = 10 + ) + + valSet, _ := factory.RandValidatorSet(5, 10) + + params := types.DefaultConsensusParams() + params.Version.AppVersion = appVersion + newParams := types.DefaultConsensusParams() + newParams.Block.MaxBytes = 10000 + + initialState := state.State{ + Version: state.Version{ + Consensus: version.Consensus{ + Block: version.BlockProtocol, + App: 10, + }, + Software: version.TMVersion, + }, + ChainID: factory.DefaultTestChainID, + InitialHeight: 10, + LastBlockID: factory.MakeBlockID(), + AppHash: factory.RandomHash(), + LastResultsHash: factory.RandomHash(), + LastBlockHeight: height, + LastValidators: valSet, + Validators: valSet.CopyIncrementProposerPriority(1), + NextValidators: valSet.CopyIncrementProposerPriority(2), + LastHeightValidatorsChanged: height + 1, + ConsensusParams: *params, + LastHeightConsensusParamsChanged: height + 1, + } + require.NoError(t, stateStore.Bootstrap(initialState)) + + height++ + block := &types.BlockMeta{ + Header: types.Header{ + Height: height, + AppHash: initialState.AppHash, + LastBlockID: initialState.LastBlockID, + LastResultsHash: initialState.LastResultsHash, + }, + } + blockStore.On("LoadBlockMeta", height).Return(block) + + appVersion++ + newParams.Version.AppVersion = appVersion + nextState := initialState.Copy() + nextState.LastBlockHeight = height + nextState.Version.Consensus.App = appVersion + nextState.LastBlockID = factory.MakeBlockID() + nextState.AppHash = factory.RandomHash() + nextState.LastValidators = initialState.Validators + nextState.Validators = initialState.NextValidators + nextState.NextValidators = initialState.NextValidators.CopyIncrementProposerPriority(1) + nextState.ConsensusParams = *newParams + nextState.LastHeightConsensusParamsChanged = height + 1 + nextState.LastHeightValidatorsChanged = height + 1 + + // update the state + require.NoError(t, stateStore.Save(nextState)) + + // rollback the state + rollbackHeight, rollbackHash, err := state.Rollback(blockStore, stateStore) + require.NoError(t, err) + require.EqualValues(t, int64(100), rollbackHeight) + require.EqualValues(t, initialState.AppHash, rollbackHash) + blockStore.AssertExpectations(t) + + // assert that we've recovered the prior state + loadedState, err := stateStore.Load() + require.NoError(t, err) + require.EqualValues(t, initialState, loadedState) +} + +func TestRollbackNoState(t *testing.T) { + stateStore := state.NewStore(dbm.NewMemDB()) + blockStore := &mocks.BlockStore{} + + _, _, err := state.Rollback(blockStore, stateStore) + require.Error(t, err) + require.Contains(t, err.Error(), "no state found") +} + +func TestRollbackNoBlocks(t *testing.T) { + stateStore := state.NewStore(dbm.NewMemDB()) + blockStore := &mocks.BlockStore{} + var ( + height int64 = 100 + appVersion uint64 = 10 + ) + + valSet, _ := factory.RandValidatorSet(5, 10) + + params := types.DefaultConsensusParams() + params.Version.AppVersion = appVersion + newParams := types.DefaultConsensusParams() + newParams.Block.MaxBytes = 10000 + + initialState := state.State{ + Version: state.Version{ + Consensus: version.Consensus{ + Block: version.BlockProtocol, + App: 10, + }, + Software: version.TMVersion, + }, + ChainID: factory.DefaultTestChainID, + InitialHeight: 10, + LastBlockID: factory.MakeBlockID(), + AppHash: factory.RandomHash(), + LastResultsHash: factory.RandomHash(), + LastBlockHeight: height, + LastValidators: valSet, + Validators: valSet.CopyIncrementProposerPriority(1), + NextValidators: valSet.CopyIncrementProposerPriority(2), + LastHeightValidatorsChanged: height + 1, + ConsensusParams: *params, + LastHeightConsensusParamsChanged: height + 1, + } + require.NoError(t, stateStore.Save(initialState)) + blockStore.On("LoadBlockMeta", height).Return(nil) + + _, _, err := state.Rollback(blockStore, stateStore) + require.Error(t, err) + require.Contains(t, err.Error(), "block at height 100 not found") +} diff --git a/internal/state/state.go b/internal/state/state.go index 5862162d1..ce9990cc2 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -64,6 +64,8 @@ func VersionFromProto(v tmstate.Version) Version { // Instead, use state.Copy() or updateState(...). // NOTE: not goroutine-safe. type State struct { + // FIXME: This can be removed as TMVersion is a constant, and version.Consensus should + // eventually be replaced by VersionParams in ConsensusParams Version Version // immutable diff --git a/test/e2e/app/app.go b/test/e2e/app/app.go index e7ba110e9..395c87024 100644 --- a/test/e2e/app/app.go +++ b/test/e2e/app/app.go @@ -11,6 +11,7 @@ import ( "github.com/tendermint/tendermint/abci/example/code" abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/libs/log" + "github.com/tendermint/tendermint/proto/tendermint/types" "github.com/tendermint/tendermint/version" ) @@ -116,6 +117,11 @@ func (app *Application) InitChain(req abci.RequestInitChain) abci.ResponseInitCh } resp := abci.ResponseInitChain{ AppHash: app.state.Hash, + ConsensusParams: &types.ConsensusParams{ + Version: &types.VersionParams{ + AppVersion: 1, + }, + }, } if resp.Validators, err = app.validatorUpdates(0); err != nil { panic(err)