@ -0,0 +1,69 @@ | |||||
package commands | |||||
import ( | |||||
"fmt" | |||||
"github.com/spf13/cobra" | |||||
dbm "github.com/tendermint/tm-db" | |||||
cfg "github.com/tendermint/tendermint/config" | |||||
"github.com/tendermint/tendermint/state" | |||||
"github.com/tendermint/tendermint/store" | |||||
) | |||||
var RollbackStateCmd = &cobra.Command{ | |||||
Use: "rollback", | |||||
Short: "rollback tendermint state by one height", | |||||
Long: ` | |||||
A state rollback is performed to recover from an incorrect application state transition, | |||||
when Tendermint has persisted an incorrect app hash and is thus unable to make | |||||
progress. Rollback overwrites a state at height n with the state at height n - 1. | |||||
The application should also roll back to height n - 1. No blocks are removed, so upon | |||||
restarting Tendermint the transactions in block n will be re-executed against the | |||||
application. | |||||
`, | |||||
RunE: func(cmd *cobra.Command, args []string) error { | |||||
height, hash, err := RollbackState(config) | |||||
if err != nil { | |||||
return fmt.Errorf("failed to rollback state: %w", err) | |||||
} | |||||
fmt.Printf("Rolled back state to height %d and hash %v", height, hash) | |||||
return nil | |||||
}, | |||||
} | |||||
// RollbackState takes the state at the current height n and overwrites it with the state | |||||
// at height n - 1. Note state here refers to tendermint state not application state. | |||||
// Returns the latest state height and app hash alongside an error if there was one. | |||||
func RollbackState(config *cfg.Config) (int64, []byte, error) { | |||||
// use the parsed config to load the block and state store | |||||
blockStore, stateStore, err := loadStateAndBlockStore(config) | |||||
if err != nil { | |||||
return -1, nil, err | |||||
} | |||||
// rollback the last state | |||||
return state.Rollback(blockStore, stateStore) | |||||
} | |||||
func loadStateAndBlockStore(config *cfg.Config) (*store.BlockStore, state.Store, error) { | |||||
dbType := dbm.BackendType(config.DBBackend) | |||||
// Get BlockStore | |||||
blockStoreDB, err := dbm.NewDB("blockstore", dbType, config.DBDir()) | |||||
if err != nil { | |||||
return nil, nil, err | |||||
} | |||||
blockStore := store.NewBlockStore(blockStoreDB) | |||||
// Get StateStore | |||||
stateDB, err := dbm.NewDB("state", dbType, config.DBDir()) | |||||
if err != nil { | |||||
return nil, nil, err | |||||
} | |||||
stateStore := state.NewStore(stateDB) | |||||
return blockStore, stateStore, nil | |||||
} |
@ -0,0 +1,194 @@ | |||||
// Code generated by mockery 2.9.0. DO NOT EDIT. | |||||
package mocks | |||||
import ( | |||||
mock "github.com/stretchr/testify/mock" | |||||
types "github.com/tendermint/tendermint/types" | |||||
) | |||||
// BlockStore is an autogenerated mock type for the BlockStore type | |||||
type BlockStore struct { | |||||
mock.Mock | |||||
} | |||||
// Base provides a mock function with given fields: | |||||
func (_m *BlockStore) Base() int64 { | |||||
ret := _m.Called() | |||||
var r0 int64 | |||||
if rf, ok := ret.Get(0).(func() int64); ok { | |||||
r0 = rf() | |||||
} else { | |||||
r0 = ret.Get(0).(int64) | |||||
} | |||||
return r0 | |||||
} | |||||
// Height provides a mock function with given fields: | |||||
func (_m *BlockStore) Height() int64 { | |||||
ret := _m.Called() | |||||
var r0 int64 | |||||
if rf, ok := ret.Get(0).(func() int64); ok { | |||||
r0 = rf() | |||||
} else { | |||||
r0 = ret.Get(0).(int64) | |||||
} | |||||
return r0 | |||||
} | |||||
// LoadBaseMeta provides a mock function with given fields: | |||||
func (_m *BlockStore) LoadBaseMeta() *types.BlockMeta { | |||||
ret := _m.Called() | |||||
var r0 *types.BlockMeta | |||||
if rf, ok := ret.Get(0).(func() *types.BlockMeta); ok { | |||||
r0 = rf() | |||||
} else { | |||||
if ret.Get(0) != nil { | |||||
r0 = ret.Get(0).(*types.BlockMeta) | |||||
} | |||||
} | |||||
return r0 | |||||
} | |||||
// LoadBlock provides a mock function with given fields: height | |||||
func (_m *BlockStore) LoadBlock(height int64) *types.Block { | |||||
ret := _m.Called(height) | |||||
var r0 *types.Block | |||||
if rf, ok := ret.Get(0).(func(int64) *types.Block); ok { | |||||
r0 = rf(height) | |||||
} else { | |||||
if ret.Get(0) != nil { | |||||
r0 = ret.Get(0).(*types.Block) | |||||
} | |||||
} | |||||
return r0 | |||||
} | |||||
// LoadBlockByHash provides a mock function with given fields: hash | |||||
func (_m *BlockStore) LoadBlockByHash(hash []byte) *types.Block { | |||||
ret := _m.Called(hash) | |||||
var r0 *types.Block | |||||
if rf, ok := ret.Get(0).(func([]byte) *types.Block); ok { | |||||
r0 = rf(hash) | |||||
} else { | |||||
if ret.Get(0) != nil { | |||||
r0 = ret.Get(0).(*types.Block) | |||||
} | |||||
} | |||||
return r0 | |||||
} | |||||
// LoadBlockCommit provides a mock function with given fields: height | |||||
func (_m *BlockStore) LoadBlockCommit(height int64) *types.Commit { | |||||
ret := _m.Called(height) | |||||
var r0 *types.Commit | |||||
if rf, ok := ret.Get(0).(func(int64) *types.Commit); ok { | |||||
r0 = rf(height) | |||||
} else { | |||||
if ret.Get(0) != nil { | |||||
r0 = ret.Get(0).(*types.Commit) | |||||
} | |||||
} | |||||
return r0 | |||||
} | |||||
// LoadBlockMeta provides a mock function with given fields: height | |||||
func (_m *BlockStore) LoadBlockMeta(height int64) *types.BlockMeta { | |||||
ret := _m.Called(height) | |||||
var r0 *types.BlockMeta | |||||
if rf, ok := ret.Get(0).(func(int64) *types.BlockMeta); ok { | |||||
r0 = rf(height) | |||||
} else { | |||||
if ret.Get(0) != nil { | |||||
r0 = ret.Get(0).(*types.BlockMeta) | |||||
} | |||||
} | |||||
return r0 | |||||
} | |||||
// LoadBlockPart provides a mock function with given fields: height, index | |||||
func (_m *BlockStore) LoadBlockPart(height int64, index int) *types.Part { | |||||
ret := _m.Called(height, index) | |||||
var r0 *types.Part | |||||
if rf, ok := ret.Get(0).(func(int64, int) *types.Part); ok { | |||||
r0 = rf(height, index) | |||||
} else { | |||||
if ret.Get(0) != nil { | |||||
r0 = ret.Get(0).(*types.Part) | |||||
} | |||||
} | |||||
return r0 | |||||
} | |||||
// LoadSeenCommit provides a mock function with given fields: height | |||||
func (_m *BlockStore) LoadSeenCommit(height int64) *types.Commit { | |||||
ret := _m.Called(height) | |||||
var r0 *types.Commit | |||||
if rf, ok := ret.Get(0).(func(int64) *types.Commit); ok { | |||||
r0 = rf(height) | |||||
} else { | |||||
if ret.Get(0) != nil { | |||||
r0 = ret.Get(0).(*types.Commit) | |||||
} | |||||
} | |||||
return r0 | |||||
} | |||||
// PruneBlocks provides a mock function with given fields: height | |||||
func (_m *BlockStore) PruneBlocks(height int64) (uint64, error) { | |||||
ret := _m.Called(height) | |||||
var r0 uint64 | |||||
if rf, ok := ret.Get(0).(func(int64) uint64); ok { | |||||
r0 = rf(height) | |||||
} else { | |||||
r0 = ret.Get(0).(uint64) | |||||
} | |||||
var r1 error | |||||
if rf, ok := ret.Get(1).(func(int64) error); ok { | |||||
r1 = rf(height) | |||||
} else { | |||||
r1 = ret.Error(1) | |||||
} | |||||
return r0, r1 | |||||
} | |||||
// SaveBlock provides a mock function with given fields: block, blockParts, seenCommit | |||||
func (_m *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) { | |||||
_m.Called(block, blockParts, seenCommit) | |||||
} | |||||
// Size provides a mock function with given fields: | |||||
func (_m *BlockStore) Size() int64 { | |||||
ret := _m.Called() | |||||
var r0 int64 | |||||
if rf, ok := ret.Get(0).(func() int64); ok { | |||||
r0 = rf() | |||||
} else { | |||||
r0 = ret.Get(0).(int64) | |||||
} | |||||
return r0 | |||||
} |
@ -0,0 +1,89 @@ | |||||
package state | |||||
import ( | |||||
"errors" | |||||
"fmt" | |||||
tmstate "github.com/tendermint/tendermint/proto/tendermint/state" | |||||
tmversion "github.com/tendermint/tendermint/proto/tendermint/version" | |||||
"github.com/tendermint/tendermint/version" | |||||
) | |||||
// Rollback overwrites the current Tendermint state (height n) with the most | |||||
// recent previous state (height n - 1). | |||||
// Note that this function does not affect application state. | |||||
func Rollback(bs BlockStore, ss Store) (int64, []byte, error) { | |||||
invalidState, err := ss.Load() | |||||
if err != nil { | |||||
return -1, nil, err | |||||
} | |||||
if invalidState.IsEmpty() { | |||||
return -1, nil, errors.New("no state found") | |||||
} | |||||
rollbackHeight := invalidState.LastBlockHeight | |||||
rollbackBlock := bs.LoadBlockMeta(rollbackHeight) | |||||
if rollbackBlock == nil { | |||||
return -1, nil, fmt.Errorf("block at height %d not found", rollbackHeight) | |||||
} | |||||
previousValidatorSet, err := ss.LoadValidators(rollbackHeight - 1) | |||||
if err != nil { | |||||
return -1, nil, err | |||||
} | |||||
previousParams, err := ss.LoadConsensusParams(rollbackHeight) | |||||
if err != nil { | |||||
return -1, nil, err | |||||
} | |||||
valChangeHeight := invalidState.LastHeightValidatorsChanged | |||||
// this can only happen if the validator set changed since the last block | |||||
if valChangeHeight > rollbackHeight { | |||||
valChangeHeight = rollbackHeight | |||||
} | |||||
paramsChangeHeight := invalidState.LastHeightConsensusParamsChanged | |||||
// this can only happen if params changed from the last block | |||||
if paramsChangeHeight > rollbackHeight { | |||||
paramsChangeHeight = rollbackHeight | |||||
} | |||||
// build the new state from the old state and the prior block | |||||
rolledBackState := State{ | |||||
Version: tmstate.Version{ | |||||
Consensus: tmversion.Consensus{ | |||||
Block: version.BlockProtocol, | |||||
App: previousParams.Version.AppVersion, | |||||
}, | |||||
Software: version.TMCoreSemVer, | |||||
}, | |||||
// immutable fields | |||||
ChainID: invalidState.ChainID, | |||||
InitialHeight: invalidState.InitialHeight, | |||||
LastBlockHeight: invalidState.LastBlockHeight - 1, | |||||
LastBlockID: rollbackBlock.Header.LastBlockID, | |||||
LastBlockTime: rollbackBlock.Header.Time, | |||||
NextValidators: invalidState.Validators, | |||||
Validators: invalidState.LastValidators, | |||||
LastValidators: previousValidatorSet, | |||||
LastHeightValidatorsChanged: valChangeHeight, | |||||
ConsensusParams: previousParams, | |||||
LastHeightConsensusParamsChanged: paramsChangeHeight, | |||||
LastResultsHash: rollbackBlock.Header.LastResultsHash, | |||||
AppHash: rollbackBlock.Header.AppHash, | |||||
} | |||||
// persist the new state. This overrides the invalid one. NOTE: this will also | |||||
// persist the validator set and consensus params over the existing structures, | |||||
// but both should be the same | |||||
if err := ss.Save(rolledBackState); err != nil { | |||||
return -1, nil, fmt.Errorf("failed to save rolled back state: %w", err) | |||||
} | |||||
return rolledBackState.LastBlockHeight, rolledBackState.AppHash, nil | |||||
} |
@ -0,0 +1,165 @@ | |||||
package state_test | |||||
import ( | |||||
"crypto/rand" | |||||
"testing" | |||||
"github.com/stretchr/testify/require" | |||||
dbm "github.com/tendermint/tm-db" | |||||
"github.com/tendermint/tendermint/crypto/tmhash" | |||||
tmstate "github.com/tendermint/tendermint/proto/tendermint/state" | |||||
tmversion "github.com/tendermint/tendermint/proto/tendermint/version" | |||||
"github.com/tendermint/tendermint/state" | |||||
"github.com/tendermint/tendermint/state/mocks" | |||||
"github.com/tendermint/tendermint/types" | |||||
"github.com/tendermint/tendermint/version" | |||||
) | |||||
func TestRollback(t *testing.T) { | |||||
stateStore := state.NewStore(dbm.NewMemDB()) | |||||
blockStore := &mocks.BlockStore{} | |||||
var ( | |||||
height int64 = 100 | |||||
appVersion uint64 = 10 | |||||
) | |||||
valSet, _ := types.RandValidatorSet(5, 10) | |||||
params := types.DefaultConsensusParams() | |||||
params.Version.AppVersion = appVersion | |||||
newParams := types.DefaultConsensusParams() | |||||
newParams.Block.MaxBytes = 10000 | |||||
initialState := state.State{ | |||||
Version: tmstate.Version{ | |||||
Consensus: tmversion.Consensus{ | |||||
Block: version.BlockProtocol, | |||||
App: 10, | |||||
}, | |||||
Software: version.TMCoreSemVer, | |||||
}, | |||||
ChainID: "test-chain", | |||||
InitialHeight: 10, | |||||
LastBlockID: makeBlockIDRandom(), | |||||
AppHash: tmhash.Sum([]byte("app_hash")), | |||||
LastResultsHash: tmhash.Sum([]byte("last_results_hash")), | |||||
LastBlockHeight: height, | |||||
LastValidators: valSet, | |||||
Validators: valSet.CopyIncrementProposerPriority(1), | |||||
NextValidators: valSet.CopyIncrementProposerPriority(2), | |||||
LastHeightValidatorsChanged: height + 1, | |||||
ConsensusParams: *params, | |||||
LastHeightConsensusParamsChanged: height + 1, | |||||
} | |||||
require.NoError(t, stateStore.Bootstrap(initialState)) | |||||
height++ | |||||
block := &types.BlockMeta{ | |||||
Header: types.Header{ | |||||
Height: height, | |||||
AppHash: initialState.AppHash, | |||||
LastBlockID: initialState.LastBlockID, | |||||
LastResultsHash: initialState.LastResultsHash, | |||||
}, | |||||
} | |||||
blockStore.On("LoadBlockMeta", height).Return(block) | |||||
appVersion++ | |||||
newParams.Version.AppVersion = appVersion | |||||
nextState := initialState.Copy() | |||||
nextState.LastBlockHeight = height | |||||
nextState.Version.Consensus.App = appVersion | |||||
nextState.LastBlockID = makeBlockIDRandom() | |||||
nextState.AppHash = tmhash.Sum([]byte("next_app_hash")) | |||||
nextState.LastValidators = initialState.Validators | |||||
nextState.Validators = initialState.NextValidators | |||||
nextState.NextValidators = initialState.NextValidators.CopyIncrementProposerPriority(1) | |||||
nextState.ConsensusParams = *newParams | |||||
nextState.LastHeightConsensusParamsChanged = height + 1 | |||||
nextState.LastHeightValidatorsChanged = height + 1 | |||||
// update the state | |||||
require.NoError(t, stateStore.Save(nextState)) | |||||
// rollback the state | |||||
rollbackHeight, rollbackHash, err := state.Rollback(blockStore, stateStore) | |||||
require.NoError(t, err) | |||||
require.EqualValues(t, int64(100), rollbackHeight) | |||||
require.EqualValues(t, initialState.AppHash, rollbackHash) | |||||
blockStore.AssertExpectations(t) | |||||
// assert that we've recovered the prior state | |||||
loadedState, err := stateStore.Load() | |||||
require.NoError(t, err) | |||||
require.EqualValues(t, initialState, loadedState) | |||||
} | |||||
func TestRollbackNoState(t *testing.T) { | |||||
stateStore := state.NewStore(dbm.NewMemDB()) | |||||
blockStore := &mocks.BlockStore{} | |||||
_, _, err := state.Rollback(blockStore, stateStore) | |||||
require.Error(t, err) | |||||
require.Contains(t, err.Error(), "no state found") | |||||
} | |||||
func TestRollbackNoBlocks(t *testing.T) { | |||||
stateStore := state.NewStore(dbm.NewMemDB()) | |||||
blockStore := &mocks.BlockStore{} | |||||
var ( | |||||
height int64 = 100 | |||||
appVersion uint64 = 10 | |||||
) | |||||
valSet, _ := types.RandValidatorSet(5, 10) | |||||
params := types.DefaultConsensusParams() | |||||
params.Version.AppVersion = appVersion | |||||
newParams := types.DefaultConsensusParams() | |||||
newParams.Block.MaxBytes = 10000 | |||||
initialState := state.State{ | |||||
Version: tmstate.Version{ | |||||
Consensus: tmversion.Consensus{ | |||||
Block: version.BlockProtocol, | |||||
App: 10, | |||||
}, | |||||
Software: version.TMCoreSemVer, | |||||
}, | |||||
ChainID: "test-chain", | |||||
InitialHeight: 10, | |||||
LastBlockID: makeBlockIDRandom(), | |||||
AppHash: tmhash.Sum([]byte("app_hash")), | |||||
LastResultsHash: tmhash.Sum([]byte("last_results_hash")), | |||||
LastBlockHeight: height, | |||||
LastValidators: valSet, | |||||
Validators: valSet.CopyIncrementProposerPriority(1), | |||||
NextValidators: valSet.CopyIncrementProposerPriority(2), | |||||
LastHeightValidatorsChanged: height + 1, | |||||
ConsensusParams: *params, | |||||
LastHeightConsensusParamsChanged: height + 1, | |||||
} | |||||
require.NoError(t, stateStore.Save(initialState)) | |||||
blockStore.On("LoadBlockMeta", height).Return(nil) | |||||
_, _, err := state.Rollback(blockStore, stateStore) | |||||
require.Error(t, err) | |||||
require.Contains(t, err.Error(), "block at height 100 not found") | |||||
} | |||||
func makeBlockIDRandom() types.BlockID { | |||||
var ( | |||||
blockHash = make([]byte, tmhash.Size) | |||||
partSetHash = make([]byte, tmhash.Size) | |||||
) | |||||
rand.Read(blockHash) //nolint: errcheck // ignore errcheck for read | |||||
rand.Read(partSetHash) //nolint: errcheck // ignore errcheck for read | |||||
return types.BlockID{ | |||||
Hash: blockHash, | |||||
PartSetHeader: types.PartSetHeader{ | |||||
Total: 123, | |||||
Hash: partSetHash, | |||||
}, | |||||
} | |||||
} |