From 4298bbcc4e25be78e3c4f21979d6aa01aede6e87 Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Fri, 3 Apr 2020 10:38:32 +0200 Subject: [PATCH] add support for block pruning via ABCI Commit response (#4588) * Added BlockStore.DeleteBlock() * Added initial block pruner prototype * wip * Added BlockStore.PruneBlocks() * Added consensus setting for block pruning * Added BlockStore base * Error on replay if base does not have blocks * Handle missing blocks when sending VoteSetMaj23Message * Error message tweak * Properly update blockstore state * Error message fix again * blockchain: ignore peer missing blocks * Added FIXME * Added test for block replay with truncated history * Handle peer base in blockchain reactor * Improved replay error handling * Added tests for Store.PruneBlocks() * Fix non-RPC handling of truncated block history * Panic on missing block meta in needProofBlock() * Updated changelog * Handle truncated block history in RPC layer * Added info about earliest block in /status RPC * Reorder height and base in blockchain reactor messages * Updated changelog * Fix tests * Appease linter * Minor review fixes * Non-empty BlockStores should always have base > 0 * Update code to assume base > 0 invariant * Added blockstore tests for pruning to 0 * Make sure we don't prune below the current base * Added BlockStore.Size() * config: added retain_blocks recommendations * Update v1 blockchain reactor to handle blockstore base * Added state database pruning * Propagate errors on missing validator sets * Comment tweaks * Improved error message Co-Authored-By: Anton Kaliaev * use ABCI field ResponseCommit.retain_height instead of retain-blocks config option * remove State.RetainHeight, return value instead * fix minor issues * rename pruneHeights() to pruneBlocks() * noop to fix GitHub borkage Co-authored-by: Anton Kaliaev --- CHANGELOG_PENDING.md | 8 + abci/example/kvstore/kvstore.go | 10 +- abci/types/types.pb.go | 341 ++++++++++++++++------------- abci/types/types.proto | 3 +- blockchain/v0/pool.go | 17 +- blockchain/v0/pool_test.go | 15 +- blockchain/v0/reactor.go | 46 +++- blockchain/v0/reactor_test.go | 2 +- blockchain/v1/peer.go | 6 +- blockchain/v1/peer_test.go | 16 +- blockchain/v1/pool.go | 11 +- blockchain/v1/pool_test.go | 66 ++++-- blockchain/v1/reactor.go | 35 ++- blockchain/v1/reactor_fsm.go | 7 +- blockchain/v1/reactor_test.go | 2 +- blockchain/v2/io.go | 9 +- blockchain/v2/processor_context.go | 2 +- blockchain/v2/reactor.go | 26 ++- blockchain/v2/reactor_test.go | 8 +- blockchain/v2/scheduler.go | 30 ++- blockchain/v2/scheduler_test.go | 92 ++++++-- config/config_test.go | 57 +++-- consensus/reactor.go | 30 +-- consensus/replay.go | 9 +- consensus/replay_test.go | 52 ++++- consensus/state.go | 44 +++- evidence/pool.go | 9 +- rpc/core/blocks.go | 29 +-- rpc/core/blocks_test.go | 51 +++-- rpc/core/consensus.go | 4 +- rpc/core/status.go | 30 ++- rpc/core/types/responses.go | 8 +- rpc/swagger/swagger.yaml | 12 + state/errors.go | 26 ++- state/execution.go | 29 +-- state/execution_test.go | 12 +- state/helpers_test.go | 2 +- state/services.go | 4 + state/store.go | 96 ++++++++ state/store_test.go | 115 ++++++++++ store/store.go | 117 +++++++++- store/store_test.go | 115 +++++++++- 42 files changed, 1209 insertions(+), 394 deletions(-) diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index d605e36d3..79d66bc05 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -1,5 +1,7 @@ ## v0.33.3 +- Nodes are no longer guaranteed to contain all blocks up to the latest height. The ABCI app can now control which blocks to retain through the ABCI field `ResponseCommit.retain_height`, all blocks and associated data below this height will be removed. + \*\* Special thanks to external contributors on this release: @@ -12,6 +14,8 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi - Apps +- P2P Protocol + - Go API - [rpc/client] [\#4628](https://github.com/tendermint/tendermint/pull/4628) Split out HTTP and local clients into `http` and `local` packages (@erikgrinaker). @@ -20,10 +24,14 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi ### FEATURES: +- [abci] Add `ResponseCommit.retain_height` field, which will automatically remove blocks below this height. +- [rpc] Add `/status` response fields for the earliest block available on the node - [rpc] [\#4611](https://github.com/tendermint/tendermint/pull/4611) Add `codespace` to `ResultBroadcastTx` (@whylee259) ### IMPROVEMENTS: +- [blockchain] Add `Base` to blockchain reactor P2P messages `StatusRequest` and `StatusResponse` +- [example/kvstore] Add `RetainBlocks` option to control block retention - [p2p] [\#4548](https://github.com/tendermint/tendermint/pull/4548) Add ban list to address book (@cmwaters) - [privval] \#4534 Add `error` as a return value on`GetPubKey()` - [Docker] \#4569 Default configuration added to docker image (you can still mount your own config the same way) (@greg-szabo) diff --git a/abci/example/kvstore/kvstore.go b/abci/example/kvstore/kvstore.go index ef9dcee92..42f00231f 100644 --- a/abci/example/kvstore/kvstore.go +++ b/abci/example/kvstore/kvstore.go @@ -64,7 +64,8 @@ var _ types.Application = (*Application)(nil) type Application struct { types.BaseApplication - state State + state State + RetainBlocks int64 // blocks to retain after commit (via ResponseCommit.RetainHeight) } func NewApplication() *Application { @@ -119,7 +120,12 @@ func (app *Application) Commit() types.ResponseCommit { app.state.AppHash = appHash app.state.Height++ saveState(app.state) - return types.ResponseCommit{Data: appHash} + + resp := types.ResponseCommit{Data: appHash} + if app.RetainBlocks > 0 && app.state.Height >= app.RetainBlocks { + resp.RetainHeight = app.state.Height - app.RetainBlocks + 1 + } + return resp } // Returns an associated value or nil if missing. diff --git a/abci/types/types.pb.go b/abci/types/types.pb.go index 6c18cfbd1..51ff4aedd 100644 --- a/abci/types/types.pb.go +++ b/abci/types/types.pb.go @@ -1865,6 +1865,7 @@ func (m *ResponseEndBlock) GetEvents() []Event { type ResponseCommit struct { // reserve 1 Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` + RetainHeight int64 `protobuf:"varint,3,opt,name=retain_height,json=retainHeight,proto3" json:"retain_height,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1910,6 +1911,13 @@ func (m *ResponseCommit) GetData() []byte { return nil } +func (m *ResponseCommit) GetRetainHeight() int64 { + if m != nil { + return m.RetainHeight + } + return 0 +} + // ConsensusParams contains all consensus-relevant parameters // that can be adjusted by the abci app type ConsensusParams struct { @@ -2960,155 +2968,156 @@ func init() { proto.RegisterFile("abci/types/types.proto", fileDescriptor_9f1eaa func init() { golang_proto.RegisterFile("abci/types/types.proto", fileDescriptor_9f1eaa49c51fa1ac) } var fileDescriptor_9f1eaa49c51fa1ac = []byte{ - // 2370 bytes of a gzipped FileDescriptorProto + // 2386 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x59, 0x4d, 0x90, 0x1b, 0x47, - 0x15, 0xde, 0xd1, 0x6a, 0x57, 0xd2, 0xd3, 0xee, 0x4a, 0x69, 0x3b, 0x89, 0x22, 0x92, 0x5d, 0xd7, - 0xf8, 0x6f, 0x9d, 0x04, 0x6d, 0x58, 0x2a, 0x54, 0x8c, 0x5d, 0xa1, 0x56, 0x6b, 0x07, 0xa9, 0x62, - 0x3b, 0x9b, 0xb1, 0xbd, 0x18, 0xa8, 0xca, 0x54, 0x4b, 0xd3, 0x96, 0xa6, 0x56, 0x9a, 0x99, 0xcc, - 0xb4, 0x64, 0x89, 0xe2, 0x4e, 0x51, 0xc5, 0x81, 0x0b, 0x55, 0x5c, 0xb8, 0x73, 0xe4, 0xc0, 0x21, - 0x47, 0x8e, 0x39, 0x70, 0xe0, 0xc0, 0xd9, 0xc0, 0xc2, 0x89, 0xca, 0x91, 0xa2, 0x38, 0x52, 0xfd, - 0xba, 0xe7, 0x4f, 0x2b, 0xad, 0xc6, 0xc1, 0x37, 0x2e, 0xd2, 0x74, 0xf7, 0x7b, 0xaf, 0xbb, 0x5f, - 0xbf, 0x7e, 0xdf, 0x7b, 0xaf, 0xe1, 0x35, 0xda, 0xe9, 0xda, 0x7b, 0x7c, 0xea, 0xb1, 0x40, 0xfe, - 0x36, 0x3c, 0xdf, 0xe5, 0x2e, 0x79, 0x95, 0x33, 0xc7, 0x62, 0xfe, 0xd0, 0x76, 0x78, 0x43, 0x90, - 0x34, 0x70, 0xb0, 0x7e, 0x8d, 0xf7, 0x6d, 0xdf, 0x32, 0x3d, 0xea, 0xf3, 0xe9, 0x1e, 0x52, 0xee, - 0xf5, 0xdc, 0x9e, 0x1b, 0x7f, 0x49, 0xf6, 0x7a, 0xbd, 0xeb, 0x4f, 0x3d, 0xee, 0xee, 0x0d, 0x99, - 0x7f, 0x32, 0x60, 0xea, 0x4f, 0x8d, 0x5d, 0x18, 0xd8, 0x9d, 0x60, 0xef, 0x64, 0x9c, 0x9c, 0xaf, - 0xbe, 0xd3, 0x73, 0xdd, 0xde, 0x80, 0x49, 0x99, 0x9d, 0xd1, 0xd3, 0x3d, 0x6e, 0x0f, 0x59, 0xc0, - 0xe9, 0xd0, 0x53, 0x04, 0xdb, 0xb3, 0x04, 0xd6, 0xc8, 0xa7, 0xdc, 0x76, 0x1d, 0x39, 0xae, 0xff, - 0x7b, 0x0d, 0x0a, 0x06, 0xfb, 0x7c, 0xc4, 0x02, 0x4e, 0x3e, 0x80, 0x3c, 0xeb, 0xf6, 0xdd, 0x5a, - 0xee, 0x92, 0xb6, 0x5b, 0xde, 0xd7, 0x1b, 0x73, 0xf7, 0xd2, 0x50, 0xd4, 0x77, 0xbb, 0x7d, 0xb7, - 0xb5, 0x62, 0x20, 0x07, 0xb9, 0x05, 0x6b, 0x4f, 0x07, 0xa3, 0xa0, 0x5f, 0x5b, 0x45, 0xd6, 0xcb, - 0xe7, 0xb3, 0x7e, 0x24, 0x48, 0x5b, 0x2b, 0x86, 0xe4, 0x11, 0xd3, 0xda, 0xce, 0x53, 0xb7, 0x96, - 0xcf, 0x32, 0x6d, 0xdb, 0x79, 0x8a, 0xd3, 0x0a, 0x0e, 0xd2, 0x02, 0x08, 0x18, 0x37, 0x5d, 0x4f, - 0x6c, 0xa8, 0xb6, 0x86, 0xfc, 0xd7, 0xcf, 0xe7, 0x7f, 0xc8, 0xf8, 0x27, 0x48, 0xde, 0x5a, 0x31, - 0x4a, 0x41, 0xd8, 0x10, 0x92, 0x6c, 0xc7, 0xe6, 0x66, 0xb7, 0x4f, 0x6d, 0xa7, 0xb6, 0x9e, 0x45, - 0x52, 0xdb, 0xb1, 0xf9, 0xa1, 0x20, 0x17, 0x92, 0xec, 0xb0, 0x21, 0x54, 0xf1, 0xf9, 0x88, 0xf9, - 0xd3, 0x5a, 0x21, 0x8b, 0x2a, 0x3e, 0x15, 0xa4, 0x42, 0x15, 0xc8, 0x43, 0x3e, 0x86, 0x72, 0x87, - 0xf5, 0x6c, 0xc7, 0xec, 0x0c, 0xdc, 0xee, 0x49, 0xad, 0x88, 0x22, 0x76, 0xcf, 0x17, 0xd1, 0x14, - 0x0c, 0x4d, 0x41, 0xdf, 0x5a, 0x31, 0xa0, 0x13, 0xb5, 0x48, 0x13, 0x8a, 0xdd, 0x3e, 0xeb, 0x9e, - 0x98, 0x7c, 0x52, 0x2b, 0xa1, 0xa4, 0xab, 0xe7, 0x4b, 0x3a, 0x14, 0xd4, 0x8f, 0x26, 0xad, 0x15, - 0xa3, 0xd0, 0x95, 0x9f, 0x42, 0x2f, 0x16, 0x1b, 0xd8, 0x63, 0xe6, 0x0b, 0x29, 0x17, 0xb2, 0xe8, - 0xe5, 0x8e, 0xa4, 0x47, 0x39, 0x25, 0x2b, 0x6c, 0x90, 0xbb, 0x50, 0x62, 0x8e, 0xa5, 0x36, 0x56, - 0x46, 0x41, 0xd7, 0x96, 0x58, 0x98, 0x63, 0x85, 0xdb, 0x2a, 0x32, 0xf5, 0x4d, 0x3e, 0x84, 0xf5, - 0xae, 0x3b, 0x1c, 0xda, 0xbc, 0xb6, 0x81, 0x32, 0xae, 0x2c, 0xd9, 0x12, 0xd2, 0xb6, 0x56, 0x0c, - 0xc5, 0xd5, 0x2c, 0xc0, 0xda, 0x98, 0x0e, 0x46, 0x4c, 0xbf, 0x0e, 0xe5, 0x84, 0x25, 0x93, 0x1a, - 0x14, 0x86, 0x2c, 0x08, 0x68, 0x8f, 0xd5, 0xb4, 0x4b, 0xda, 0x6e, 0xc9, 0x08, 0x9b, 0xfa, 0x16, - 0x6c, 0x24, 0xed, 0x56, 0x1f, 0x46, 0x8c, 0xc2, 0x16, 0x05, 0xe3, 0x98, 0xf9, 0x81, 0x30, 0x40, - 0xc5, 0xa8, 0x9a, 0xe4, 0x32, 0x6c, 0xe2, 0x6e, 0xcd, 0x70, 0x5c, 0xdc, 0xab, 0xbc, 0xb1, 0x81, - 0x9d, 0xc7, 0x8a, 0x68, 0x07, 0xca, 0xde, 0xbe, 0x17, 0x91, 0xac, 0x22, 0x09, 0x78, 0xfb, 0x9e, - 0x22, 0xd0, 0xbf, 0x0b, 0xd5, 0x59, 0xd3, 0x25, 0x55, 0x58, 0x3d, 0x61, 0x53, 0x35, 0x9f, 0xf8, - 0x24, 0x17, 0xd5, 0xb6, 0x70, 0x8e, 0x92, 0xa1, 0xf6, 0xf8, 0xbb, 0x5c, 0xc4, 0x1c, 0x59, 0xab, - 0xb8, 0x6e, 0xc2, 0x49, 0x20, 0x77, 0x79, 0xbf, 0xde, 0x90, 0x0e, 0xa2, 0x11, 0x3a, 0x88, 0xc6, - 0xa3, 0xd0, 0x83, 0x34, 0x8b, 0x5f, 0x3e, 0xdf, 0x59, 0xf9, 0xe5, 0x5f, 0x76, 0x34, 0x03, 0x39, - 0xc8, 0x1b, 0xc2, 0xa0, 0xa8, 0xed, 0x98, 0xb6, 0xa5, 0xe6, 0x29, 0x60, 0xbb, 0x6d, 0x91, 0x4f, - 0xa1, 0xda, 0x75, 0x9d, 0x80, 0x39, 0xc1, 0x28, 0x10, 0x6e, 0x8e, 0x0e, 0x03, 0xe5, 0x0b, 0x16, - 0x1d, 0xf2, 0x61, 0x48, 0x7e, 0x84, 0xd4, 0x46, 0xa5, 0x9b, 0xee, 0x20, 0xf7, 0x00, 0xc6, 0x74, - 0x60, 0x5b, 0x94, 0xbb, 0x7e, 0x50, 0xcb, 0x5f, 0x5a, 0x3d, 0x47, 0xd8, 0x71, 0x48, 0xf8, 0xd8, - 0xb3, 0x28, 0x67, 0xcd, 0xbc, 0x58, 0xb9, 0x91, 0xe0, 0x27, 0xd7, 0xa0, 0x42, 0x3d, 0xcf, 0x0c, - 0x38, 0xe5, 0xcc, 0xec, 0x4c, 0x39, 0x0b, 0xd0, 0x5f, 0x6c, 0x18, 0x9b, 0xd4, 0xf3, 0x1e, 0x8a, - 0xde, 0xa6, 0xe8, 0xd4, 0xad, 0xe8, 0xb4, 0xf1, 0x6a, 0x12, 0x02, 0x79, 0x8b, 0x72, 0x8a, 0xda, - 0xda, 0x30, 0xf0, 0x5b, 0xf4, 0x79, 0x94, 0xf7, 0x95, 0x0e, 0xf0, 0x9b, 0xbc, 0x06, 0xeb, 0x7d, - 0x66, 0xf7, 0xfa, 0x1c, 0xb7, 0xbd, 0x6a, 0xa8, 0x96, 0x38, 0x18, 0xcf, 0x77, 0xc7, 0x0c, 0xbd, - 0x5b, 0xd1, 0x90, 0x0d, 0xfd, 0x57, 0x39, 0x78, 0xe5, 0xcc, 0xf5, 0x15, 0x72, 0xfb, 0x34, 0xe8, - 0x87, 0x73, 0x89, 0x6f, 0x72, 0x4b, 0xc8, 0xa5, 0x16, 0xf3, 0x95, 0x57, 0x7e, 0x6b, 0x81, 0x06, - 0x5a, 0x48, 0xa4, 0x36, 0xae, 0x58, 0xc8, 0x63, 0xa8, 0x0e, 0x68, 0xc0, 0x4d, 0x69, 0xfb, 0x26, - 0x7a, 0xd9, 0xd5, 0x73, 0x3d, 0xc1, 0x3d, 0x1a, 0xde, 0x19, 0x61, 0xdc, 0x4a, 0xdc, 0xd6, 0x20, - 0xd5, 0x4b, 0x9e, 0xc0, 0xc5, 0xce, 0xf4, 0x27, 0xd4, 0xe1, 0xb6, 0xc3, 0xcc, 0x33, 0x67, 0xb4, - 0xb3, 0x40, 0xf4, 0xdd, 0xb1, 0x6d, 0x31, 0xa7, 0x1b, 0x1e, 0xce, 0x85, 0x48, 0x44, 0x74, 0x78, - 0x81, 0xfe, 0x04, 0xb6, 0xd2, 0xbe, 0x88, 0x6c, 0x41, 0x8e, 0x4f, 0x94, 0x46, 0x72, 0x7c, 0x42, - 0xbe, 0x03, 0x79, 0x21, 0x0e, 0xb5, 0xb1, 0xb5, 0x10, 0x2c, 0x14, 0xf7, 0xa3, 0xa9, 0xc7, 0x0c, - 0xa4, 0xd7, 0xf5, 0xe8, 0x26, 0x44, 0xfe, 0x69, 0x56, 0xb6, 0x7e, 0x03, 0x2a, 0x33, 0xae, 0x27, - 0x71, 0xac, 0x5a, 0xf2, 0x58, 0xf5, 0x0a, 0x6c, 0xa6, 0x3c, 0x8c, 0xfe, 0xc7, 0x75, 0x28, 0x1a, - 0x2c, 0xf0, 0x84, 0x11, 0x93, 0x16, 0x94, 0xd8, 0xa4, 0xcb, 0x24, 0x2c, 0x69, 0x4b, 0x9c, 0xb8, - 0xe4, 0xb9, 0x1b, 0xd2, 0x0b, 0xaf, 0x19, 0x31, 0x93, 0x9b, 0x29, 0x48, 0xbe, 0xbc, 0x4c, 0x48, - 0x12, 0x93, 0x6f, 0xa7, 0x31, 0xf9, 0xca, 0x12, 0xde, 0x19, 0x50, 0xbe, 0x99, 0x02, 0xe5, 0x65, - 0x13, 0xa7, 0x50, 0xb9, 0x3d, 0x07, 0x95, 0x97, 0x6d, 0x7f, 0x01, 0x2c, 0xb7, 0xe7, 0xc0, 0xf2, - 0xee, 0xd2, 0xb5, 0xcc, 0xc5, 0xe5, 0xdb, 0x69, 0x5c, 0x5e, 0xa6, 0x8e, 0x19, 0x60, 0xbe, 0x37, - 0x0f, 0x98, 0x6f, 0x2c, 0x91, 0xb1, 0x10, 0x99, 0x0f, 0xcf, 0x20, 0xf3, 0xb5, 0x25, 0xa2, 0xe6, - 0x40, 0x73, 0x3b, 0x05, 0xcd, 0x90, 0x49, 0x37, 0x0b, 0xb0, 0xf9, 0xa3, 0xb3, 0xd8, 0x7c, 0x7d, - 0x99, 0xa9, 0xcd, 0x03, 0xe7, 0xef, 0xcd, 0x80, 0xf3, 0xd5, 0x65, 0xbb, 0x5a, 0x88, 0xce, 0x37, - 0x84, 0x7f, 0x9c, 0xb9, 0x19, 0xc2, 0x97, 0x32, 0xdf, 0x77, 0x7d, 0x05, 0x7c, 0xb2, 0xa1, 0xef, - 0x0a, 0x8f, 0x1d, 0xdb, 0xff, 0x39, 0x48, 0x8e, 0x97, 0x36, 0x61, 0xed, 0xfa, 0x17, 0x5a, 0xcc, - 0x8b, 0x9e, 0x2d, 0xe9, 0xed, 0x4b, 0xca, 0xdb, 0x27, 0x00, 0x3e, 0x97, 0x06, 0xf8, 0x1d, 0x28, - 0x0b, 0x4c, 0x99, 0xc1, 0x6e, 0xea, 0x85, 0xd8, 0x4d, 0xde, 0x86, 0x57, 0xd0, 0xff, 0xca, 0x30, - 0x40, 0x39, 0x92, 0x3c, 0x3a, 0x92, 0x8a, 0x18, 0x90, 0x1a, 0x94, 0x40, 0xf1, 0x4d, 0xb8, 0x90, - 0xa0, 0x15, 0x72, 0x11, 0x0b, 0x24, 0x48, 0x55, 0x23, 0xea, 0x03, 0xcf, 0x6b, 0xd1, 0xa0, 0xaf, - 0xdf, 0x8f, 0x15, 0x14, 0xc7, 0x05, 0x04, 0xf2, 0x5d, 0xd7, 0x92, 0xfb, 0xde, 0x34, 0xf0, 0x5b, - 0xc4, 0x0a, 0x03, 0xb7, 0x87, 0x8b, 0x2b, 0x19, 0xe2, 0x53, 0x50, 0x45, 0x57, 0xbb, 0x24, 0xef, - 0xac, 0xfe, 0x7b, 0x2d, 0x96, 0x17, 0x87, 0x0a, 0xf3, 0x50, 0x5d, 0x7b, 0x99, 0xa8, 0x9e, 0xfb, - 0xdf, 0x50, 0x5d, 0xff, 0x97, 0x16, 0x1f, 0x69, 0x84, 0xd7, 0x5f, 0x4f, 0x05, 0xc2, 0xba, 0x6c, - 0xc7, 0x62, 0x13, 0x54, 0xf9, 0xaa, 0x21, 0x1b, 0x61, 0xa8, 0xb5, 0x8e, 0xc7, 0x90, 0x0e, 0xb5, - 0x0a, 0xd8, 0x27, 0x1b, 0xe4, 0x7d, 0xc4, 0x79, 0xf7, 0xa9, 0x72, 0x0d, 0x29, 0x10, 0x94, 0x49, - 0x5d, 0x43, 0x65, 0x73, 0x47, 0x82, 0xcc, 0x90, 0xd4, 0x09, 0x7c, 0x29, 0xa5, 0xc2, 0x86, 0x37, - 0xa1, 0x24, 0x96, 0x1e, 0x78, 0xb4, 0xcb, 0xf0, 0x6e, 0x97, 0x8c, 0xb8, 0x43, 0xb7, 0x80, 0x9c, - 0xf5, 0x31, 0xe4, 0x01, 0xac, 0xb3, 0x31, 0x73, 0xb8, 0x38, 0x23, 0xa1, 0xd6, 0x37, 0x17, 0x02, - 0x31, 0x73, 0x78, 0xb3, 0x26, 0x94, 0xf9, 0xcf, 0xe7, 0x3b, 0x55, 0xc9, 0xf3, 0xae, 0x3b, 0xb4, - 0x39, 0x1b, 0x7a, 0x7c, 0x6a, 0x28, 0x29, 0xfa, 0xcf, 0x72, 0x02, 0x0f, 0x53, 0xfe, 0x67, 0xae, - 0x7a, 0xc3, 0x4b, 0x93, 0x4b, 0x84, 0x48, 0xd9, 0x54, 0xfe, 0x16, 0x40, 0x8f, 0x06, 0xe6, 0x33, - 0xea, 0x70, 0x66, 0x29, 0xbd, 0x97, 0x7a, 0x34, 0xf8, 0x01, 0x76, 0x88, 0x78, 0x53, 0x0c, 0x8f, - 0x02, 0x66, 0xe1, 0x01, 0xac, 0x1a, 0x85, 0x1e, 0x0d, 0x1e, 0x07, 0xcc, 0x4a, 0xec, 0xb5, 0xf0, - 0x32, 0xf6, 0x9a, 0xd6, 0x77, 0x71, 0x56, 0xdf, 0x3f, 0xcf, 0xc5, 0xb7, 0x23, 0x0e, 0x1f, 0xfe, - 0x3f, 0x75, 0xf1, 0x1b, 0xcc, 0x29, 0xd2, 0x20, 0x40, 0x7e, 0x08, 0xaf, 0x44, 0xb7, 0xd2, 0x1c, - 0xe1, 0x6d, 0x0d, 0xad, 0xf0, 0xc5, 0x2e, 0x77, 0x75, 0x9c, 0xee, 0x0e, 0xc8, 0x67, 0xf0, 0xfa, - 0x8c, 0x0f, 0x8a, 0x26, 0xc8, 0xbd, 0x90, 0x2b, 0x7a, 0x35, 0xed, 0x8a, 0x42, 0xf9, 0xb1, 0xf6, - 0x56, 0x5f, 0xca, 0xad, 0xb9, 0x22, 0x42, 0xd8, 0x24, 0xbc, 0xcd, 0xb3, 0x09, 0xfd, 0xcf, 0x1a, - 0x54, 0x66, 0x16, 0x48, 0x3e, 0x80, 0x35, 0x89, 0xc0, 0xda, 0xb9, 0x85, 0x10, 0xd4, 0xb8, 0xda, - 0x93, 0x64, 0x20, 0x07, 0x50, 0x64, 0x2a, 0xba, 0x56, 0x4a, 0xb9, 0xba, 0x24, 0x08, 0x57, 0xfc, - 0x11, 0x1b, 0xb9, 0x03, 0xa5, 0x48, 0xf5, 0x4b, 0x32, 0xb7, 0xe8, 0xe4, 0x94, 0x90, 0x98, 0x51, - 0x3f, 0x84, 0x72, 0x62, 0x79, 0xe4, 0x1b, 0x50, 0x1a, 0xd2, 0x89, 0x4a, 0xb7, 0x64, 0x00, 0x5d, - 0x1c, 0xd2, 0x09, 0x66, 0x5a, 0xe4, 0x75, 0x28, 0x88, 0xc1, 0x1e, 0x95, 0x07, 0xb9, 0x6a, 0xac, - 0x0f, 0xe9, 0xe4, 0xfb, 0x34, 0xd0, 0x7f, 0xa1, 0xc1, 0x56, 0x7a, 0x9d, 0xe4, 0x1d, 0x20, 0x82, - 0x96, 0xf6, 0x98, 0xe9, 0x8c, 0x86, 0x12, 0x23, 0x43, 0x89, 0x95, 0x21, 0x9d, 0x1c, 0xf4, 0xd8, - 0x83, 0xd1, 0x10, 0xa7, 0x0e, 0xc8, 0x7d, 0xa8, 0x86, 0xc4, 0x61, 0xb1, 0x4b, 0x69, 0xe5, 0x8d, - 0x33, 0xc9, 0xee, 0x1d, 0x45, 0x20, 0x73, 0xdd, 0x5f, 0x8b, 0x5c, 0x77, 0x4b, 0xca, 0x0b, 0x47, - 0xf4, 0xf7, 0xa1, 0x32, 0xb3, 0x63, 0xa2, 0xc3, 0xa6, 0x37, 0xea, 0x98, 0x27, 0x6c, 0x6a, 0xa2, - 0x4a, 0xd0, 0xd4, 0x4b, 0x46, 0xd9, 0x1b, 0x75, 0x3e, 0x66, 0x53, 0x91, 0x75, 0x04, 0x7a, 0x17, - 0xb6, 0xd2, 0xc9, 0x94, 0x00, 0x0e, 0xdf, 0x1d, 0x39, 0x16, 0xae, 0x7b, 0xcd, 0x90, 0x0d, 0x72, - 0x0b, 0xd6, 0xc6, 0xae, 0xb4, 0xe6, 0xf3, 0xb2, 0xa7, 0x63, 0x97, 0xb3, 0x44, 0x4a, 0x26, 0x79, - 0xf4, 0x00, 0xd6, 0xd0, 0x2e, 0x85, 0x8d, 0x61, 0x5a, 0xa4, 0x02, 0x17, 0xf1, 0x4d, 0x8e, 0x01, - 0x28, 0xe7, 0xbe, 0xdd, 0x19, 0xc5, 0xe2, 0x6b, 0x49, 0xf1, 0x03, 0xbb, 0x13, 0x34, 0x4e, 0xc6, - 0x8d, 0x23, 0x6a, 0xfb, 0xcd, 0x37, 0x95, 0x65, 0x5f, 0x8c, 0x79, 0x12, 0xd6, 0x9d, 0x90, 0xa4, - 0x7f, 0x95, 0x87, 0x75, 0x99, 0x6e, 0x92, 0x0f, 0xd3, 0xc5, 0x8f, 0xf2, 0xfe, 0xf6, 0xa2, 0xe5, - 0x4b, 0x2a, 0xb5, 0xfa, 0x28, 0x82, 0xba, 0x36, 0x5b, 0x51, 0x68, 0x96, 0x4f, 0x9f, 0xef, 0x14, - 0x30, 0xfa, 0x68, 0xdf, 0x89, 0xcb, 0x0b, 0x8b, 0xb2, 0xeb, 0xb0, 0x96, 0x91, 0x7f, 0xe1, 0x5a, - 0x46, 0x0b, 0x36, 0x13, 0xe1, 0x96, 0x6d, 0xa9, 0x3c, 0x65, 0xfb, 0xbc, 0x4b, 0xd7, 0xbe, 0xa3, - 0xd6, 0x5f, 0x8e, 0xc2, 0xb1, 0xb6, 0x45, 0x76, 0xd3, 0x49, 0x36, 0x46, 0x6d, 0x32, 0x5c, 0x48, - 0xe4, 0xcd, 0x22, 0x66, 0x13, 0xd7, 0x41, 0x5c, 0x7e, 0x49, 0x22, 0xa3, 0x87, 0xa2, 0xe8, 0xc0, - 0xc1, 0xeb, 0x50, 0x89, 0x03, 0x1b, 0x49, 0x52, 0x94, 0x52, 0xe2, 0x6e, 0x24, 0x7c, 0x0f, 0x2e, - 0x3a, 0x6c, 0xc2, 0xcd, 0x59, 0xea, 0x12, 0x52, 0x13, 0x31, 0x76, 0x9c, 0xe6, 0xb8, 0x0a, 0x5b, - 0xb1, 0x0b, 0x45, 0x5a, 0x90, 0xa5, 0x8f, 0xa8, 0x17, 0xc9, 0xde, 0x80, 0x62, 0x14, 0x76, 0x96, - 0x91, 0xa0, 0x40, 0x65, 0xb4, 0x19, 0x05, 0xb2, 0x3e, 0x0b, 0x46, 0x03, 0xae, 0x84, 0x6c, 0x20, - 0x0d, 0x06, 0xb2, 0x86, 0xec, 0x47, 0xda, 0xcb, 0xb0, 0x19, 0x7a, 0x15, 0x49, 0xb7, 0x89, 0x74, - 0x1b, 0x61, 0x27, 0x12, 0xdd, 0x80, 0xaa, 0xe7, 0xbb, 0x9e, 0x1b, 0x30, 0xdf, 0xa4, 0x96, 0xe5, - 0xb3, 0x20, 0xa8, 0x6d, 0x49, 0x79, 0x61, 0xff, 0x81, 0xec, 0xd6, 0xbf, 0x05, 0x85, 0x30, 0x9e, - 0xbe, 0x08, 0x6b, 0xcd, 0xc8, 0x43, 0xe6, 0x0d, 0xd9, 0x10, 0xf8, 0x7a, 0xe0, 0x79, 0xaa, 0xba, - 0x26, 0x3e, 0xf5, 0x01, 0x14, 0xd4, 0x81, 0xcd, 0xad, 0xa9, 0xdc, 0x87, 0x0d, 0x8f, 0xfa, 0x62, - 0x1b, 0xc9, 0xca, 0xca, 0xa2, 0x8c, 0xf0, 0x88, 0xfa, 0xfc, 0x21, 0xe3, 0xa9, 0x02, 0x4b, 0x19, - 0xf9, 0x65, 0x97, 0x7e, 0x13, 0x36, 0x53, 0x34, 0x62, 0x99, 0xdc, 0xe5, 0x74, 0x10, 0x5e, 0x74, - 0x6c, 0x44, 0x2b, 0xc9, 0xc5, 0x2b, 0xd1, 0x6f, 0x41, 0x29, 0x3a, 0x2b, 0x91, 0x68, 0x84, 0xaa, - 0xd0, 0x94, 0xfa, 0x65, 0x13, 0x8b, 0x48, 0xee, 0x33, 0xe6, 0x2b, 0xeb, 0x97, 0x0d, 0x9d, 0x25, - 0x1c, 0x93, 0x44, 0x33, 0x72, 0x1b, 0x0a, 0xca, 0x31, 0xa9, 0xfb, 0xb8, 0xa8, 0x5c, 0x74, 0x84, - 0x9e, 0x2a, 0x2c, 0x17, 0x49, 0xbf, 0x15, 0x4f, 0x93, 0x4b, 0x4e, 0xf3, 0x53, 0x28, 0x86, 0xce, - 0x27, 0x8d, 0x12, 0x72, 0x86, 0x4b, 0xcb, 0x50, 0x42, 0x4d, 0x12, 0x33, 0x0a, 0x6b, 0x0a, 0xec, - 0x9e, 0xc3, 0x2c, 0x33, 0xbe, 0x82, 0x38, 0x67, 0xd1, 0xa8, 0xc8, 0x81, 0x7b, 0xe1, 0xfd, 0xd2, - 0xdf, 0x83, 0x75, 0xb9, 0xd6, 0xb9, 0x2e, 0x6e, 0x1e, 0xb4, 0xfe, 0x43, 0x83, 0x62, 0x08, 0x1f, - 0x73, 0x99, 0x52, 0x9b, 0xc8, 0x7d, 0xdd, 0x4d, 0xbc, 0x7c, 0x97, 0xf4, 0x2e, 0x10, 0xb4, 0x14, - 0x73, 0xec, 0x72, 0xdb, 0xe9, 0x99, 0xf2, 0x2c, 0x64, 0x24, 0x58, 0xc5, 0x91, 0x63, 0x1c, 0x38, - 0x12, 0xfd, 0x6f, 0x5f, 0x86, 0x72, 0xa2, 0xca, 0x45, 0x0a, 0xb0, 0xfa, 0x80, 0x3d, 0xab, 0xae, - 0x90, 0x32, 0x14, 0x0c, 0x86, 0x35, 0x82, 0xaa, 0xb6, 0xff, 0x55, 0x01, 0x2a, 0x07, 0xcd, 0xc3, - 0xf6, 0x81, 0xe7, 0x0d, 0xec, 0x2e, 0xe2, 0x19, 0xf9, 0x04, 0xf2, 0x98, 0x27, 0x67, 0x78, 0xdf, - 0xa9, 0x67, 0x29, 0x38, 0x11, 0x03, 0xd6, 0x30, 0x9d, 0x26, 0x59, 0x9e, 0x7d, 0xea, 0x99, 0xea, - 0x50, 0x62, 0x91, 0x68, 0x70, 0x19, 0x5e, 0x83, 0xea, 0x59, 0x8a, 0x53, 0xe4, 0x33, 0x28, 0xc5, - 0x79, 0x72, 0xd6, 0x37, 0xa2, 0x7a, 0xe6, 0xb2, 0x95, 0x90, 0x1f, 0x67, 0x06, 0x59, 0x5f, 0x48, - 0xea, 0x99, 0xeb, 0x35, 0xe4, 0x09, 0x14, 0xc2, 0x1c, 0x2c, 0xdb, 0x2b, 0x4e, 0x3d, 0x63, 0x49, - 0x49, 0x1c, 0x9f, 0x4c, 0x9d, 0xb3, 0x3c, 0x55, 0xd5, 0x33, 0xd5, 0xcd, 0xc8, 0x63, 0x58, 0x57, - 0xc1, 0x6f, 0xa6, 0xf7, 0x99, 0x7a, 0xb6, 0x42, 0x91, 0x50, 0x72, 0x5c, 0x9c, 0xc8, 0xfa, 0x3c, - 0x57, 0xcf, 0x5c, 0x30, 0x24, 0x14, 0x20, 0x91, 0x4f, 0x67, 0x7e, 0x77, 0xab, 0x67, 0x2f, 0x04, - 0x92, 0x1f, 0x43, 0x31, 0xca, 0x9a, 0x32, 0xbe, 0x7f, 0xd5, 0xb3, 0xd6, 0xe2, 0x9a, 0xed, 0xff, - 0xfc, 0x6d, 0x5b, 0xfb, 0xed, 0xe9, 0xb6, 0xf6, 0xc5, 0xe9, 0xb6, 0xf6, 0xe5, 0xe9, 0xb6, 0xf6, - 0xa7, 0xd3, 0x6d, 0xed, 0xaf, 0xa7, 0xdb, 0xda, 0x1f, 0xfe, 0xbe, 0xad, 0xfd, 0xe8, 0x9d, 0x9e, - 0xcd, 0xfb, 0xa3, 0x4e, 0xa3, 0xeb, 0x0e, 0xf7, 0x62, 0x81, 0xc9, 0xcf, 0xf8, 0x51, 0xbb, 0xb3, - 0x8e, 0x0e, 0xeb, 0xdb, 0xff, 0x0d, 0x00, 0x00, 0xff, 0xff, 0xce, 0x64, 0xb9, 0xe4, 0xe9, 0x1e, + 0x15, 0xde, 0xd1, 0x6a, 0x57, 0xd2, 0xd3, 0xfe, 0xc8, 0x6d, 0x27, 0x91, 0x85, 0xb3, 0xeb, 0x9a, + 0x8d, 0xed, 0x75, 0x12, 0xb4, 0x61, 0xa9, 0x50, 0x31, 0x76, 0x85, 0x5a, 0xad, 0x1d, 0xa4, 0x8a, + 0xed, 0x6c, 0xc6, 0xf6, 0x62, 0xa0, 0x2a, 0x53, 0x2d, 0x4d, 0x5b, 0x9a, 0x5a, 0x69, 0x66, 0x32, + 0xd3, 0x92, 0x25, 0x8a, 0x3b, 0x45, 0x15, 0x07, 0x2e, 0x54, 0x71, 0xe1, 0xce, 0x91, 0x03, 0x87, + 0x1c, 0x39, 0xe6, 0xc0, 0x81, 0x03, 0x67, 0x03, 0x0b, 0x27, 0x2a, 0x47, 0x8a, 0xe2, 0x48, 0xf5, + 0xeb, 0x9e, 0x3f, 0xad, 0xb4, 0x1a, 0x07, 0xdf, 0xb8, 0x48, 0xd3, 0x3d, 0xef, 0xbd, 0xee, 0x7e, + 0xfd, 0xde, 0xfb, 0xde, 0x7b, 0x03, 0xaf, 0xd3, 0x76, 0xc7, 0xde, 0xe3, 0x13, 0x8f, 0x05, 0xf2, + 0xb7, 0xee, 0xf9, 0x2e, 0x77, 0xc9, 0x6b, 0x9c, 0x39, 0x16, 0xf3, 0x07, 0xb6, 0xc3, 0xeb, 0x82, + 0xa4, 0x8e, 0x2f, 0x6b, 0xd7, 0x79, 0xcf, 0xf6, 0x2d, 0xd3, 0xa3, 0x3e, 0x9f, 0xec, 0x21, 0xe5, + 0x5e, 0xd7, 0xed, 0xba, 0xf1, 0x93, 0x64, 0xaf, 0xd5, 0x3a, 0xfe, 0xc4, 0xe3, 0xee, 0xde, 0x80, + 0xf9, 0x27, 0x7d, 0xa6, 0xfe, 0xd4, 0xbb, 0x8b, 0x7d, 0xbb, 0x1d, 0xec, 0x9d, 0x8c, 0x92, 0xeb, + 0xd5, 0xb6, 0xbb, 0xae, 0xdb, 0xed, 0x33, 0x29, 0xb3, 0x3d, 0x7c, 0xb6, 0xc7, 0xed, 0x01, 0x0b, + 0x38, 0x1d, 0x78, 0x8a, 0x60, 0x6b, 0x9a, 0xc0, 0x1a, 0xfa, 0x94, 0xdb, 0xae, 0x23, 0xdf, 0xeb, + 0xff, 0x5e, 0x81, 0x82, 0xc1, 0x3e, 0x1f, 0xb2, 0x80, 0x93, 0x0f, 0x20, 0xcf, 0x3a, 0x3d, 0xb7, + 0x9a, 0xbb, 0xaa, 0xed, 0x96, 0xf7, 0xf5, 0xfa, 0xcc, 0xb3, 0xd4, 0x15, 0xf5, 0xbd, 0x4e, 0xcf, + 0x6d, 0x2e, 0x19, 0xc8, 0x41, 0x6e, 0xc3, 0xca, 0xb3, 0xfe, 0x30, 0xe8, 0x55, 0x97, 0x91, 0x75, + 0xe7, 0x7c, 0xd6, 0x8f, 0x04, 0x69, 0x73, 0xc9, 0x90, 0x3c, 0x62, 0x59, 0xdb, 0x79, 0xe6, 0x56, + 0xf3, 0x59, 0x96, 0x6d, 0x39, 0xcf, 0x70, 0x59, 0xc1, 0x41, 0x9a, 0x00, 0x01, 0xe3, 0xa6, 0xeb, + 0x89, 0x03, 0x55, 0x57, 0x90, 0xff, 0xc6, 0xf9, 0xfc, 0x8f, 0x18, 0xff, 0x04, 0xc9, 0x9b, 0x4b, + 0x46, 0x29, 0x08, 0x07, 0x42, 0x92, 0xed, 0xd8, 0xdc, 0xec, 0xf4, 0xa8, 0xed, 0x54, 0x57, 0xb3, + 0x48, 0x6a, 0x39, 0x36, 0x3f, 0x14, 0xe4, 0x42, 0x92, 0x1d, 0x0e, 0x84, 0x2a, 0x3e, 0x1f, 0x32, + 0x7f, 0x52, 0x2d, 0x64, 0x51, 0xc5, 0xa7, 0x82, 0x54, 0xa8, 0x02, 0x79, 0xc8, 0xc7, 0x50, 0x6e, + 0xb3, 0xae, 0xed, 0x98, 0xed, 0xbe, 0xdb, 0x39, 0xa9, 0x16, 0x51, 0xc4, 0xee, 0xf9, 0x22, 0x1a, + 0x82, 0xa1, 0x21, 0xe8, 0x9b, 0x4b, 0x06, 0xb4, 0xa3, 0x11, 0x69, 0x40, 0xb1, 0xd3, 0x63, 0x9d, + 0x13, 0x93, 0x8f, 0xab, 0x25, 0x94, 0x74, 0xed, 0x7c, 0x49, 0x87, 0x82, 0xfa, 0xf1, 0xb8, 0xb9, + 0x64, 0x14, 0x3a, 0xf2, 0x51, 0xe8, 0xc5, 0x62, 0x7d, 0x7b, 0xc4, 0x7c, 0x21, 0xe5, 0x62, 0x16, + 0xbd, 0xdc, 0x95, 0xf4, 0x28, 0xa7, 0x64, 0x85, 0x03, 0x72, 0x0f, 0x4a, 0xcc, 0xb1, 0xd4, 0xc1, + 0xca, 0x28, 0xe8, 0xfa, 0x02, 0x0b, 0x73, 0xac, 0xf0, 0x58, 0x45, 0xa6, 0x9e, 0xc9, 0x87, 0xb0, + 0xda, 0x71, 0x07, 0x03, 0x9b, 0x57, 0xd7, 0x50, 0xc6, 0x5b, 0x0b, 0x8e, 0x84, 0xb4, 0xcd, 0x25, + 0x43, 0x71, 0x35, 0x0a, 0xb0, 0x32, 0xa2, 0xfd, 0x21, 0xd3, 0x6f, 0x40, 0x39, 0x61, 0xc9, 0xa4, + 0x0a, 0x85, 0x01, 0x0b, 0x02, 0xda, 0x65, 0x55, 0xed, 0xaa, 0xb6, 0x5b, 0x32, 0xc2, 0xa1, 0xbe, + 0x01, 0x6b, 0x49, 0xbb, 0xd5, 0x07, 0x11, 0xa3, 0xb0, 0x45, 0xc1, 0x38, 0x62, 0x7e, 0x20, 0x0c, + 0x50, 0x31, 0xaa, 0x21, 0xd9, 0x81, 0x75, 0x3c, 0xad, 0x19, 0xbe, 0x17, 0x7e, 0x95, 0x37, 0xd6, + 0x70, 0xf2, 0x58, 0x11, 0x6d, 0x43, 0xd9, 0xdb, 0xf7, 0x22, 0x92, 0x65, 0x24, 0x01, 0x6f, 0xdf, + 0x53, 0x04, 0xfa, 0x77, 0xa1, 0x32, 0x6d, 0xba, 0xa4, 0x02, 0xcb, 0x27, 0x6c, 0xa2, 0xd6, 0x13, + 0x8f, 0xe4, 0x92, 0x3a, 0x16, 0xae, 0x51, 0x32, 0xd4, 0x19, 0x7f, 0x97, 0x8b, 0x98, 0x23, 0x6b, + 0x15, 0xee, 0x26, 0x82, 0x04, 0x72, 0x97, 0xf7, 0x6b, 0x75, 0x19, 0x20, 0xea, 0x61, 0x80, 0xa8, + 0x3f, 0x0e, 0x23, 0x48, 0xa3, 0xf8, 0xe5, 0x8b, 0xed, 0xa5, 0x5f, 0xfe, 0x65, 0x5b, 0x33, 0x90, + 0x83, 0x5c, 0x16, 0x06, 0x45, 0x6d, 0xc7, 0xb4, 0x2d, 0xb5, 0x4e, 0x01, 0xc7, 0x2d, 0x8b, 0x7c, + 0x0a, 0x95, 0x8e, 0xeb, 0x04, 0xcc, 0x09, 0x86, 0x81, 0x08, 0x73, 0x74, 0x10, 0xa8, 0x58, 0x30, + 0xef, 0x92, 0x0f, 0x43, 0xf2, 0x23, 0xa4, 0x36, 0x36, 0x3b, 0xe9, 0x09, 0x72, 0x1f, 0x60, 0x44, + 0xfb, 0xb6, 0x45, 0xb9, 0xeb, 0x07, 0xd5, 0xfc, 0xd5, 0xe5, 0x73, 0x84, 0x1d, 0x87, 0x84, 0x4f, + 0x3c, 0x8b, 0x72, 0xd6, 0xc8, 0x8b, 0x9d, 0x1b, 0x09, 0x7e, 0x72, 0x1d, 0x36, 0xa9, 0xe7, 0x99, + 0x01, 0xa7, 0x9c, 0x99, 0xed, 0x09, 0x67, 0x01, 0xc6, 0x8b, 0x35, 0x63, 0x9d, 0x7a, 0xde, 0x23, + 0x31, 0xdb, 0x10, 0x93, 0xba, 0x15, 0xdd, 0x36, 0xba, 0x26, 0x21, 0x90, 0xb7, 0x28, 0xa7, 0xa8, + 0xad, 0x35, 0x03, 0x9f, 0xc5, 0x9c, 0x47, 0x79, 0x4f, 0xe9, 0x00, 0x9f, 0xc9, 0xeb, 0xb0, 0xda, + 0x63, 0x76, 0xb7, 0xc7, 0xf1, 0xd8, 0xcb, 0x86, 0x1a, 0x89, 0x8b, 0xf1, 0x7c, 0x77, 0xc4, 0x30, + 0xba, 0x15, 0x0d, 0x39, 0xd0, 0x7f, 0x95, 0x83, 0x0b, 0x67, 0xdc, 0x57, 0xc8, 0xed, 0xd1, 0xa0, + 0x17, 0xae, 0x25, 0x9e, 0xc9, 0x6d, 0x21, 0x97, 0x5a, 0xcc, 0x57, 0x51, 0xf9, 0xcd, 0x39, 0x1a, + 0x68, 0x22, 0x91, 0x3a, 0xb8, 0x62, 0x21, 0x4f, 0xa0, 0xd2, 0xa7, 0x01, 0x37, 0xa5, 0xed, 0x9b, + 0x18, 0x65, 0x97, 0xcf, 0x8d, 0x04, 0xf7, 0x69, 0xe8, 0x33, 0xc2, 0xb8, 0x95, 0xb8, 0x8d, 0x7e, + 0x6a, 0x96, 0x3c, 0x85, 0x4b, 0xed, 0xc9, 0x4f, 0xa8, 0xc3, 0x6d, 0x87, 0x99, 0x67, 0xee, 0x68, + 0x7b, 0x8e, 0xe8, 0x7b, 0x23, 0xdb, 0x62, 0x4e, 0x27, 0xbc, 0x9c, 0x8b, 0x91, 0x88, 0xe8, 0xf2, + 0x02, 0xfd, 0x29, 0x6c, 0xa4, 0x63, 0x11, 0xd9, 0x80, 0x1c, 0x1f, 0x2b, 0x8d, 0xe4, 0xf8, 0x98, + 0x7c, 0x07, 0xf2, 0x42, 0x1c, 0x6a, 0x63, 0x63, 0x2e, 0x58, 0x28, 0xee, 0xc7, 0x13, 0x8f, 0x19, + 0x48, 0xaf, 0xeb, 0x91, 0x27, 0x44, 0xf1, 0x69, 0x5a, 0xb6, 0x7e, 0x13, 0x36, 0xa7, 0x42, 0x4f, + 0xe2, 0x5a, 0xb5, 0xe4, 0xb5, 0xea, 0x9b, 0xb0, 0x9e, 0x8a, 0x30, 0xfa, 0x1f, 0x57, 0xa1, 0x68, + 0xb0, 0xc0, 0x13, 0x46, 0x4c, 0x9a, 0x50, 0x62, 0xe3, 0x0e, 0x93, 0xb0, 0xa4, 0x2d, 0x08, 0xe2, + 0x92, 0xe7, 0x5e, 0x48, 0x2f, 0xa2, 0x66, 0xc4, 0x4c, 0x6e, 0xa5, 0x20, 0x79, 0x67, 0x91, 0x90, + 0x24, 0x26, 0xdf, 0x49, 0x63, 0xf2, 0x5b, 0x0b, 0x78, 0xa7, 0x40, 0xf9, 0x56, 0x0a, 0x94, 0x17, + 0x2d, 0x9c, 0x42, 0xe5, 0xd6, 0x0c, 0x54, 0x5e, 0x74, 0xfc, 0x39, 0xb0, 0xdc, 0x9a, 0x01, 0xcb, + 0xbb, 0x0b, 0xf7, 0x32, 0x13, 0x97, 0xef, 0xa4, 0x71, 0x79, 0x91, 0x3a, 0xa6, 0x80, 0xf9, 0xfe, + 0x2c, 0x60, 0xbe, 0xb9, 0x40, 0xc6, 0x5c, 0x64, 0x3e, 0x3c, 0x83, 0xcc, 0xd7, 0x17, 0x88, 0x9a, + 0x01, 0xcd, 0xad, 0x14, 0x34, 0x43, 0x26, 0xdd, 0xcc, 0xc1, 0xe6, 0x8f, 0xce, 0x62, 0xf3, 0x8d, + 0x45, 0xa6, 0x36, 0x0b, 0x9c, 0xbf, 0x37, 0x05, 0xce, 0xd7, 0x16, 0x9d, 0x6a, 0x2e, 0x3a, 0xdf, + 0x14, 0xf1, 0x71, 0xca, 0x33, 0x44, 0x2c, 0x65, 0xbe, 0xef, 0xfa, 0x0a, 0xf8, 0xe4, 0x40, 0xdf, + 0x15, 0x11, 0x3b, 0xb6, 0xff, 0x73, 0x90, 0x1c, 0x9d, 0x36, 0x61, 0xed, 0xfa, 0x17, 0x5a, 0xcc, + 0x8b, 0x91, 0x2d, 0x19, 0xed, 0x4b, 0x2a, 0xda, 0x27, 0x00, 0x3e, 0x97, 0x06, 0xf8, 0x6d, 0x28, + 0x0b, 0x4c, 0x99, 0xc2, 0x6e, 0xea, 0x85, 0xd8, 0x4d, 0xde, 0x86, 0x0b, 0x18, 0x7f, 0x65, 0x1a, + 0xa0, 0x02, 0x49, 0x1e, 0x03, 0xc9, 0xa6, 0x78, 0x21, 0x35, 0x28, 0x81, 0xe2, 0x9b, 0x70, 0x31, + 0x41, 0x2b, 0xe4, 0x22, 0x16, 0x48, 0x90, 0xaa, 0x44, 0xd4, 0x07, 0x9e, 0xd7, 0xa4, 0x41, 0x4f, + 0x7f, 0x10, 0x2b, 0x28, 0xce, 0x0b, 0x08, 0xe4, 0x3b, 0xae, 0x25, 0xcf, 0xbd, 0x6e, 0xe0, 0xb3, + 0xc8, 0x15, 0xfa, 0x6e, 0x17, 0x37, 0x57, 0x32, 0xc4, 0xa3, 0xa0, 0x8a, 0x5c, 0xbb, 0x24, 0x7d, + 0x56, 0xff, 0xbd, 0x16, 0xcb, 0x8b, 0x53, 0x85, 0x59, 0xa8, 0xae, 0xbd, 0x4a, 0x54, 0xcf, 0xfd, + 0x6f, 0xa8, 0xae, 0xff, 0x4b, 0x8b, 0xaf, 0x34, 0xc2, 0xeb, 0xaf, 0xa7, 0x02, 0x61, 0x5d, 0xb6, + 0x63, 0xb1, 0x31, 0xaa, 0x7c, 0xd9, 0x90, 0x83, 0x30, 0xd5, 0x5a, 0xc5, 0x6b, 0x48, 0xa7, 0x5a, + 0x05, 0x9c, 0x93, 0x03, 0xf2, 0x3e, 0xe2, 0xbc, 0xfb, 0x4c, 0x85, 0x86, 0x14, 0x08, 0xca, 0xa2, + 0xae, 0xae, 0xaa, 0xb9, 0x23, 0x41, 0x66, 0x48, 0xea, 0x04, 0xbe, 0x94, 0x52, 0x69, 0xc3, 0x15, + 0x28, 0x89, 0xad, 0x07, 0x1e, 0xed, 0x30, 0xf4, 0xed, 0x92, 0x11, 0x4f, 0xe8, 0x16, 0x90, 0xb3, + 0x31, 0x86, 0x3c, 0x84, 0x55, 0x36, 0x62, 0x0e, 0x17, 0x77, 0x24, 0xd4, 0x7a, 0x65, 0x2e, 0x10, + 0x33, 0x87, 0x37, 0xaa, 0x42, 0x99, 0xff, 0x7c, 0xb1, 0x5d, 0x91, 0x3c, 0xef, 0xba, 0x03, 0x9b, + 0xb3, 0x81, 0xc7, 0x27, 0x86, 0x92, 0xa2, 0xff, 0x2c, 0x27, 0xf0, 0x30, 0x15, 0x7f, 0x66, 0xaa, + 0x37, 0x74, 0x9a, 0x5c, 0x22, 0x45, 0xca, 0xa6, 0xf2, 0x37, 0x01, 0xba, 0x34, 0x30, 0x9f, 0x53, + 0x87, 0x33, 0x4b, 0xe9, 0xbd, 0xd4, 0xa5, 0xc1, 0x0f, 0x70, 0x42, 0xe4, 0x9b, 0xe2, 0xf5, 0x30, + 0x60, 0x16, 0x5e, 0xc0, 0xb2, 0x51, 0xe8, 0xd2, 0xe0, 0x49, 0xc0, 0xac, 0xc4, 0x59, 0x0b, 0xaf, + 0xe2, 0xac, 0x69, 0x7d, 0x17, 0xa7, 0xf5, 0xfd, 0xf3, 0x5c, 0xec, 0x1d, 0x71, 0xfa, 0xf0, 0xff, + 0xa9, 0x8b, 0xdf, 0x60, 0x4d, 0x91, 0x06, 0x01, 0xf2, 0x43, 0xb8, 0x10, 0x79, 0xa5, 0x39, 0x44, + 0x6f, 0x0d, 0xad, 0xf0, 0xe5, 0x9c, 0xbb, 0x32, 0x4a, 0x4f, 0x07, 0xe4, 0x33, 0x78, 0x63, 0x2a, + 0x06, 0x45, 0x0b, 0xe4, 0x5e, 0x2a, 0x14, 0xbd, 0x96, 0x0e, 0x45, 0xa1, 0xfc, 0x58, 0x7b, 0xcb, + 0xaf, 0xc4, 0x6b, 0x5a, 0x22, 0x85, 0x4d, 0xc2, 0xdb, 0x4c, 0x9b, 0xd8, 0x81, 0x75, 0x9f, 0x71, + 0x51, 0x4b, 0xa5, 0xaa, 0x86, 0x35, 0x39, 0x29, 0x21, 0x41, 0xff, 0xb3, 0x06, 0x9b, 0x53, 0xa7, + 0x20, 0x1f, 0xc0, 0x8a, 0x84, 0x69, 0xed, 0xdc, 0x6e, 0x09, 0x5e, 0x8b, 0x3a, 0xb8, 0x64, 0x20, + 0x07, 0x50, 0x64, 0x2a, 0x05, 0x57, 0x9a, 0xbb, 0xb6, 0x20, 0x53, 0x57, 0xfc, 0x11, 0x1b, 0xb9, + 0x0b, 0xa5, 0xe8, 0x7e, 0x16, 0x94, 0x77, 0xd1, 0xf5, 0x2a, 0x21, 0x31, 0xa3, 0x7e, 0x08, 0xe5, + 0xc4, 0xf6, 0xc8, 0x37, 0xa0, 0x34, 0xa0, 0x63, 0x55, 0x93, 0xc9, 0x2c, 0xbb, 0x38, 0xa0, 0x63, + 0x2c, 0xc7, 0xc8, 0x1b, 0x50, 0x10, 0x2f, 0xbb, 0x54, 0xde, 0xf6, 0xb2, 0xb1, 0x3a, 0xa0, 0xe3, + 0xef, 0xd3, 0x40, 0xff, 0x85, 0x06, 0x1b, 0xe9, 0x7d, 0x92, 0x77, 0x80, 0x08, 0x5a, 0xda, 0x65, + 0xa6, 0x33, 0x1c, 0x48, 0x20, 0x0d, 0x25, 0x6e, 0x0e, 0xe8, 0xf8, 0xa0, 0xcb, 0x1e, 0x0e, 0x07, + 0xb8, 0x74, 0x40, 0x1e, 0x40, 0x25, 0x24, 0x0e, 0x3b, 0x62, 0x4a, 0x2b, 0x97, 0xcf, 0x54, 0xc4, + 0x77, 0x15, 0x81, 0x2c, 0x88, 0x7f, 0x2d, 0x0a, 0xe2, 0x0d, 0x29, 0x2f, 0x7c, 0xa3, 0xbf, 0x0f, + 0x9b, 0x53, 0x27, 0x26, 0x3a, 0xac, 0x7b, 0xc3, 0xb6, 0x79, 0xc2, 0x26, 0x26, 0xaa, 0x04, 0xfd, + 0xa1, 0x64, 0x94, 0xbd, 0x61, 0xfb, 0x63, 0x36, 0x11, 0xa5, 0x49, 0xa0, 0x77, 0x60, 0x23, 0x5d, + 0x71, 0x09, 0x74, 0xf1, 0xdd, 0xa1, 0x63, 0xe1, 0xbe, 0x57, 0x0c, 0x39, 0x20, 0xb7, 0x61, 0x65, + 0xe4, 0x4a, 0x93, 0x3f, 0xaf, 0xc4, 0x3a, 0x76, 0x39, 0x4b, 0xd4, 0x6d, 0x92, 0x47, 0x0f, 0x60, + 0x05, 0x8d, 0x57, 0x18, 0x22, 0xd6, 0x4e, 0x2a, 0xbb, 0x11, 0xcf, 0xe4, 0x18, 0x80, 0x72, 0xee, + 0xdb, 0xed, 0x61, 0x2c, 0xbe, 0x9a, 0x14, 0xdf, 0xb7, 0xdb, 0x41, 0xfd, 0x64, 0x54, 0x3f, 0xa2, + 0xb6, 0xdf, 0xb8, 0xa2, 0xcc, 0xff, 0x52, 0xcc, 0x93, 0x70, 0x81, 0x84, 0x24, 0xfd, 0xab, 0x3c, + 0xac, 0xca, 0x9a, 0x94, 0x7c, 0x98, 0xee, 0x90, 0x94, 0xf7, 0xb7, 0xe6, 0x6d, 0x5f, 0x52, 0xa9, + 0xdd, 0x47, 0x69, 0xd6, 0xf5, 0xe9, 0xb6, 0x43, 0xa3, 0x7c, 0xfa, 0x62, 0xbb, 0x80, 0x29, 0x4a, + 0xeb, 0x6e, 0xdc, 0x83, 0x98, 0x57, 0x82, 0x87, 0x0d, 0x8f, 0xfc, 0x4b, 0x37, 0x3c, 0x9a, 0xb0, + 0x9e, 0xc8, 0xc9, 0x6c, 0x4b, 0x15, 0x33, 0x5b, 0xe7, 0x39, 0x5d, 0xeb, 0xae, 0xda, 0x7f, 0x39, + 0xca, 0xd9, 0x5a, 0x16, 0xd9, 0x4d, 0x57, 0xe2, 0x98, 0xda, 0xc9, 0x9c, 0x22, 0x51, 0x5c, 0x8b, + 0xc4, 0x4e, 0xb8, 0x83, 0x88, 0x10, 0x92, 0x44, 0xa6, 0x18, 0x45, 0x31, 0x81, 0x2f, 0x6f, 0xc0, + 0x66, 0x9c, 0xfd, 0x48, 0x92, 0xa2, 0x94, 0x12, 0x4f, 0x23, 0xe1, 0x7b, 0x70, 0xc9, 0x61, 0x63, + 0x6e, 0x4e, 0x53, 0x97, 0x90, 0x9a, 0x88, 0x77, 0xc7, 0x69, 0x8e, 0x6b, 0xb0, 0x11, 0xc7, 0x59, + 0xa4, 0x05, 0xd9, 0x1f, 0x89, 0x66, 0x91, 0xec, 0x32, 0x14, 0xa3, 0xdc, 0xb4, 0x8c, 0x04, 0x05, + 0x2a, 0x53, 0xd2, 0x28, 0xdb, 0xf5, 0x59, 0x30, 0xec, 0x73, 0x25, 0x64, 0x0d, 0x69, 0x30, 0xdb, + 0x35, 0xe4, 0x3c, 0xd2, 0xee, 0xc0, 0x7a, 0x18, 0x55, 0x24, 0xdd, 0x3a, 0xd2, 0xad, 0x85, 0x93, + 0x48, 0x74, 0x13, 0x2a, 0x9e, 0xef, 0x7a, 0x6e, 0xc0, 0x7c, 0x93, 0x5a, 0x96, 0xcf, 0x82, 0xa0, + 0xba, 0x21, 0xe5, 0x85, 0xf3, 0x07, 0x72, 0x5a, 0xff, 0x16, 0x14, 0xc2, 0xa4, 0xfb, 0x12, 0xac, + 0x34, 0xa2, 0x08, 0x99, 0x37, 0xe4, 0x40, 0x80, 0xf0, 0x81, 0xe7, 0xa9, 0x16, 0x9c, 0x78, 0xd4, + 0xfb, 0x50, 0x50, 0x17, 0x36, 0xb3, 0xf1, 0xf2, 0x00, 0xd6, 0x3c, 0xea, 0x8b, 0x63, 0x24, 0xdb, + 0x2f, 0xf3, 0xca, 0xc6, 0x23, 0xea, 0xf3, 0x47, 0x8c, 0xa7, 0xba, 0x30, 0x65, 0xe4, 0x97, 0x53, + 0xfa, 0x2d, 0x58, 0x4f, 0xd1, 0x88, 0x6d, 0x72, 0x97, 0xd3, 0x7e, 0xe8, 0xe8, 0x38, 0x88, 0x76, + 0x92, 0x8b, 0x77, 0xa2, 0xdf, 0x86, 0x52, 0x74, 0x57, 0xa2, 0x1a, 0x09, 0x55, 0xa1, 0x29, 0xf5, + 0xcb, 0x21, 0x76, 0x9a, 0xdc, 0xe7, 0xcc, 0x57, 0xd6, 0x2f, 0x07, 0x3a, 0x4b, 0x04, 0x26, 0x09, + 0x79, 0xe4, 0x0e, 0x14, 0x54, 0x60, 0x52, 0xfe, 0x38, 0xaf, 0xa7, 0x74, 0x84, 0x91, 0x2a, 0xec, + 0x29, 0xc9, 0xb8, 0x15, 0x2f, 0x93, 0x4b, 0x2e, 0xf3, 0x53, 0x28, 0x86, 0xc1, 0x27, 0x8d, 0x12, + 0x72, 0x85, 0xab, 0x8b, 0x50, 0x42, 0x2d, 0x12, 0x33, 0x0a, 0x6b, 0x0a, 0xec, 0xae, 0xc3, 0x2c, + 0x33, 0x76, 0x41, 0x5c, 0xb3, 0x68, 0x6c, 0xca, 0x17, 0xf7, 0x43, 0xff, 0xd2, 0xdf, 0x83, 0x55, + 0xb9, 0xd7, 0x99, 0x21, 0x6e, 0x06, 0xfe, 0xea, 0xff, 0xd0, 0xa0, 0x18, 0xc2, 0xc7, 0x4c, 0xa6, + 0xd4, 0x21, 0x72, 0x5f, 0xf7, 0x10, 0xaf, 0x3e, 0x24, 0xbd, 0x0b, 0x04, 0x2d, 0xc5, 0x1c, 0xb9, + 0xdc, 0x76, 0xba, 0xa6, 0xbc, 0x0b, 0x99, 0x2e, 0x56, 0xf0, 0xcd, 0x31, 0xbe, 0x38, 0x12, 0xf3, + 0x6f, 0xef, 0x40, 0x39, 0xd1, 0x0a, 0x23, 0x05, 0x58, 0x7e, 0xc8, 0x9e, 0x57, 0x96, 0x48, 0x19, + 0x0a, 0x06, 0xc3, 0x46, 0x42, 0x45, 0xdb, 0xff, 0xaa, 0x00, 0x9b, 0x07, 0x8d, 0xc3, 0xd6, 0x81, + 0xe7, 0xf5, 0xed, 0x0e, 0xe2, 0x19, 0xf9, 0x04, 0xf2, 0x58, 0x4c, 0x67, 0xf8, 0x08, 0x54, 0xcb, + 0xd2, 0x95, 0x22, 0x06, 0xac, 0x60, 0xcd, 0x4d, 0xb2, 0x7c, 0x1b, 0xaa, 0x65, 0x6a, 0x56, 0x89, + 0x4d, 0xa2, 0xc1, 0x65, 0xf8, 0x64, 0x54, 0xcb, 0xd2, 0xc1, 0x22, 0x9f, 0x41, 0x29, 0x2e, 0xa6, + 0xb3, 0x7e, 0x48, 0xaa, 0x65, 0xee, 0x6d, 0x09, 0xf9, 0x71, 0xf9, 0x90, 0xf5, 0x33, 0x4a, 0x2d, + 0x73, 0x53, 0x87, 0x3c, 0x85, 0x42, 0x58, 0xa8, 0x65, 0xfb, 0xd4, 0x53, 0xcb, 0xd8, 0x77, 0x12, + 0xd7, 0x27, 0xeb, 0xeb, 0x2c, 0xdf, 0xb3, 0x6a, 0x99, 0x9a, 0x6b, 0xe4, 0x09, 0xac, 0xaa, 0x0c, + 0x39, 0xd3, 0x47, 0x9c, 0x5a, 0xb6, 0x6e, 0x92, 0x50, 0x72, 0xdc, 0xc1, 0xc8, 0xfa, 0x0d, 0xaf, + 0x96, 0xb9, 0xab, 0x48, 0x28, 0x40, 0xa2, 0xe8, 0xce, 0xfc, 0x71, 0xae, 0x96, 0xbd, 0x5b, 0x48, + 0x7e, 0x0c, 0xc5, 0xa8, 0xb4, 0xca, 0xf8, 0x91, 0xac, 0x96, 0xb5, 0x61, 0xd7, 0x68, 0xfd, 0xe7, + 0x6f, 0x5b, 0xda, 0x6f, 0x4f, 0xb7, 0xb4, 0x2f, 0x4e, 0xb7, 0xb4, 0x2f, 0x4f, 0xb7, 0xb4, 0x3f, + 0x9d, 0x6e, 0x69, 0x7f, 0x3d, 0xdd, 0xd2, 0xfe, 0xf0, 0xf7, 0x2d, 0xed, 0x47, 0xef, 0x74, 0x6d, + 0xde, 0x1b, 0xb6, 0xeb, 0x1d, 0x77, 0xb0, 0x17, 0x0b, 0x4c, 0x3e, 0xc6, 0x5f, 0xbe, 0xdb, 0xab, + 0x18, 0xb0, 0xbe, 0xfd, 0xdf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x0c, 0x66, 0x8a, 0xe9, 0x0e, 0x1f, 0x00, 0x00, } @@ -4512,6 +4521,9 @@ func (this *ResponseCommit) Equal(that interface{}) bool { if !bytes.Equal(this.Data, that1.Data) { return false } + if this.RetainHeight != that1.RetainHeight { + return false + } if !bytes.Equal(this.XXX_unrecognized, that1.XXX_unrecognized) { return false } @@ -7143,6 +7155,11 @@ func (m *ResponseCommit) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.RetainHeight != 0 { + i = encodeVarintTypes(dAtA, i, uint64(m.RetainHeight)) + i-- + dAtA[i] = 0x18 + } if len(m.Data) > 0 { i -= len(m.Data) copy(dAtA[i:], m.Data) @@ -8479,8 +8496,12 @@ func NewPopulatedResponseCommit(r randyTypes, easy bool) *ResponseCommit { for i := 0; i < v30; i++ { this.Data[i] = byte(r.Intn(256)) } + this.RetainHeight = int64(r.Int63()) + if r.Intn(2) == 0 { + this.RetainHeight *= -1 + } if !easy && r.Intn(10) != 0 { - this.XXX_unrecognized = randUnrecognizedTypes(r, 3) + this.XXX_unrecognized = randUnrecognizedTypes(r, 4) } return this } @@ -9665,6 +9686,9 @@ func (m *ResponseCommit) Size() (n int) { if l > 0 { n += 1 + l + sovTypes(uint64(l)) } + if m.RetainHeight != 0 { + n += 1 + sovTypes(uint64(m.RetainHeight)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -14046,6 +14070,25 @@ func (m *ResponseCommit) Unmarshal(dAtA []byte) error { m.Data = []byte{} } iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field RetainHeight", wireType) + } + m.RetainHeight = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.RetainHeight |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipTypes(dAtA[iNdEx:]) diff --git a/abci/types/types.proto b/abci/types/types.proto index 0d47ad9b3..351329de1 100644 --- a/abci/types/types.proto +++ b/abci/types/types.proto @@ -207,7 +207,8 @@ message ResponseEndBlock { message ResponseCommit { // reserve 1 - bytes data = 2; + bytes data = 2; + int64 retain_height = 3; } //---------------------------------------- diff --git a/blockchain/v0/pool.go b/blockchain/v0/pool.go index 1931d7960..bd8165752 100644 --- a/blockchain/v0/pool.go +++ b/blockchain/v0/pool.go @@ -284,16 +284,17 @@ func (pool *BlockPool) MaxPeerHeight() int64 { return pool.maxPeerHeight } -// SetPeerHeight sets the peer's alleged blockchain height. -func (pool *BlockPool) SetPeerHeight(peerID p2p.ID, height int64) { +// SetPeerRange sets the peer's alleged blockchain base and height. +func (pool *BlockPool) SetPeerRange(peerID p2p.ID, base int64, height int64) { pool.mtx.Lock() defer pool.mtx.Unlock() peer := pool.peers[peerID] if peer != nil { + peer.base = base peer.height = height } else { - peer = newBPPeer(pool, peerID, height) + peer = newBPPeer(pool, peerID, base, height) peer.setLogger(pool.Logger.With("peer", peerID)) pool.peers[peerID] = peer } @@ -346,9 +347,9 @@ func (pool *BlockPool) updateMaxPeerHeight() { pool.maxPeerHeight = max } -// Pick an available peer with at least the given minHeight. +// Pick an available peer with the given height available. // If no peers are available, returns nil. -func (pool *BlockPool) pickIncrAvailablePeer(minHeight int64) *bpPeer { +func (pool *BlockPool) pickIncrAvailablePeer(height int64) *bpPeer { pool.mtx.Lock() defer pool.mtx.Unlock() @@ -360,7 +361,7 @@ func (pool *BlockPool) pickIncrAvailablePeer(minHeight int64) *bpPeer { if peer.numPending >= maxPendingRequestsPerPeer { continue } - if peer.height < minHeight { + if height < peer.base || height > peer.height { continue } peer.incrPending() @@ -432,6 +433,7 @@ type bpPeer struct { didTimeout bool numPending int32 height int64 + base int64 pool *BlockPool id p2p.ID recvMonitor *flow.Monitor @@ -441,10 +443,11 @@ type bpPeer struct { logger log.Logger } -func newBPPeer(pool *BlockPool, peerID p2p.ID, height int64) *bpPeer { +func newBPPeer(pool *BlockPool, peerID p2p.ID, base int64, height int64) *bpPeer { peer := &bpPeer{ pool: pool, id: peerID, + base: base, height: height, numPending: 0, logger: log.NewNopLogger(), diff --git a/blockchain/v0/pool_test.go b/blockchain/v0/pool_test.go index 783ff2526..9a3dd299c 100644 --- a/blockchain/v0/pool_test.go +++ b/blockchain/v0/pool_test.go @@ -20,6 +20,7 @@ func init() { type testPeer struct { id p2p.ID + base int64 height int64 inputChan chan inputData //make sure each peer's data is sequential } @@ -67,7 +68,11 @@ func makePeers(numPeers int, minHeight, maxHeight int64) testPeers { for i := 0; i < numPeers; i++ { peerID := p2p.ID(tmrand.Str(12)) height := minHeight + tmrand.Int63n(maxHeight-minHeight) - peers[peerID] = testPeer{peerID, height, make(chan inputData, 10)} + base := minHeight + int64(i) + if base > height { + base = height + } + peers[peerID] = testPeer{peerID, base, height, make(chan inputData, 10)} } return peers } @@ -93,7 +98,7 @@ func TestBlockPoolBasic(t *testing.T) { // Introduce each peer. go func() { for _, peer := range peers { - pool.SetPeerHeight(peer.id, peer.height) + pool.SetPeerRange(peer.id, peer.base, peer.height) } }() @@ -148,7 +153,7 @@ func TestBlockPoolTimeout(t *testing.T) { // Introduce each peer. go func() { for _, peer := range peers { - pool.SetPeerHeight(peer.id, peer.height) + pool.SetPeerRange(peer.id, peer.base, peer.height) } }() @@ -192,7 +197,7 @@ func TestBlockPoolRemovePeer(t *testing.T) { for i := 0; i < 10; i++ { peerID := p2p.ID(fmt.Sprintf("%d", i+1)) height := int64(i + 1) - peers[peerID] = testPeer{peerID, height, make(chan inputData)} + peers[peerID] = testPeer{peerID, 0, height, make(chan inputData)} } requestsCh := make(chan BlockRequest) errorsCh := make(chan peerError) @@ -205,7 +210,7 @@ func TestBlockPoolRemovePeer(t *testing.T) { // add peers for peerID, peer := range peers { - pool.SetPeerHeight(peerID, peer.height) + pool.SetPeerRange(peerID, peer.base, peer.height) } assert.EqualValues(t, 10, pool.MaxPeerHeight()) diff --git a/blockchain/v0/reactor.go b/blockchain/v0/reactor.go index d47e892c2..247222160 100644 --- a/blockchain/v0/reactor.go +++ b/blockchain/v0/reactor.go @@ -140,12 +140,15 @@ func (bcR *BlockchainReactor) GetChannels() []*p2p.ChannelDescriptor { // AddPeer implements Reactor by sending our state to peer. func (bcR *BlockchainReactor) AddPeer(peer p2p.Peer) { - msgBytes := cdc.MustMarshalBinaryBare(&bcStatusResponseMessage{bcR.store.Height()}) + msgBytes := cdc.MustMarshalBinaryBare(&bcStatusResponseMessage{ + Height: bcR.store.Height(), + Base: bcR.store.Base(), + }) peer.Send(BlockchainChannel, msgBytes) // it's OK if send fails. will try later in poolRoutine // peer is added to the pool once we receive the first - // bcStatusResponseMessage from the peer and call pool.SetPeerHeight + // bcStatusResponseMessage from the peer and call pool.SetPeerRange } // RemovePeer implements Reactor by removing peer from the pool. @@ -155,8 +158,6 @@ func (bcR *BlockchainReactor) RemovePeer(peer p2p.Peer, reason interface{}) { // respondToPeer loads a block and sends it to the requesting peer, // if we have it. Otherwise, we'll respond saying we don't have it. -// According to the Tendermint spec, if all nodes are honest, -// no node should be requesting for a block that's non-existent. func (bcR *BlockchainReactor) respondToPeer(msg *bcBlockRequestMessage, src p2p.Peer) (queued bool) { @@ -196,11 +197,15 @@ func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) bcR.pool.AddBlock(src.ID(), msg.Block, len(msgBytes)) case *bcStatusRequestMessage: // Send peer our state. - msgBytes := cdc.MustMarshalBinaryBare(&bcStatusResponseMessage{bcR.store.Height()}) - src.TrySend(BlockchainChannel, msgBytes) + src.TrySend(BlockchainChannel, cdc.MustMarshalBinaryBare(&bcStatusResponseMessage{ + Height: bcR.store.Height(), + Base: bcR.store.Base(), + })) case *bcStatusResponseMessage: // Got a peer status. Unverified. - bcR.pool.SetPeerHeight(src.ID(), msg.Height) + bcR.pool.SetPeerRange(src.ID(), msg.Base, msg.Height) + case *bcNoBlockResponseMessage: + bcR.Logger.Debug("Peer does not have requested block", "peer", src, "height", msg.Height) default: bcR.Logger.Error(fmt.Sprintf("Unknown message type %v", reflect.TypeOf(msg))) } @@ -338,7 +343,7 @@ FOR_LOOP: // TODO: same thing for app - but we would need a way to // get the hash without persisting the state var err error - state, err = bcR.blockExec.ApplyBlock(state, firstID, first) + state, _, err = bcR.blockExec.ApplyBlock(state, firstID, first) if err != nil { // TODO This is bad, are we zombie? panic(fmt.Sprintf("Failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err)) @@ -360,9 +365,12 @@ FOR_LOOP: } } -// BroadcastStatusRequest broadcasts `BlockStore` height. +// BroadcastStatusRequest broadcasts `BlockStore` base and height. func (bcR *BlockchainReactor) BroadcastStatusRequest() error { - msgBytes := cdc.MustMarshalBinaryBare(&bcStatusRequestMessage{bcR.store.Height()}) + msgBytes := cdc.MustMarshalBinaryBare(&bcStatusRequestMessage{ + Base: bcR.store.Base(), + Height: bcR.store.Height(), + }) bcR.Switch.Broadcast(BlockchainChannel, msgBytes) return nil } @@ -446,34 +454,48 @@ func (m *bcBlockResponseMessage) String() string { type bcStatusRequestMessage struct { Height int64 + Base int64 } // ValidateBasic performs basic validation. func (m *bcStatusRequestMessage) ValidateBasic() error { + if m.Base < 0 { + return errors.New("negative Base") + } if m.Height < 0 { return errors.New("negative Height") } + if m.Base > m.Height { + return fmt.Errorf("base %v cannot be greater than height %v", m.Base, m.Height) + } return nil } func (m *bcStatusRequestMessage) String() string { - return fmt.Sprintf("[bcStatusRequestMessage %v]", m.Height) + return fmt.Sprintf("[bcStatusRequestMessage %v:%v]", m.Base, m.Height) } //------------------------------------- type bcStatusResponseMessage struct { Height int64 + Base int64 } // ValidateBasic performs basic validation. func (m *bcStatusResponseMessage) ValidateBasic() error { + if m.Base < 0 { + return errors.New("negative Base") + } if m.Height < 0 { return errors.New("negative Height") } + if m.Base > m.Height { + return fmt.Errorf("base %v cannot be greater than height %v", m.Base, m.Height) + } return nil } func (m *bcStatusResponseMessage) String() string { - return fmt.Sprintf("[bcStatusResponseMessage %v]", m.Height) + return fmt.Sprintf("[bcStatusResponseMessage %v:%v]", m.Base, m.Height) } diff --git a/blockchain/v0/reactor_test.go b/blockchain/v0/reactor_test.go index 74811dd0a..a31c9a141 100644 --- a/blockchain/v0/reactor_test.go +++ b/blockchain/v0/reactor_test.go @@ -112,7 +112,7 @@ func newBlockchainReactor( thisParts := thisBlock.MakePartSet(types.BlockPartSizeBytes) blockID := types.BlockID{Hash: thisBlock.Hash(), PartsHeader: thisParts.Header()} - state, err = blockExec.ApplyBlock(state, blockID, thisBlock) + state, _, err = blockExec.ApplyBlock(state, blockID, thisBlock) if err != nil { panic(errors.Wrap(err, "error apply block")) } diff --git a/blockchain/v1/peer.go b/blockchain/v1/peer.go index 02b1b4fc1..ad26585b3 100644 --- a/blockchain/v1/peer.go +++ b/blockchain/v1/peer.go @@ -27,6 +27,7 @@ type BpPeer struct { logger log.Logger ID p2p.ID + Base int64 // the peer reported base Height int64 // the peer reported height NumPendingBlockRequests int // number of requests still waiting for block responses blocks map[int64]*types.Block // blocks received or expected to be received from this peer @@ -38,14 +39,15 @@ type BpPeer struct { } // NewBpPeer creates a new peer. -func NewBpPeer( - peerID p2p.ID, height int64, onErr func(err error, peerID p2p.ID), params *BpPeerParams) *BpPeer { +func NewBpPeer(peerID p2p.ID, base int64, height int64, + onErr func(err error, peerID p2p.ID), params *BpPeerParams) *BpPeer { if params == nil { params = BpPeerDefaultParams() } return &BpPeer{ ID: peerID, + Base: base, Height: height, blocks: make(map[int64]*types.Block, maxRequestsPerPeer), logger: log.NewNopLogger(), diff --git a/blockchain/v1/peer_test.go b/blockchain/v1/peer_test.go index aac03db7e..0e7a73473 100644 --- a/blockchain/v1/peer_test.go +++ b/blockchain/v1/peer_test.go @@ -16,7 +16,7 @@ import ( func TestPeerMonitor(t *testing.T) { peer := NewBpPeer( - p2p.ID(tmrand.Str(12)), 10, + p2p.ID(tmrand.Str(12)), 0, 10, func(err error, _ p2p.ID) {}, nil) peer.SetLogger(log.TestingLogger()) @@ -35,7 +35,7 @@ func TestPeerResetBlockResponseTimer(t *testing.T) { params := &BpPeerParams{timeout: 2 * time.Millisecond} peer := NewBpPeer( - p2p.ID(tmrand.Str(12)), 10, + p2p.ID(tmrand.Str(12)), 0, 10, func(err error, _ p2p.ID) { peerTestMtx.Lock() defer peerTestMtx.Unlock() @@ -75,7 +75,7 @@ func TestPeerRequestSent(t *testing.T) { params := &BpPeerParams{timeout: 2 * time.Millisecond} peer := NewBpPeer( - p2p.ID(tmrand.Str(12)), 10, + p2p.ID(tmrand.Str(12)), 0, 10, func(err error, _ p2p.ID) {}, params) @@ -94,7 +94,7 @@ func TestPeerRequestSent(t *testing.T) { func TestPeerGetAndRemoveBlock(t *testing.T) { peer := NewBpPeer( - p2p.ID(tmrand.Str(12)), 100, + p2p.ID(tmrand.Str(12)), 0, 100, func(err error, _ p2p.ID) {}, nil) @@ -142,7 +142,7 @@ func TestPeerGetAndRemoveBlock(t *testing.T) { func TestPeerAddBlock(t *testing.T) { peer := NewBpPeer( - p2p.ID(tmrand.Str(12)), 100, + p2p.ID(tmrand.Str(12)), 0, 100, func(err error, _ p2p.ID) {}, nil) @@ -189,7 +189,7 @@ func TestPeerOnErrFuncCalledDueToExpiration(t *testing.T) { ) peer := NewBpPeer( - p2p.ID(tmrand.Str(12)), 10, + p2p.ID(tmrand.Str(12)), 0, 10, func(err error, _ p2p.ID) { peerTestMtx.Lock() defer peerTestMtx.Unlock() @@ -215,7 +215,7 @@ func TestPeerCheckRate(t *testing.T) { minRecvRate: int64(100), // 100 bytes/sec exponential moving average } peer := NewBpPeer( - p2p.ID(tmrand.Str(12)), 10, + p2p.ID(tmrand.Str(12)), 0, 10, func(err error, _ p2p.ID) {}, params) peer.SetLogger(log.TestingLogger()) @@ -249,7 +249,7 @@ func TestPeerCleanup(t *testing.T) { params := &BpPeerParams{timeout: 2 * time.Millisecond} peer := NewBpPeer( - p2p.ID(tmrand.Str(12)), 10, + p2p.ID(tmrand.Str(12)), 0, 10, func(err error, _ p2p.ID) {}, params) peer.SetLogger(log.TestingLogger()) diff --git a/blockchain/v1/pool.go b/blockchain/v1/pool.go index be2edbc21..27e0f3a04 100644 --- a/blockchain/v1/pool.go +++ b/blockchain/v1/pool.go @@ -66,9 +66,9 @@ func (pool *BlockPool) updateMaxPeerHeight() { pool.MaxPeerHeight = newMax } -// UpdatePeer adds a new peer or updates an existing peer with a new height. +// UpdatePeer adds a new peer or updates an existing peer with a new base and height. // If a peer is short it is not added. -func (pool *BlockPool) UpdatePeer(peerID p2p.ID, height int64) error { +func (pool *BlockPool) UpdatePeer(peerID p2p.ID, base int64, height int64) error { peer := pool.peers[peerID] @@ -79,10 +79,10 @@ func (pool *BlockPool) UpdatePeer(peerID p2p.ID, height int64) error { return errPeerTooShort } // Add new peer. - peer = NewBpPeer(peerID, height, pool.toBcR.sendPeerError, nil) + peer = NewBpPeer(peerID, base, height, pool.toBcR.sendPeerError, nil) peer.SetLogger(pool.logger.With("peer", peerID)) pool.peers[peerID] = peer - pool.logger.Info("added peer", "peerID", peerID, "height", height, "num_peers", len(pool.peers)) + pool.logger.Info("added peer", "peerID", peerID, "base", base, "height", height, "num_peers", len(pool.peers)) } else { // Check if peer is lowering its height. This is not allowed. if height < peer.Height { @@ -90,6 +90,7 @@ func (pool *BlockPool) UpdatePeer(peerID p2p.ID, height int64) error { return errPeerLowersItsHeight } // Update existing peer. + peer.Base = base peer.Height = height } @@ -213,7 +214,7 @@ func (pool *BlockPool) sendRequest(height int64) bool { if peer.NumPendingBlockRequests >= maxRequestsPerPeer { continue } - if peer.Height < height { + if peer.Base > height || peer.Height < height { continue } diff --git a/blockchain/v1/pool_test.go b/blockchain/v1/pool_test.go index e612eb43e..31b9d09f7 100644 --- a/blockchain/v1/pool_test.go +++ b/blockchain/v1/pool_test.go @@ -13,6 +13,7 @@ import ( type testPeer struct { id p2p.ID + base int64 height int64 } @@ -70,7 +71,7 @@ func makeBlockPool(bcr *testBcR, height int64, peers []BpPeer, blocks map[int64] if p.Height > maxH { maxH = p.Height } - bPool.peers[p.ID] = NewBpPeer(p.ID, p.Height, bcr.sendPeerError, nil) + bPool.peers[p.ID] = NewBpPeer(p.ID, p.Base, p.Height, bcr.sendPeerError, nil) bPool.peers[p.ID].SetLogger(bcr.logger) } @@ -93,6 +94,7 @@ func assertPeerSetsEquivalent(t *testing.T, set1 map[p2p.ID]*BpPeer, set2 map[p2 assert.NotNil(t, peer2) assert.Equal(t, peer1.NumPendingBlockRequests, peer2.NumPendingBlockRequests) assert.Equal(t, peer1.Height, peer2.Height) + assert.Equal(t, peer1.Base, peer2.Base) assert.Equal(t, len(peer1.blocks), len(peer2.blocks)) for h, block1 := range peer1.blocks { block2 := peer2.blocks[h] @@ -123,26 +125,32 @@ func TestBlockPoolUpdatePeer(t *testing.T) { { name: "add a first short peer", pool: makeBlockPool(testBcR, 100, []BpPeer{}, map[int64]tPBlocks{}), - args: testPeer{"P1", 50}, + args: testPeer{"P1", 0, 50}, errWanted: errPeerTooShort, poolWanted: makeBlockPool(testBcR, 100, []BpPeer{}, map[int64]tPBlocks{}), }, { name: "add a first good peer", pool: makeBlockPool(testBcR, 100, []BpPeer{}, map[int64]tPBlocks{}), - args: testPeer{"P1", 101}, + args: testPeer{"P1", 0, 101}, poolWanted: makeBlockPool(testBcR, 100, []BpPeer{{ID: "P1", Height: 101}}, map[int64]tPBlocks{}), }, + { + name: "add a first good peer with base", + pool: makeBlockPool(testBcR, 100, []BpPeer{}, map[int64]tPBlocks{}), + args: testPeer{"P1", 10, 101}, + poolWanted: makeBlockPool(testBcR, 100, []BpPeer{{ID: "P1", Base: 10, Height: 101}}, map[int64]tPBlocks{}), + }, { name: "increase the height of P1 from 120 to 123", pool: makeBlockPool(testBcR, 100, []BpPeer{{ID: "P1", Height: 120}}, map[int64]tPBlocks{}), - args: testPeer{"P1", 123}, + args: testPeer{"P1", 0, 123}, poolWanted: makeBlockPool(testBcR, 100, []BpPeer{{ID: "P1", Height: 123}}, map[int64]tPBlocks{}), }, { name: "decrease the height of P1 from 120 to 110", pool: makeBlockPool(testBcR, 100, []BpPeer{{ID: "P1", Height: 120}}, map[int64]tPBlocks{}), - args: testPeer{"P1", 110}, + args: testPeer{"P1", 0, 110}, errWanted: errPeerLowersItsHeight, poolWanted: makeBlockPool(testBcR, 100, []BpPeer{}, map[int64]tPBlocks{}), }, @@ -151,7 +159,7 @@ func TestBlockPoolUpdatePeer(t *testing.T) { pool: makeBlockPool(testBcR, 100, []BpPeer{{ID: "P1", Height: 105}}, map[int64]tPBlocks{ 100: {"P1", true}, 101: {"P1", true}, 102: {"P1", true}}), - args: testPeer{"P1", 102}, + args: testPeer{"P1", 0, 102}, errWanted: errPeerLowersItsHeight, poolWanted: makeBlockPool(testBcR, 100, []BpPeer{}, map[int64]tPBlocks{}), @@ -162,7 +170,7 @@ func TestBlockPoolUpdatePeer(t *testing.T) { tt := tt t.Run(tt.name, func(t *testing.T) { pool := tt.pool - err := pool.UpdatePeer(tt.args.id, tt.args.height) + err := pool.UpdatePeer(tt.args.id, tt.args.base, tt.args.height) assert.Equal(t, tt.errWanted, err) assert.Equal(t, tt.poolWanted.blocks, tt.pool.blocks) assertPeerSetsEquivalent(t, tt.poolWanted.peers, tt.pool.peers) @@ -300,20 +308,34 @@ func TestBlockPoolSendRequestBatch(t *testing.T) { testBcR := newTestBcR() tests := []struct { - name string - pool *BlockPool - maxRequestsPerPeer int - expRequests map[int64]bool - expPeerResults []testPeerResult - expnumPendingBlockRequests int + name string + pool *BlockPool + maxRequestsPerPeer int + expRequests map[int64]bool + expRequestsSent int + expPeerResults []testPeerResult }{ { - name: "one peer - send up to maxRequestsPerPeer block requests", - pool: makeBlockPool(testBcR, 10, []BpPeer{{ID: "P1", Height: 100}}, map[int64]tPBlocks{}), - maxRequestsPerPeer: 2, - expRequests: map[int64]bool{10: true, 11: true}, - expPeerResults: []testPeerResult{{id: "P1", numPendingBlockRequests: 2}}, - expnumPendingBlockRequests: 2, + name: "one peer - send up to maxRequestsPerPeer block requests", + pool: makeBlockPool(testBcR, 10, []BpPeer{{ID: "P1", Height: 100}}, map[int64]tPBlocks{}), + maxRequestsPerPeer: 2, + expRequests: map[int64]bool{10: true, 11: true}, + expRequestsSent: 2, + expPeerResults: []testPeerResult{{id: "P1", numPendingBlockRequests: 2}}, + }, + { + name: "multiple peers - stops at gap between height and base", + pool: makeBlockPool(testBcR, 10, []BpPeer{ + {ID: "P1", Base: 1, Height: 12}, + {ID: "P2", Base: 15, Height: 100}, + }, map[int64]tPBlocks{}), + maxRequestsPerPeer: 10, + expRequests: map[int64]bool{10: true, 11: true, 12: true}, + expRequestsSent: 3, + expPeerResults: []testPeerResult{ + {id: "P1", numPendingBlockRequests: 3}, + {id: "P2", numPendingBlockRequests: 0}, + }, }, { name: "n peers - send n*maxRequestsPerPeer block requests", @@ -324,10 +346,10 @@ func TestBlockPoolSendRequestBatch(t *testing.T) { map[int64]tPBlocks{}), maxRequestsPerPeer: 2, expRequests: map[int64]bool{10: true, 11: true}, + expRequestsSent: 4, expPeerResults: []testPeerResult{ {id: "P1", numPendingBlockRequests: 2}, {id: "P2", numPendingBlockRequests: 2}}, - expnumPendingBlockRequests: 4, }, } @@ -339,15 +361,13 @@ func TestBlockPoolSendRequestBatch(t *testing.T) { var pool = tt.pool maxRequestsPerPeer = tt.maxRequestsPerPeer pool.MakeNextRequests(10) - assert.Equal(t, testResults.numRequestsSent, maxRequestsPerPeer*len(pool.peers)) + assert.Equal(t, tt.expRequestsSent, testResults.numRequestsSent) for _, tPeer := range tt.expPeerResults { var peer = pool.peers[tPeer.id] assert.NotNil(t, peer) assert.Equal(t, tPeer.numPendingBlockRequests, peer.NumPendingBlockRequests) } - assert.Equal(t, testResults.numRequestsSent, maxRequestsPerPeer*len(pool.peers)) - }) } } diff --git a/blockchain/v1/reactor.go b/blockchain/v1/reactor.go index d3b9b0216..28a314b8a 100644 --- a/blockchain/v1/reactor.go +++ b/blockchain/v1/reactor.go @@ -169,7 +169,10 @@ func (bcR *BlockchainReactor) GetChannels() []*p2p.ChannelDescriptor { // AddPeer implements Reactor by sending our state to peer. func (bcR *BlockchainReactor) AddPeer(peer p2p.Peer) { - msgBytes := cdc.MustMarshalBinaryBare(&bcStatusResponseMessage{bcR.store.Height()}) + msgBytes := cdc.MustMarshalBinaryBare(&bcStatusResponseMessage{ + Base: bcR.store.Base(), + Height: bcR.store.Height(), + }) peer.Send(BlockchainChannel, msgBytes) // it's OK if send fails. will try later in poolRoutine @@ -196,7 +199,10 @@ func (bcR *BlockchainReactor) sendBlockToPeer(msg *bcBlockRequestMessage, } func (bcR *BlockchainReactor) sendStatusResponseToPeer(msg *bcStatusRequestMessage, src p2p.Peer) (queued bool) { - msgBytes := cdc.MustMarshalBinaryBare(&bcStatusResponseMessage{bcR.store.Height()}) + msgBytes := cdc.MustMarshalBinaryBare(&bcStatusResponseMessage{ + Base: bcR.store.Base(), + Height: bcR.store.Height(), + }) return src.TrySend(BlockchainChannel, msgBytes) } @@ -430,7 +436,7 @@ func (bcR *BlockchainReactor) processBlock() error { bcR.store.SaveBlock(first, firstParts, second.LastCommit) - bcR.state, err = bcR.blockExec.ApplyBlock(bcR.state, firstID, first) + bcR.state, _, err = bcR.blockExec.ApplyBlock(bcR.state, firstID, first) if err != nil { panic(fmt.Sprintf("failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err)) } @@ -441,7 +447,10 @@ func (bcR *BlockchainReactor) processBlock() error { // Implements bcRNotifier // sendStatusRequest broadcasts `BlockStore` height. func (bcR *BlockchainReactor) sendStatusRequest() { - msgBytes := cdc.MustMarshalBinaryBare(&bcStatusRequestMessage{bcR.store.Height()}) + msgBytes := cdc.MustMarshalBinaryBare(&bcStatusRequestMessage{ + Base: bcR.store.Base(), + Height: bcR.store.Height(), + }) bcR.Switch.Broadcast(BlockchainChannel, msgBytes) } @@ -590,6 +599,7 @@ func (m *bcBlockResponseMessage) String() string { type bcStatusRequestMessage struct { Height int64 + Base int64 } // ValidateBasic performs basic validation. @@ -597,17 +607,24 @@ func (m *bcStatusRequestMessage) ValidateBasic() error { if m.Height < 0 { return errors.New("negative Height") } + if m.Base < 0 { + return errors.New("negative Base") + } + if m.Base > m.Height { + return fmt.Errorf("base %v cannot be greater than height %v", m.Base, m.Height) + } return nil } func (m *bcStatusRequestMessage) String() string { - return fmt.Sprintf("[bcStatusRequestMessage %v]", m.Height) + return fmt.Sprintf("[bcStatusRequestMessage %v:%v]", m.Base, m.Height) } //------------------------------------- type bcStatusResponseMessage struct { Height int64 + Base int64 } // ValidateBasic performs basic validation. @@ -615,9 +632,15 @@ func (m *bcStatusResponseMessage) ValidateBasic() error { if m.Height < 0 { return errors.New("negative Height") } + if m.Base < 0 { + return errors.New("negative Base") + } + if m.Base > m.Height { + return fmt.Errorf("base %v cannot be greater than height %v", m.Base, m.Height) + } return nil } func (m *bcStatusResponseMessage) String() string { - return fmt.Sprintf("[bcStatusResponseMessage %v]", m.Height) + return fmt.Sprintf("[bcStatusResponseMessage %v:%v]", m.Base, m.Height) } diff --git a/blockchain/v1/reactor_fsm.go b/blockchain/v1/reactor_fsm.go index 8d3a363ae..0f65f9d66 100644 --- a/blockchain/v1/reactor_fsm.go +++ b/blockchain/v1/reactor_fsm.go @@ -58,6 +58,7 @@ func NewFSM(height int64, toBcR bcReactor) *BcReactorFSM { type bReactorEventData struct { peerID p2p.ID err error // for peer error: timeout, slow; for processed block event if error occurred + base int64 // for status response height int64 // for status response; for processed block event block *types.Block // for block response stateName string // for state timeout events @@ -89,7 +90,7 @@ func (msg *bcReactorMessage) String() string { case startFSMEv: dataStr = "" case statusResponseEv: - dataStr = fmt.Sprintf("peer=%v height=%v", msg.data.peerID, msg.data.height) + dataStr = fmt.Sprintf("peer=%v base=%v height=%v", msg.data.peerID, msg.data.base, msg.data.height) case blockResponseEv: dataStr = fmt.Sprintf("peer=%v block.height=%v length=%v", msg.data.peerID, msg.data.block.Height, msg.data.length) @@ -213,7 +214,7 @@ func init() { return finished, errNoTallerPeer case statusResponseEv: - if err := fsm.pool.UpdatePeer(data.peerID, data.height); err != nil { + if err := fsm.pool.UpdatePeer(data.peerID, data.base, data.height); err != nil { if fsm.pool.NumPeers() == 0 { return waitForPeer, err } @@ -246,7 +247,7 @@ func init() { switch ev { case statusResponseEv: - err := fsm.pool.UpdatePeer(data.peerID, data.height) + err := fsm.pool.UpdatePeer(data.peerID, data.base, data.height) if fsm.pool.NumPeers() == 0 { return waitForPeer, err } diff --git a/blockchain/v1/reactor_test.go b/blockchain/v1/reactor_test.go index ec9707f67..e0b3472bf 100644 --- a/blockchain/v1/reactor_test.go +++ b/blockchain/v1/reactor_test.go @@ -131,7 +131,7 @@ func newBlockchainReactor( thisParts := thisBlock.MakePartSet(types.BlockPartSizeBytes) blockID := types.BlockID{Hash: thisBlock.Hash(), PartsHeader: thisParts.Header()} - state, err = blockExec.ApplyBlock(state, blockID, thisBlock) + state, _, err = blockExec.ApplyBlock(state, blockID, thisBlock) if err != nil { panic(errors.Wrap(err, "error apply block")) } diff --git a/blockchain/v2/io.go b/blockchain/v2/io.go index 3db48c8c0..fde446f6e 100644 --- a/blockchain/v2/io.go +++ b/blockchain/v2/io.go @@ -14,7 +14,7 @@ type iIO interface { sendBlockNotFound(height int64, peerID p2p.ID) error sendStatusResponse(height int64, peerID p2p.ID) error - broadcastStatusRequest(height int64) + broadcastStatusRequest(base int64, height int64) trySwitchToConsensus(state state.State, blocksSynced int) } @@ -104,8 +104,11 @@ func (sio *switchIO) trySwitchToConsensus(state state.State, blocksSynced int) { } } -func (sio *switchIO) broadcastStatusRequest(height int64) { - msgBytes := cdc.MustMarshalBinaryBare(&bcStatusRequestMessage{height}) +func (sio *switchIO) broadcastStatusRequest(base int64, height int64) { + msgBytes := cdc.MustMarshalBinaryBare(&bcStatusRequestMessage{ + Base: base, + Height: height, + }) // XXX: maybe we should use an io specific peer list here sio.sw.Broadcast(BlockchainChannel, msgBytes) } diff --git a/blockchain/v2/processor_context.go b/blockchain/v2/processor_context.go index 7e96a3a69..2e8142adc 100644 --- a/blockchain/v2/processor_context.go +++ b/blockchain/v2/processor_context.go @@ -29,7 +29,7 @@ func newProcessorContext(st blockStore, ex blockApplier, s state.State) *pContex } func (pc *pContext) applyBlock(blockID types.BlockID, block *types.Block) error { - newState, err := pc.applier.ApplyBlock(pc.state, blockID, block) + newState, _, err := pc.applier.ApplyBlock(pc.state, blockID, block) pc.state = newState return err } diff --git a/blockchain/v2/reactor.go b/blockchain/v2/reactor.go index e09e0adde..88ec6268d 100644 --- a/blockchain/v2/reactor.go +++ b/blockchain/v2/reactor.go @@ -72,41 +72,56 @@ func (m *bcBlockResponseMessage) String() string { type bcStatusRequestMessage struct { Height int64 + Base int64 } // ValidateBasic performs basic validation. func (m *bcStatusRequestMessage) ValidateBasic() error { + if m.Base < 0 { + return errors.New("negative Base") + } if m.Height < 0 { return errors.New("negative Height") } + if m.Base > m.Height { + return fmt.Errorf("base %v cannot be greater than height %v", m.Base, m.Height) + } return nil } func (m *bcStatusRequestMessage) String() string { - return fmt.Sprintf("[bcStatusRequestMessage %v]", m.Height) + return fmt.Sprintf("[bcStatusRequestMessage %v:%v]", m.Base, m.Height) } //------------------------------------- type bcStatusResponseMessage struct { Height int64 + Base int64 } // ValidateBasic performs basic validation. func (m *bcStatusResponseMessage) ValidateBasic() error { + if m.Base < 0 { + return errors.New("negative Base") + } if m.Height < 0 { return errors.New("negative Height") } + if m.Base > m.Height { + return fmt.Errorf("base %v cannot be greater than height %v", m.Base, m.Height) + } return nil } func (m *bcStatusResponseMessage) String() string { - return fmt.Sprintf("[bcStatusResponseMessage %v]", m.Height) + return fmt.Sprintf("[bcStatusResponseMessage %v:%v]", m.Base, m.Height) } type blockStore interface { LoadBlock(height int64) *types.Block SaveBlock(*types.Block, *types.PartSet, *types.Commit) + Base() int64 Height() int64 } @@ -136,7 +151,7 @@ type blockVerifier interface { //nolint:deadcode type blockApplier interface { - ApplyBlock(state state.State, blockID types.BlockID, block *types.Block) (state.State, error) + ApplyBlock(state state.State, blockID types.BlockID, block *types.Block) (state.State, int64, error) } // XXX: unify naming in this package around tmState @@ -266,6 +281,7 @@ type bcStatusResponse struct { priorityNormal time time.Time peerID p2p.ID + base int64 height int64 } @@ -337,7 +353,7 @@ func (r *BlockchainReactor) demux() { case <-doProcessBlockCh: r.processor.send(rProcessBlock{}) case <-doStatusCh: - r.io.broadcastStatusRequest(r.SyncHeight()) + r.io.broadcastStatusRequest(r.store.Base(), r.SyncHeight()) // Events from peers case event := <-r.events: @@ -483,7 +499,7 @@ func (r *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) { } case *bcStatusResponseMessage: - r.events <- bcStatusResponse{peerID: src.ID(), height: msg.Height} + r.events <- bcStatusResponse{peerID: src.ID(), base: msg.Base, height: msg.Height} case *bcBlockResponseMessage: r.events <- bcBlockResponse{ diff --git a/blockchain/v2/reactor_test.go b/blockchain/v2/reactor_test.go index 1d3e51e77..108f6f500 100644 --- a/blockchain/v2/reactor_test.go +++ b/blockchain/v2/reactor_test.go @@ -77,9 +77,9 @@ type mockBlockApplier struct { } // XXX: Add whitelist/blacklist? -func (mba *mockBlockApplier) ApplyBlock(state sm.State, blockID types.BlockID, block *types.Block) (sm.State, error) { +func (mba *mockBlockApplier) ApplyBlock(state sm.State, blockID types.BlockID, block *types.Block) (sm.State, int64, error) { state.LastBlockHeight++ - return state, nil + return state, 0, nil } type mockSwitchIo struct { @@ -127,7 +127,7 @@ func (sio *mockSwitchIo) hasSwitchedToConsensus() bool { return sio.switchedToConsensus } -func (sio *mockSwitchIo) broadcastStatusRequest(height int64) { +func (sio *mockSwitchIo) broadcastStatusRequest(base int64, height int64) { } type testReactorParams struct { @@ -511,7 +511,7 @@ func newReactorStore( thisParts := thisBlock.MakePartSet(types.BlockPartSizeBytes) blockID := types.BlockID{Hash: thisBlock.Hash(), PartsHeader: thisParts.Header()} - state, err = blockExec.ApplyBlock(state, blockID, thisBlock) + state, _, err = blockExec.ApplyBlock(state, blockID, thisBlock) if err != nil { panic(errors.Wrap(err, "error apply block")) } diff --git a/blockchain/v2/scheduler.go b/blockchain/v2/scheduler.go index 3cf0b2468..803955b22 100644 --- a/blockchain/v2/scheduler.go +++ b/blockchain/v2/scheduler.go @@ -111,20 +111,22 @@ type scPeer struct { // updated to Removed when peer is removed state peerState + base int64 // updated when statusResponse is received height int64 // updated when statusResponse is received lastTouched time.Time lastRate int64 // last receive rate in bytes } func (p scPeer) String() string { - return fmt.Sprintf("{state %v, height %d, lastTouched %v, lastRate %d, id %v}", - p.state, p.height, p.lastTouched, p.lastRate, p.peerID) + return fmt.Sprintf("{state %v, base %d, height %d, lastTouched %v, lastRate %d, id %v}", + p.state, p.base, p.height, p.lastTouched, p.lastRate, p.peerID) } func newScPeer(peerID p2p.ID) *scPeer { return &scPeer{ peerID: peerID, state: peerStateNew, + base: -1, height: -1, lastTouched: time.Time{}, } @@ -280,7 +282,7 @@ func (sc *scheduler) addNewBlocks() { } } -func (sc *scheduler) setPeerHeight(peerID p2p.ID, height int64) error { +func (sc *scheduler) setPeerRange(peerID p2p.ID, base int64, height int64) error { peer, ok := sc.peers[peerID] if !ok { return fmt.Errorf("cannot find peer %s", peerID) @@ -295,6 +297,11 @@ func (sc *scheduler) setPeerHeight(peerID p2p.ID, height int64) error { return fmt.Errorf("cannot move peer height lower. from %d to %d", peer.height, height) } + if base > height { + return fmt.Errorf("cannot set peer base higher than its height") + } + + peer.base = base peer.height = height peer.state = peerStateReady @@ -312,13 +319,13 @@ func (sc *scheduler) getStateAtHeight(height int64) blockState { } } -func (sc *scheduler) getPeersAtHeightOrAbove(height int64) []p2p.ID { +func (sc *scheduler) getPeersWithHeight(height int64) []p2p.ID { peers := make([]p2p.ID, 0) for _, peer := range sc.peers { if peer.state != peerStateReady { continue } - if peer.height >= height { + if peer.base <= height && peer.height >= height { peers = append(peers, peer.peerID) } } @@ -395,6 +402,11 @@ func (sc *scheduler) markPending(peerID p2p.ID, height int64, time time.Time) er height, peerID, peer.height) } + if height < peer.base { + return fmt.Errorf("cannot request height %d for peer %s with base %d", + height, peerID, peer.base) + } + sc.setStateAtHeight(height, blockStatePending) sc.pendingBlocks[height] = peerID sc.pendingTime[height] = time @@ -463,7 +475,7 @@ func (sc *scheduler) pendingFrom(peerID p2p.ID) []int64 { } func (sc *scheduler) selectPeer(height int64) (p2p.ID, error) { - peers := sc.getPeersAtHeightOrAbove(height) + peers := sc.getPeersWithHeight(height) if len(peers) == 0 { return "", fmt.Errorf("cannot find peer for height %d", height) } @@ -535,8 +547,8 @@ func (sc *scheduler) handleNoBlockResponse(event bcNoBlockResponse) (Event, erro _ = sc.removePeer(event.peerID) return scPeerError{peerID: event.peerID, - reason: fmt.Errorf("peer %v with height %d claims no block for %d", - event.peerID, peer.height, event.height)}, nil + reason: fmt.Errorf("peer %v with base %d height %d claims no block for %d", + event.peerID, peer.base, peer.height, event.height)}, nil } func (sc *scheduler) handleBlockProcessed(event pcBlockProcessed) (Event, error) { @@ -653,7 +665,7 @@ func (sc *scheduler) handleTrySchedule(event rTrySchedule) (Event, error) { } func (sc *scheduler) handleStatusResponse(event bcStatusResponse) (Event, error) { - err := sc.setPeerHeight(event.peerID, event.height) + err := sc.setPeerRange(event.peerID, event.base, event.height) if err != nil { return scPeerError{peerID: event.peerID, reason: err}, nil } diff --git a/blockchain/v2/scheduler_test.go b/blockchain/v2/scheduler_test.go index c91fdd366..4ec81e123 100644 --- a/blockchain/v2/scheduler_test.go +++ b/blockchain/v2/scheduler_test.go @@ -145,8 +145,8 @@ func TestScMaxHeights(t *testing.T) { sc: scheduler{ height: 1, peers: map[p2p.ID]*scPeer{ - "P1": {height: -1, state: peerStateNew}, - "P2": {height: -1, state: peerStateNew}}, + "P1": {base: -1, height: -1, state: peerStateNew}, + "P2": {base: -1, height: -1, state: peerStateNew}}, }, wantMax: 0, }, @@ -194,15 +194,15 @@ func TestScAddPeer(t *testing.T) { name: "add first peer", fields: scTestParams{}, args: args{peerID: "P1"}, - wantFields: scTestParams{peers: map[string]*scPeer{"P1": {height: -1, state: peerStateNew}}}, + wantFields: scTestParams{peers: map[string]*scPeer{"P1": {base: -1, height: -1, state: peerStateNew}}}, }, { name: "add second peer", - fields: scTestParams{peers: map[string]*scPeer{"P1": {height: -1, state: peerStateNew}}}, + fields: scTestParams{peers: map[string]*scPeer{"P1": {base: -1, height: -1, state: peerStateNew}}}, args: args{peerID: "P2"}, wantFields: scTestParams{peers: map[string]*scPeer{ - "P1": {height: -1, state: peerStateNew}, - "P2": {height: -1, state: peerStateNew}}}, + "P1": {base: -1, height: -1, state: peerStateNew}, + "P2": {base: -1, height: -1, state: peerStateNew}}}, }, { name: "attempt to add duplicate peer", @@ -501,10 +501,11 @@ func TestScRemovePeer(t *testing.T) { } } -func TestScSetPeerHeight(t *testing.T) { +func TestScSetPeerRange(t *testing.T) { type args struct { peerID p2p.ID + base int64 height int64 } tests := []struct { @@ -576,13 +577,37 @@ func TestScSetPeerHeight(t *testing.T) { peers: map[string]*scPeer{"P2": {height: 10000000000, state: peerStateReady}}, allB: []int64{1, 2, 3, 4}}, }, + { + name: "add peer with base > height should error", + fields: scTestParams{ + peers: map[string]*scPeer{"P1": {height: 4, state: peerStateReady}}, + allB: []int64{1, 2, 3, 4}}, + args: args{peerID: "P1", base: 6, height: 5}, + wantFields: scTestParams{ + peers: map[string]*scPeer{"P1": {height: 4, state: peerStateReady}}, + allB: []int64{1, 2, 3, 4}}, + wantErr: true, + }, + { + name: "add peer with base == height is fine", + fields: scTestParams{ + peers: map[string]*scPeer{"P1": {height: 4, state: peerStateNew}}, + targetPending: 4, + }, + args: args{peerID: "P1", base: 6, height: 6}, + wantFields: scTestParams{ + targetPending: 4, + peers: map[string]*scPeer{"P1": {base: 6, height: 6, state: peerStateReady}}, + allB: []int64{1, 2, 3, 4}}, + }, } for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { sc := newTestScheduler(tt.fields) - if err := sc.setPeerHeight(tt.args.peerID, tt.args.height); (err != nil) != tt.wantErr { + err := sc.setPeerRange(tt.args.peerID, tt.args.base, tt.args.height) + if (err != nil) != tt.wantErr { t.Errorf("setPeerHeight() wantErr %v, error = %v", tt.wantErr, err) } wantSc := newTestScheduler(tt.wantFields) @@ -591,7 +616,7 @@ func TestScSetPeerHeight(t *testing.T) { } } -func TestScGetPeersAtHeight(t *testing.T) { +func TestScGetPeersWithHeight(t *testing.T) { type args struct { height int64 @@ -648,6 +673,26 @@ func TestScGetPeersAtHeight(t *testing.T) { args: args{height: 4}, wantResult: []p2p.ID{"P1"}, }, + { + name: "one Ready higher peer at base", + fields: scTestParams{ + targetPending: 4, + peers: map[string]*scPeer{"P1": {base: 4, height: 20, state: peerStateReady}}, + allB: []int64{1, 2, 3, 4}, + }, + args: args{height: 4}, + wantResult: []p2p.ID{"P1"}, + }, + { + name: "one Ready higher peer with higher base", + fields: scTestParams{ + targetPending: 4, + peers: map[string]*scPeer{"P1": {base: 10, height: 20, state: peerStateReady}}, + allB: []int64{1, 2, 3, 4}, + }, + args: args{height: 4}, + wantResult: []p2p.ID{}, + }, { name: "multiple mixed peers", fields: scTestParams{ @@ -669,9 +714,9 @@ func TestScGetPeersAtHeight(t *testing.T) { tt := tt t.Run(tt.name, func(t *testing.T) { sc := newTestScheduler(tt.fields) - // getPeersAtHeight should not mutate the scheduler + // getPeersWithHeight should not mutate the scheduler wantSc := sc - res := sc.getPeersAtHeightOrAbove(tt.args.height) + res := sc.getPeersWithHeight(tt.args.height) sort.Sort(PeerByID(res)) assert.Equal(t, tt.wantResult, res) assert.Equal(t, wantSc, sc) @@ -695,7 +740,7 @@ func TestScMarkPending(t *testing.T) { wantErr bool }{ { - name: "attempt mark pending an unknown block", + name: "attempt mark pending an unknown block above height", fields: scTestParams{ peers: map[string]*scPeer{"P1": {height: 2, state: peerStateReady}}, allB: []int64{1, 2}}, @@ -705,6 +750,17 @@ func TestScMarkPending(t *testing.T) { allB: []int64{1, 2}}, wantErr: true, }, + { + name: "attempt mark pending an unknown block below base", + fields: scTestParams{ + peers: map[string]*scPeer{"P1": {base: 4, height: 6, state: peerStateReady}}, + allB: []int64{1, 2, 3, 4, 5, 6}}, + args: args{peerID: "P1", height: 3, tm: now}, + wantFields: scTestParams{ + peers: map[string]*scPeer{"P1": {base: 4, height: 6, state: peerStateReady}}, + allB: []int64{1, 2, 3, 4, 5, 6}}, + wantErr: true, + }, { name: "attempt mark pending from non existing peer", fields: scTestParams{ @@ -1202,6 +1258,16 @@ func TestScSelectPeer(t *testing.T) { args: args{height: 4}, wantResult: "P1", }, + { + name: "one Ready higher peer with higher base", + fields: scTestParams{ + peers: map[string]*scPeer{"P1": {base: 4, height: 6, state: peerStateReady}}, + allB: []int64{1, 2, 3, 4, 5, 6}, + }, + args: args{height: 3}, + wantResult: "", + wantError: true, + }, { name: "many Ready higher peers with different number of pending requests", fields: scTestParams{ @@ -1990,7 +2056,7 @@ func TestScHandle(t *testing.T) { args: args{event: bcAddNewPeer{peerID: "P1"}}, wantEvent: noOpEvent{}, wantSc: &scTestParams{startTime: now, peers: map[string]*scPeer{ - "P1": {height: -1, state: peerStateNew}}, height: 1}, + "P1": {base: -1, height: -1, state: peerStateNew}}, height: 1}, }, { // set height of P1 args: args{event: bcStatusResponse{peerID: "P1", time: tick[0], height: 3}}, diff --git a/config/config_test.go b/config/config_test.go index 6da032d07..c83f1c3f5 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -134,27 +134,44 @@ func TestFastSyncConfigValidateBasic(t *testing.T) { assert.Error(t, cfg.ValidateBasic()) } -func TestConsensusConfigValidateBasic(t *testing.T) { - cfg := TestConsensusConfig() - assert.NoError(t, cfg.ValidateBasic()) - - fieldsToTest := []string{ - "TimeoutPropose", - "TimeoutProposeDelta", - "TimeoutPrevote", - "TimeoutPrevoteDelta", - "TimeoutPrecommit", - "TimeoutPrecommitDelta", - "TimeoutCommit", - "CreateEmptyBlocksInterval", - "PeerGossipSleepDuration", - "PeerQueryMaj23SleepDuration", +func TestConsensusConfig_ValidateBasic(t *testing.T) { + // nolint: lll + testcases := map[string]struct { + modify func(*ConsensusConfig) + expectErr bool + }{ + "TimeoutPropose": {func(c *ConsensusConfig) { c.TimeoutPropose = time.Second }, false}, + "TimeoutPropose negative": {func(c *ConsensusConfig) { c.TimeoutPropose = -1 }, true}, + "TimeoutProposeDelta": {func(c *ConsensusConfig) { c.TimeoutProposeDelta = time.Second }, false}, + "TimeoutProposeDelta negative": {func(c *ConsensusConfig) { c.TimeoutProposeDelta = -1 }, true}, + "TimeoutPrevote": {func(c *ConsensusConfig) { c.TimeoutPrevote = time.Second }, false}, + "TimeoutPrevote negative": {func(c *ConsensusConfig) { c.TimeoutPrevote = -1 }, true}, + "TimeoutPrevoteDelta": {func(c *ConsensusConfig) { c.TimeoutPrevoteDelta = time.Second }, false}, + "TimeoutPrevoteDelta negative": {func(c *ConsensusConfig) { c.TimeoutPrevoteDelta = -1 }, true}, + "TimeoutPrecommit": {func(c *ConsensusConfig) { c.TimeoutPrecommit = time.Second }, false}, + "TimeoutPrecommit negative": {func(c *ConsensusConfig) { c.TimeoutPrecommit = -1 }, true}, + "TimeoutPrecommitDelta": {func(c *ConsensusConfig) { c.TimeoutPrecommitDelta = time.Second }, false}, + "TimeoutPrecommitDelta negative": {func(c *ConsensusConfig) { c.TimeoutPrecommitDelta = -1 }, true}, + "TimeoutCommit": {func(c *ConsensusConfig) { c.TimeoutCommit = time.Second }, false}, + "TimeoutCommit negative": {func(c *ConsensusConfig) { c.TimeoutCommit = -1 }, true}, + "PeerGossipSleepDuration": {func(c *ConsensusConfig) { c.PeerGossipSleepDuration = time.Second }, false}, + "PeerGossipSleepDuration negative": {func(c *ConsensusConfig) { c.PeerGossipSleepDuration = -1 }, true}, + "PeerQueryMaj23SleepDuration": {func(c *ConsensusConfig) { c.PeerQueryMaj23SleepDuration = time.Second }, false}, + "PeerQueryMaj23SleepDuration negative": {func(c *ConsensusConfig) { c.PeerQueryMaj23SleepDuration = -1 }, true}, } - - for _, fieldName := range fieldsToTest { - reflect.ValueOf(cfg).Elem().FieldByName(fieldName).SetInt(-1) - assert.Error(t, cfg.ValidateBasic()) - reflect.ValueOf(cfg).Elem().FieldByName(fieldName).SetInt(0) + for desc, tc := range testcases { + tc := tc // appease linter + t.Run(desc, func(t *testing.T) { + cfg := DefaultConsensusConfig() + tc.modify(cfg) + + err := cfg.ValidateBasic() + if tc.expectErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) } } diff --git a/consensus/reactor.go b/consensus/reactor.go index 3182bba89..c8c344ac8 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -494,8 +494,8 @@ OUTER_LOOP: } } - // If the peer is on a previous height, help catch up. - if (0 < prs.Height) && (prs.Height < rs.Height) { + // If the peer is on a previous height that we have, help catch up. + if (0 < prs.Height) && (prs.Height < rs.Height) && (prs.Height >= conR.conS.blockStore.Base()) { heightLogger := logger.With("height", prs.Height) // if we never received the commit message from the peer, the block parts wont be initialized @@ -503,7 +503,7 @@ OUTER_LOOP: blockMeta := conR.conS.blockStore.LoadBlockMeta(prs.Height) if blockMeta == nil { heightLogger.Error("Failed to load block meta", - "blockstoreHeight", conR.conS.blockStore.Height()) + "blockstoreBase", conR.conS.blockStore.Base(), "blockstoreHeight", conR.conS.blockStore.Height()) time.Sleep(conR.conS.config.PeerGossipSleepDuration) } else { ps.InitProposalBlockParts(blockMeta.BlockID.PartsHeader) @@ -567,8 +567,8 @@ func (conR *Reactor) gossipDataForCatchup(logger log.Logger, rs *cstypes.RoundSt // Ensure that the peer's PartSetHeader is correct blockMeta := conR.conS.blockStore.LoadBlockMeta(prs.Height) if blockMeta == nil { - logger.Error("Failed to load block meta", - "ourHeight", rs.Height, "blockstoreHeight", conR.conS.blockStore.Height()) + logger.Error("Failed to load block meta", "ourHeight", rs.Height, + "blockstoreBase", conR.conS.blockStore.Base(), "blockstoreHeight", conR.conS.blockStore.Height()) time.Sleep(conR.conS.config.PeerGossipSleepDuration) return } else if !blockMeta.BlockID.PartsHeader.Equals(prs.ProposalBlockPartsHeader) { @@ -803,15 +803,17 @@ OUTER_LOOP: // Maybe send Height/CatchupCommitRound/CatchupCommit. { prs := ps.GetRoundState() - if prs.CatchupCommitRound != -1 && 0 < prs.Height && prs.Height <= conR.conS.blockStore.Height() { - commit := conR.conS.LoadCommit(prs.Height) - peer.TrySend(StateChannel, cdc.MustMarshalBinaryBare(&VoteSetMaj23Message{ - Height: prs.Height, - Round: commit.Round, - Type: types.PrecommitType, - BlockID: commit.BlockID, - })) - time.Sleep(conR.conS.config.PeerQueryMaj23SleepDuration) + if prs.CatchupCommitRound != -1 && prs.Height > 0 && prs.Height <= conR.conS.blockStore.Height() && + prs.Height >= conR.conS.blockStore.Base() { + if commit := conR.conS.LoadCommit(prs.Height); commit != nil { + peer.TrySend(StateChannel, cdc.MustMarshalBinaryBare(&VoteSetMaj23Message{ + Height: prs.Height, + Round: commit.Round, + Type: types.PrecommitType, + BlockID: commit.BlockID, + })) + time.Sleep(conR.conS.config.PeerQueryMaj23SleepDuration) + } } } diff --git a/consensus/replay.go b/consensus/replay.go index 1453849cc..3d9d6614b 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -288,6 +288,7 @@ func (h *Handshaker) ReplayBlocks( appBlockHeight int64, proxyApp proxy.AppConns, ) ([]byte, error) { + storeBlockBase := h.store.Base() storeBlockHeight := h.store.Height() stateBlockHeight := state.LastBlockHeight h.logger.Info( @@ -341,12 +342,16 @@ func (h *Handshaker) ReplayBlocks( } } - // First handle edge cases and constraints on the storeBlockHeight. + // First handle edge cases and constraints on the storeBlockHeight and storeBlockBase. switch { case storeBlockHeight == 0: assertAppHashEqualsOneFromState(appHash, state) return appHash, nil + case appBlockHeight < storeBlockBase-1: + // the app is too far behind truncated store (can be 1 behind since we replay the next) + return appHash, sm.ErrAppBlockHeightTooLow{AppHeight: appBlockHeight, StoreBase: storeBlockBase} + case storeBlockHeight < appBlockHeight: // the app should never be ahead of the store (but this is under app's control) return appHash, sm.ErrAppBlockHeightTooHigh{CoreHeight: storeBlockHeight, AppHeight: appBlockHeight} @@ -472,7 +477,7 @@ func (h *Handshaker) replayBlock(state sm.State, height int64, proxyApp proxy.Ap blockExec.SetEventBus(h.eventBus) var err error - state, err = blockExec.ApplyBlock(state, meta.BlockID, block) + state, _, err = blockExec.ApplyBlock(state, meta.BlockID, block) if err != nil { return sm.State{}, err } diff --git a/consensus/replay_test.go b/consensus/replay_test.go index bbb73c9be..f886cdeeb 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -307,7 +307,8 @@ var ( // 0 - all synced up // 1 - saved block but app and state are behind // 2 - save block and committed but state is behind -var modes = []uint{0, 1, 2} +// 3 - save block and committed with truncated block store and state behind +var modes = []uint{0, 1, 2, 3} // This is actually not a test, it's for storing validator change tx data for testHandshakeReplay func TestSimulateValidatorsChange(t *testing.T) { @@ -531,10 +532,10 @@ func TestHandshakeReplayAll(t *testing.T) { // Sync many, not from scratch func TestHandshakeReplaySome(t *testing.T) { for _, m := range modes { - testHandshakeReplay(t, config, 1, m, false) + testHandshakeReplay(t, config, 2, m, false) } for _, m := range modes { - testHandshakeReplay(t, config, 1, m, true) + testHandshakeReplay(t, config, 2, m, true) } } @@ -637,7 +638,7 @@ func testHandshakeReplay(t *testing.T, config *cfg.Config, nBlocks int, mode uin stateDB = dbm.NewMemDB() genisisState = sim.GenesisState config = sim.Config - chain = sim.Chain + chain = append([]*types.Block{}, sim.Chain...) // copy chain commits = sim.Commits store = newMockBlockStore(config, genisisState.ConsensusParams) } else { //test single node @@ -685,6 +686,15 @@ func testHandshakeReplay(t *testing.T, config *cfg.Config, nBlocks int, mode uin buildAppStateFromChain(proxyApp, stateDB1, genisisState, chain, nBlocks, mode) } + // Prune block store if requested + expectError := false + if mode == 3 { + pruned, err := store.PruneBlocks(2) + require.NoError(t, err) + require.EqualValues(t, 1, pruned) + expectError = int64(nBlocks) < 2 + } + // now start the app using the handshake - it should sync genDoc, _ := sm.MakeGenesisDocFromFile(config.GenesisFile()) handshaker := NewHandshaker(stateDB, state, store, genDoc) @@ -693,7 +703,11 @@ func testHandshakeReplay(t *testing.T, config *cfg.Config, nBlocks int, mode uin t.Fatalf("Error starting proxy app connections: %v", err) } defer proxyApp.Stop() - if err := handshaker.Handshake(proxyApp); err != nil { + err := handshaker.Handshake(proxyApp) + if expectError { + require.Error(t, err) + return + } else if err != nil { t.Fatalf("Error on abci handshake: %v", err) } @@ -728,7 +742,7 @@ func applyBlock(stateDB dbm.DB, st sm.State, blk *types.Block, proxyApp proxy.Ap blockExec := sm.NewBlockExecutor(stateDB, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool) blkID := types.BlockID{Hash: blk.Hash(), PartsHeader: blk.MakePartSet(testPartSize).Header()} - newState, err := blockExec.ApplyBlock(st, blkID, blk) + newState, _, err := blockExec.ApplyBlock(st, blkID, blk) if err != nil { panic(err) } @@ -758,17 +772,19 @@ func buildAppStateFromChain(proxyApp proxy.AppConns, stateDB dbm.DB, block := chain[i] state = applyBlock(stateDB, state, block, proxyApp) } - case 1, 2: + case 1, 2, 3: for i := 0; i < nBlocks-1; i++ { block := chain[i] state = applyBlock(stateDB, state, block, proxyApp) } - if mode == 2 { + if mode == 2 || mode == 3 { // update the kvstore height and apphash // as if we ran commit but not state = applyBlock(stateDB, state, chain[nBlocks-1], proxyApp) } + default: + panic(fmt.Sprintf("unknown mode %v", mode)) } } @@ -806,7 +822,7 @@ func buildTMStateFromChain( state = applyBlock(stateDB, state, block, proxyApp) } - case 1, 2: + case 1, 2, 3: // sync up to the penultimate as if we stored the block. // whether we commit or not depends on the appHash for _, block := range chain[:len(chain)-1] { @@ -816,6 +832,8 @@ func buildTMStateFromChain( // apply the final block to a state copy so we can // get the right next appHash but keep the state back applyBlock(stateDB, state, chain[len(chain)-1], proxyApp) + default: + panic(fmt.Sprintf("unknown mode %v", mode)) } return state @@ -1075,14 +1093,17 @@ type mockBlockStore struct { params types.ConsensusParams chain []*types.Block commits []*types.Commit + base int64 } // TODO: NewBlockStore(db.NewMemDB) ... func newMockBlockStore(config *cfg.Config, params types.ConsensusParams) *mockBlockStore { - return &mockBlockStore{config, params, nil, nil} + return &mockBlockStore{config, params, nil, nil, 0} } func (bs *mockBlockStore) Height() int64 { return int64(len(bs.chain)) } +func (bs *mockBlockStore) Base() int64 { return bs.base } +func (bs *mockBlockStore) Size() int64 { return bs.Height() - bs.Base() + 1 } func (bs *mockBlockStore) LoadBlock(height int64) *types.Block { return bs.chain[height-1] } func (bs *mockBlockStore) LoadBlockByHash(hash []byte) *types.Block { return bs.chain[int64(len(bs.chain))-1] @@ -1104,6 +1125,17 @@ func (bs *mockBlockStore) LoadSeenCommit(height int64) *types.Commit { return bs.commits[height-1] } +func (bs *mockBlockStore) PruneBlocks(height int64) (uint64, error) { + pruned := uint64(0) + for i := int64(0); i < height-1; i++ { + bs.chain[i] = nil + bs.commits[i] = nil + pruned++ + } + bs.base = height + return pruned, nil +} + //--------------------------------------- // Test handshake/init chain diff --git a/consensus/state.go b/consensus/state.go index cd3dbeccb..b58bb3050 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -490,6 +490,10 @@ func (cs *State) reconstructLastCommit(state sm.State) { return } seenCommit := cs.blockStore.LoadSeenCommit(state.LastBlockHeight) + if seenCommit == nil { + panic(fmt.Sprintf("Failed to reconstruct LastCommit: seen commit for height %v not found", + state.LastBlockHeight)) + } lastPrecommits := types.CommitToVoteSet(state.ChainID, seenCommit, state.LastValidators) if !lastPrecommits.HasTwoThirdsMajority() { panic("Failed to reconstruct LastCommit: Does not have +2/3 maj") @@ -878,6 +882,9 @@ func (cs *State) needProofBlock(height int64) bool { } lastBlockMeta := cs.blockStore.LoadBlockMeta(height - 1) + if lastBlockMeta == nil { + panic(fmt.Sprintf("needProofBlock: last block meta for height %d not found", height-1)) + } return !bytes.Equal(cs.state.AppHash, lastBlockMeta.Header.AppHash) } @@ -1448,7 +1455,8 @@ func (cs *State) finalizeCommit(height int64) { // Execute and commit the block, update and save the state, and update the mempool. // NOTE The block.AppHash wont reflect these txs until the next block. var err error - stateCopy, err = cs.blockExec.ApplyBlock( + var retainHeight int64 + stateCopy, retainHeight, err = cs.blockExec.ApplyBlock( stateCopy, types.BlockID{Hash: block.Hash(), PartsHeader: blockParts.Header()}, block) @@ -1463,6 +1471,16 @@ func (cs *State) finalizeCommit(height int64) { fail.Fail() // XXX + // Prune old heights, if requested by ABCI app. + if retainHeight > 0 { + pruned, err := cs.pruneBlocks(retainHeight) + if err != nil { + cs.Logger.Error("Failed to prune blocks", "retainHeight", retainHeight, "err", err) + } else { + cs.Logger.Info("Pruned blocks", "pruned", pruned, "retainHeight", retainHeight) + } + } + // must be called before we update state cs.recordMetrics(height, block) @@ -1481,6 +1499,22 @@ func (cs *State) finalizeCommit(height int64) { // * cs.StartTime is set to when we will start round0. } +func (cs *State) pruneBlocks(retainHeight int64) (uint64, error) { + base := cs.blockStore.Base() + if retainHeight <= base { + return 0, nil + } + pruned, err := cs.blockStore.PruneBlocks(retainHeight) + if err != nil { + return 0, fmt.Errorf("failed to prune block store: %w", err) + } + err = sm.PruneStates(cs.blockExec.DB(), base, retainHeight) + if err != nil { + return 0, fmt.Errorf("failed to prune state database: %w", err) + } + return pruned, nil +} + func (cs *State) recordMetrics(height int64, block *types.Block) { cs.metrics.Validators.Set(float64(cs.Validators.Size())) cs.metrics.ValidatorsPower.Set(float64(cs.Validators.TotalVotingPower())) @@ -1547,9 +1581,11 @@ func (cs *State) recordMetrics(height int64, block *types.Block) { if height > 1 { lastBlockMeta := cs.blockStore.LoadBlockMeta(height - 1) - cs.metrics.BlockIntervalSeconds.Set( - block.Time.Sub(lastBlockMeta.Header.Time).Seconds(), - ) + if lastBlockMeta != nil { + cs.metrics.BlockIntervalSeconds.Set( + block.Time.Sub(lastBlockMeta.Header.Time).Seconds(), + ) + } } cs.metrics.NumTxs.Set(float64(len(block.Data.Txs))) diff --git a/evidence/pool.go b/evidence/pool.go index a4d72cd2f..8c7d78694 100644 --- a/evidence/pool.go +++ b/evidence/pool.go @@ -95,7 +95,7 @@ func (evpool *Pool) Update(block *types.Block, state sm.State) { } // AddEvidence checks the evidence is valid and adds it to the pool. -func (evpool *Pool) AddEvidence(evidence types.Evidence) (err error) { +func (evpool *Pool) AddEvidence(evidence types.Evidence) error { // TODO: check if we already have evidence for this // validator at this height so we dont get spammed @@ -106,14 +106,17 @@ func (evpool *Pool) AddEvidence(evidence types.Evidence) (err error) { // fetch the validator and return its voting power as its priority // TODO: something better ? - valset, _ := sm.LoadValidators(evpool.stateDB, evidence.Height()) + valset, err := sm.LoadValidators(evpool.stateDB, evidence.Height()) + if err != nil { + return err + } _, val := valset.GetByAddress(evidence.Address()) priority := val.VotingPower added := evpool.store.AddNewEvidence(evidence, priority) if !added { // evidence already known, just ignore - return + return nil } evpool.logger.Info("Verified new evidence of byzantine behaviour", "evidence", evidence) diff --git a/rpc/core/blocks.go b/rpc/core/blocks.go index e340d4dfb..1d608534a 100644 --- a/rpc/core/blocks.go +++ b/rpc/core/blocks.go @@ -17,7 +17,7 @@ func BlockchainInfo(ctx *rpctypes.Context, minHeight, maxHeight int64) (*ctypes. // maximum 20 block metas const limit int64 = 20 var err error - minHeight, maxHeight, err = filterMinMax(blockStore.Height(), minHeight, maxHeight, limit) + minHeight, maxHeight, err = filterMinMax(blockStore.Base(), blockStore.Height(), minHeight, maxHeight, limit) if err != nil { return nil, err } @@ -34,11 +34,10 @@ func BlockchainInfo(ctx *rpctypes.Context, minHeight, maxHeight int64) (*ctypes. BlockMetas: blockMetas}, nil } -// error if either min or max are negative or min < max -// if 0, use 1 for min, latest block height for max +// error if either min or max are negative or min > max +// if 0, use blockstore base for min, latest block height for max // enforce limit. -// error if min > max -func filterMinMax(height, min, max, limit int64) (int64, int64, error) { +func filterMinMax(base, height, min, max, limit int64) (int64, int64, error) { // filter negatives if min < 0 || max < 0 { return min, max, fmt.Errorf("heights must be non-negative") @@ -55,6 +54,9 @@ func filterMinMax(height, min, max, limit int64) (int64, int64, error) { // limit max to the height max = tmmath.MinInt64(height, max) + // limit min to the base + min = tmmath.MaxInt64(base, min) + // limit min to within `limit` of max // so the total number of blocks returned will be `limit` min = tmmath.MaxInt64(min, max-limit+1) @@ -69,8 +71,7 @@ func filterMinMax(height, min, max, limit int64) (int64, int64, error) { // If no height is provided, it will fetch the latest block. // More: https://docs.tendermint.com/master/rpc/#/Info/block func Block(ctx *rpctypes.Context, heightPtr *int64) (*ctypes.ResultBlock, error) { - storeHeight := blockStore.Height() - height, err := getHeight(storeHeight, heightPtr) + height, err := getHeight(blockStore.Base(), blockStore.Height(), heightPtr) if err != nil { return nil, err } @@ -99,8 +100,7 @@ func BlockByHash(ctx *rpctypes.Context, hash []byte) (*ctypes.ResultBlock, error // If no height is provided, it will fetch the commit for the latest block. // More: https://docs.tendermint.com/master/rpc/#/Info/commit func Commit(ctx *rpctypes.Context, heightPtr *int64) (*ctypes.ResultCommit, error) { - storeHeight := blockStore.Height() - height, err := getHeight(storeHeight, heightPtr) + height, err := getHeight(blockStore.Base(), blockStore.Height(), heightPtr) if err != nil { return nil, err } @@ -113,7 +113,7 @@ func Commit(ctx *rpctypes.Context, heightPtr *int64) (*ctypes.ResultCommit, erro // If the next block has not been committed yet, // use a non-canonical commit - if height == storeHeight { + if height == blockStore.Height() { commit := blockStore.LoadSeenCommit(height) return ctypes.NewResultCommit(&header, commit, false), nil } @@ -131,8 +131,7 @@ func Commit(ctx *rpctypes.Context, heightPtr *int64) (*ctypes.ResultCommit, erro // getBlock(h).Txs[5] // More: https://docs.tendermint.com/master/rpc/#/Info/block_results func BlockResults(ctx *rpctypes.Context, heightPtr *int64) (*ctypes.ResultBlockResults, error) { - storeHeight := blockStore.Height() - height, err := getHeight(storeHeight, heightPtr) + height, err := getHeight(blockStore.Base(), blockStore.Height(), heightPtr) if err != nil { return nil, err } @@ -152,7 +151,7 @@ func BlockResults(ctx *rpctypes.Context, heightPtr *int64) (*ctypes.ResultBlockR }, nil } -func getHeight(currentHeight int64, heightPtr *int64) (int64, error) { +func getHeight(currentBase int64, currentHeight int64, heightPtr *int64) (int64, error) { if heightPtr != nil { height := *heightPtr if height <= 0 { @@ -161,6 +160,10 @@ func getHeight(currentHeight int64, heightPtr *int64) (int64, error) { if height > currentHeight { return 0, fmt.Errorf("height must be less than or equal to the current blockchain height") } + if height < currentBase { + return 0, fmt.Errorf("height %v is not available, blocks pruned at height %v", + height, currentBase) + } return height, nil } return currentHeight, nil diff --git a/rpc/core/blocks_test.go b/rpc/core/blocks_test.go index 1c4c3af13..c0561647f 100644 --- a/rpc/core/blocks_test.go +++ b/rpc/core/blocks_test.go @@ -19,42 +19,46 @@ import ( func TestBlockchainInfo(t *testing.T) { cases := []struct { min, max int64 - height int64 + base, height int64 limit int64 resultLength int64 wantErr bool }{ // min > max - {0, 0, 0, 10, 0, true}, // min set to 1 - {0, 1, 0, 10, 0, true}, // max set to height (0) - {0, 0, 1, 10, 1, false}, // max set to height (1) - {2, 0, 1, 10, 0, true}, // max set to height (1) - {2, 1, 5, 10, 0, true}, + {0, 0, 0, 0, 10, 0, true}, // min set to 1 + {0, 1, 0, 0, 10, 0, true}, // max set to height (0) + {0, 0, 0, 1, 10, 1, false}, // max set to height (1) + {2, 0, 0, 1, 10, 0, true}, // max set to height (1) + {2, 1, 0, 5, 10, 0, true}, // negative - {1, 10, 14, 10, 10, false}, // control - {-1, 10, 14, 10, 0, true}, - {1, -10, 14, 10, 0, true}, - {-9223372036854775808, -9223372036854775788, 100, 20, 0, true}, + {1, 10, 0, 14, 10, 10, false}, // control + {-1, 10, 0, 14, 10, 0, true}, + {1, -10, 0, 14, 10, 0, true}, + {-9223372036854775808, -9223372036854775788, 0, 100, 20, 0, true}, + + // check base + {1, 1, 1, 1, 1, 1, false}, + {2, 5, 3, 5, 5, 3, false}, // check limit and height - {1, 1, 1, 10, 1, false}, - {1, 1, 5, 10, 1, false}, - {2, 2, 5, 10, 1, false}, - {1, 2, 5, 10, 2, false}, - {1, 5, 1, 10, 1, false}, - {1, 5, 10, 10, 5, false}, - {1, 15, 10, 10, 10, false}, - {1, 15, 15, 10, 10, false}, - {1, 15, 15, 20, 15, false}, - {1, 20, 15, 20, 15, false}, - {1, 20, 20, 20, 20, false}, + {1, 1, 0, 1, 10, 1, false}, + {1, 1, 0, 5, 10, 1, false}, + {2, 2, 0, 5, 10, 1, false}, + {1, 2, 0, 5, 10, 2, false}, + {1, 5, 0, 1, 10, 1, false}, + {1, 5, 0, 10, 10, 5, false}, + {1, 15, 0, 10, 10, 10, false}, + {1, 15, 0, 15, 10, 10, false}, + {1, 15, 0, 15, 20, 15, false}, + {1, 20, 0, 15, 20, 15, false}, + {1, 20, 0, 20, 20, 20, false}, } for i, c := range cases { caseString := fmt.Sprintf("test %d failed", i) - min, max, err := filterMinMax(c.height, c.min, c.max, c.limit) + min, max, err := filterMinMax(c.base, c.height, c.min, c.max, c.limit) if c.wantErr { require.Error(t, err, caseString) } else { @@ -112,12 +116,15 @@ type mockBlockStore struct { height int64 } +func (mockBlockStore) Base() int64 { return 1 } func (store mockBlockStore) Height() int64 { return store.height } +func (store mockBlockStore) Size() int64 { return store.height } func (mockBlockStore) LoadBlockMeta(height int64) *types.BlockMeta { return nil } func (mockBlockStore) LoadBlock(height int64) *types.Block { return nil } func (mockBlockStore) LoadBlockByHash(hash []byte) *types.Block { return nil } func (mockBlockStore) LoadBlockPart(height int64, index int) *types.Part { return nil } func (mockBlockStore) LoadBlockCommit(height int64) *types.Commit { return nil } func (mockBlockStore) LoadSeenCommit(height int64) *types.Commit { return nil } +func (mockBlockStore) PruneBlocks(height int64) (uint64, error) { return 0, nil } func (mockBlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) { } diff --git a/rpc/core/consensus.go b/rpc/core/consensus.go index a2a619ea5..8ea2dde4f 100644 --- a/rpc/core/consensus.go +++ b/rpc/core/consensus.go @@ -18,7 +18,7 @@ func Validators(ctx *rpctypes.Context, heightPtr *int64, page, perPage int) (*ct // The latest validator that we know is the // NextValidator of the last block. height := consensusState.GetState().LastBlockHeight + 1 - height, err := getHeight(height, heightPtr) + height, err := getHeight(blockStore.Base(), height, heightPtr) if err != nil { return nil, err } @@ -91,7 +91,7 @@ func ConsensusState(ctx *rpctypes.Context) (*ctypes.ResultConsensusState, error) // More: https://docs.tendermint.com/master/rpc/#/Info/consensus_params func ConsensusParams(ctx *rpctypes.Context, heightPtr *int64) (*ctypes.ResultConsensusParams, error) { height := consensusState.GetState().LastBlockHeight + 1 - height, err := getHeight(height, heightPtr) + height, err := getHeight(blockStore.Base(), height, heightPtr) if err != nil { return nil, err } diff --git a/rpc/core/status.go b/rpc/core/status.go index e6438009a..4e950d4a3 100644 --- a/rpc/core/status.go +++ b/rpc/core/status.go @@ -16,6 +16,20 @@ import ( // hash, app hash, block height and time. // More: https://docs.tendermint.com/master/rpc/#/Info/status func Status(ctx *rpctypes.Context) (*ctypes.ResultStatus, error) { + var ( + earliestBlockMeta *types.BlockMeta + earliestBlockHash tmbytes.HexBytes + earliestAppHash tmbytes.HexBytes + earliestBlockTimeNano int64 + ) + earliestBlockHeight := blockStore.Base() + earliestBlockMeta = blockStore.LoadBlockMeta(earliestBlockHeight) + if earliestBlockMeta != nil { + earliestAppHash = earliestBlockMeta.Header.AppHash + earliestBlockHash = earliestBlockMeta.BlockID.Hash + earliestBlockTimeNano = earliestBlockMeta.Header.Time.UnixNano() + } + var latestHeight int64 if consensusReactor.FastSync() { latestHeight = blockStore.Height() @@ -36,8 +50,6 @@ func Status(ctx *rpctypes.Context) (*ctypes.ResultStatus, error) { latestBlockTimeNano = latestBlockMeta.Header.Time.UnixNano() } - latestBlockTime := time.Unix(0, latestBlockTimeNano) - var votingPower int64 if val := validatorAtHeight(latestHeight); val != nil { votingPower = val.VotingPower @@ -46,11 +58,15 @@ func Status(ctx *rpctypes.Context) (*ctypes.ResultStatus, error) { result := &ctypes.ResultStatus{ NodeInfo: p2pTransport.NodeInfo().(p2p.DefaultNodeInfo), SyncInfo: ctypes.SyncInfo{ - LatestBlockHash: latestBlockHash, - LatestAppHash: latestAppHash, - LatestBlockHeight: latestHeight, - LatestBlockTime: latestBlockTime, - CatchingUp: consensusReactor.FastSync(), + LatestBlockHash: latestBlockHash, + LatestAppHash: latestAppHash, + LatestBlockHeight: latestHeight, + LatestBlockTime: time.Unix(0, latestBlockTimeNano), + EarliestBlockHash: earliestBlockHash, + EarliestAppHash: earliestAppHash, + EarliestBlockHeight: earliestBlockHeight, + EarliestBlockTime: time.Unix(0, earliestBlockTimeNano), + CatchingUp: consensusReactor.FastSync(), }, ValidatorInfo: ctypes.ValidatorInfo{ Address: pubKey.Address(), diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index ed435bbe0..18b2109ed 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -65,7 +65,13 @@ type SyncInfo struct { LatestAppHash bytes.HexBytes `json:"latest_app_hash"` LatestBlockHeight int64 `json:"latest_block_height"` LatestBlockTime time.Time `json:"latest_block_time"` - CatchingUp bool `json:"catching_up"` + + EarliestBlockHash bytes.HexBytes `json:"earliest_block_hash"` + EarliestAppHash bytes.HexBytes `json:"earliest_app_hash"` + EarliestBlockHeight int64 `json:"earliest_block_height"` + EarliestBlockTime time.Time `json:"earliest_block_time"` + + CatchingUp bool `json:"catching_up"` } // Info about the node's validator diff --git a/rpc/swagger/swagger.yaml b/rpc/swagger/swagger.yaml index e0f40262b..000ea972c 100644 --- a/rpc/swagger/swagger.yaml +++ b/rpc/swagger/swagger.yaml @@ -1123,6 +1123,18 @@ components: latest_block_time: type: string example: "2019-08-01T11:52:22.818762194Z" + earliest_block_hash: + type: string + example: "790BA84C3545FCCC49A5C629CEE6EA58A6E875C3862175BDC11EE7AF54703501" + earliest_app_hash: + type: string + example: "C9AEBB441B787D9F1D846DE51F3826F4FD386108B59B08239653ABF59455C3F8" + earliest_block_height: + type: string + example: "1262196" + earliest_block_time: + type: string + example: "2019-08-01T11:52:22.818762194Z" catching_up: type: boolean example: false diff --git a/state/errors.go b/state/errors.go index cd4cd7824..6e0cdfa47 100644 --- a/state/errors.go +++ b/state/errors.go @@ -21,6 +21,11 @@ type ( AppHeight int64 } + ErrAppBlockHeightTooLow struct { + AppHeight int64 + StoreBase int64 + } + ErrLastStateMismatch struct { Height int64 Core []byte @@ -46,12 +51,12 @@ type ( ) func (e ErrUnknownBlock) Error() string { - return fmt.Sprintf("Could not find block #%d", e.Height) + return fmt.Sprintf("could not find block #%d", e.Height) } func (e ErrBlockHashMismatch) Error() string { return fmt.Sprintf( - "App block hash (%X) does not match core block hash (%X) for height %d", + "app block hash (%X) does not match core block hash (%X) for height %d", e.AppHash, e.CoreHash, e.Height, @@ -59,11 +64,16 @@ func (e ErrBlockHashMismatch) Error() string { } func (e ErrAppBlockHeightTooHigh) Error() string { - return fmt.Sprintf("App block height (%d) is higher than core (%d)", e.AppHeight, e.CoreHeight) + return fmt.Sprintf("app block height (%d) is higher than core (%d)", e.AppHeight, e.CoreHeight) } + +func (e ErrAppBlockHeightTooLow) Error() string { + return fmt.Sprintf("app block height (%d) is too far below block store base (%d)", e.AppHeight, e.StoreBase) +} + func (e ErrLastStateMismatch) Error() string { return fmt.Sprintf( - "Latest tendermint block (%d) LastAppHash (%X) does not match app's AppHash (%X)", + "latest tendermint block (%d) LastAppHash (%X) does not match app's AppHash (%X)", e.Height, e.Core, e.App, @@ -72,20 +82,20 @@ func (e ErrLastStateMismatch) Error() string { func (e ErrStateMismatch) Error() string { return fmt.Sprintf( - "State after replay does not match saved state. Got ----\n%v\nExpected ----\n%v\n", + "state after replay does not match saved state. Got ----\n%v\nExpected ----\n%v\n", e.Got, e.Expected, ) } func (e ErrNoValSetForHeight) Error() string { - return fmt.Sprintf("Could not find validator set for height #%d", e.Height) + return fmt.Sprintf("could not find validator set for height #%d", e.Height) } func (e ErrNoConsensusParamsForHeight) Error() string { - return fmt.Sprintf("Could not find consensus params for height #%d", e.Height) + return fmt.Sprintf("could not find consensus params for height #%d", e.Height) } func (e ErrNoABCIResponsesForHeight) Error() string { - return fmt.Sprintf("Could not find results for height #%d", e.Height) + return fmt.Sprintf("could not find results for height #%d", e.Height) } diff --git a/state/execution.go b/state/execution.go index 15f436463..c6d0b8475 100644 --- a/state/execution.go +++ b/state/execution.go @@ -119,13 +119,14 @@ func (blockExec *BlockExecutor) ValidateBlock(state State, block *types.Block) e // ApplyBlock validates the block against the state, executes it against the app, // fires the relevant events, commits the app, and saves the new state and responses. +// It returns the new state and the block height to retain (pruning older blocks). // It's the only function that needs to be called // from outside this package to process and commit an entire block. // It takes a blockID to avoid recomputing the parts hash. -func (blockExec *BlockExecutor) ApplyBlock(state State, blockID types.BlockID, block *types.Block) (State, error) { +func (blockExec *BlockExecutor) ApplyBlock(state State, blockID types.BlockID, block *types.Block) (State, int64, error) { if err := blockExec.ValidateBlock(state, block); err != nil { - return state, ErrInvalidBlock(err) + return state, 0, ErrInvalidBlock(err) } startTime := time.Now().UnixNano() @@ -133,7 +134,7 @@ func (blockExec *BlockExecutor) ApplyBlock(state State, blockID types.BlockID, b endTime := time.Now().UnixNano() blockExec.metrics.BlockProcessingTime.Observe(float64(endTime-startTime) / 1000000) if err != nil { - return state, ErrProxyAppConn(err) + return state, 0, ErrProxyAppConn(err) } fail.Fail() // XXX @@ -147,11 +148,11 @@ func (blockExec *BlockExecutor) ApplyBlock(state State, blockID types.BlockID, b abciValUpdates := abciResponses.EndBlock.ValidatorUpdates err = validateValidatorUpdates(abciValUpdates, state.ConsensusParams.Validator) if err != nil { - return state, fmt.Errorf("error in validator updates: %v", err) + return state, 0, fmt.Errorf("error in validator updates: %v", err) } validatorUpdates, err := types.PB2TM.ValidatorUpdates(abciValUpdates) if err != nil { - return state, err + return state, 0, err } if len(validatorUpdates) > 0 { blockExec.logger.Info("Updates to validators", "updates", types.ValidatorListString(validatorUpdates)) @@ -160,13 +161,13 @@ func (blockExec *BlockExecutor) ApplyBlock(state State, blockID types.BlockID, b // Update the state with the block and responses. state, err = updateState(state, blockID, &block.Header, abciResponses, validatorUpdates) if err != nil { - return state, fmt.Errorf("commit failed for application: %v", err) + return state, 0, fmt.Errorf("commit failed for application: %v", err) } // Lock mempool, commit app state, update mempoool. - appHash, err := blockExec.Commit(state, block, abciResponses.DeliverTxs) + appHash, retainHeight, err := blockExec.Commit(state, block, abciResponses.DeliverTxs) if err != nil { - return state, fmt.Errorf("commit failed for application: %v", err) + return state, 0, fmt.Errorf("commit failed for application: %v", err) } // Update evpool with the block and state. @@ -184,12 +185,12 @@ func (blockExec *BlockExecutor) ApplyBlock(state State, blockID types.BlockID, b // NOTE: if we crash between Commit and Save, events wont be fired during replay fireEvents(blockExec.logger, blockExec.eventBus, block, abciResponses, validatorUpdates) - return state, nil + return state, retainHeight, nil } // Commit locks the mempool, runs the ABCI Commit message, and updates the // mempool. -// It returns the result of calling abci.Commit (the AppHash), and an error. +// It returns the result of calling abci.Commit (the AppHash) and the height to retain (if any). // The Mempool must be locked during commit and update because state is // typically reset on Commit and old txs must be replayed against committed // state before new txs are run in the mempool, lest they be invalid. @@ -197,7 +198,7 @@ func (blockExec *BlockExecutor) Commit( state State, block *types.Block, deliverTxResponses []*abci.ResponseDeliverTx, -) ([]byte, error) { +) ([]byte, int64, error) { blockExec.mempool.Lock() defer blockExec.mempool.Unlock() @@ -206,7 +207,7 @@ func (blockExec *BlockExecutor) Commit( err := blockExec.mempool.FlushAppConn() if err != nil { blockExec.logger.Error("Client error during mempool.FlushAppConn", "err", err) - return nil, err + return nil, 0, err } // Commit block, get hash back @@ -216,7 +217,7 @@ func (blockExec *BlockExecutor) Commit( "Client error during proxyAppConn.CommitSync", "err", err, ) - return nil, err + return nil, 0, err } // ResponseCommit has no error code - just data @@ -236,7 +237,7 @@ func (blockExec *BlockExecutor) Commit( TxPostCheck(state), ) - return res.Data, err + return res.Data, res.RetainHeight, err } //--------------------------------------------------------- diff --git a/state/execution_test.go b/state/execution_test.go index 7d1337391..041f232bd 100644 --- a/state/execution_test.go +++ b/state/execution_test.go @@ -27,7 +27,9 @@ var ( ) func TestApplyBlock(t *testing.T) { - cc := proxy.NewLocalClientCreator(kvstore.NewApplication()) + app := kvstore.NewApplication() + app.RetainBlocks = 1 + cc := proxy.NewLocalClientCreator(app) proxyApp := proxy.NewAppConns(cc) err := proxyApp.Start() require.Nil(t, err) @@ -41,9 +43,9 @@ func TestApplyBlock(t *testing.T) { block := makeBlock(state, 1) blockID := types.BlockID{Hash: block.Hash(), PartsHeader: block.MakePartSet(testPartSize).Header()} - //nolint:ineffassign - state, err = blockExec.ApplyBlock(state, blockID, block) + _, retainHeight, err := blockExec.ApplyBlock(state, blockID, block) require.Nil(t, err) + assert.EqualValues(t, retainHeight, 1) // TODO check state and mempool } @@ -356,7 +358,7 @@ func TestEndBlockValidatorUpdates(t *testing.T) { {PubKey: types.TM2PB.PubKey(pubkey), Power: 10}, } - state, err = blockExec.ApplyBlock(state, blockID, block) + state, _, err = blockExec.ApplyBlock(state, blockID, block) require.Nil(t, err) // test new validator was added to NextValidators @@ -410,7 +412,7 @@ func TestEndBlockValidatorUpdatesResultingInEmptySet(t *testing.T) { {PubKey: types.TM2PB.PubKey(state.Validators.Validators[0].PubKey), Power: 0}, } - assert.NotPanics(t, func() { state, err = blockExec.ApplyBlock(state, blockID, block) }) + assert.NotPanics(t, func() { state, _, err = blockExec.ApplyBlock(state, blockID, block) }) assert.NotNil(t, err) assert.NotEmpty(t, state.NextValidators.Validators) diff --git a/state/helpers_test.go b/state/helpers_test.go index f8758f987..a85e35748 100644 --- a/state/helpers_test.go +++ b/state/helpers_test.go @@ -66,7 +66,7 @@ func makeAndApplyGoodBlock(state sm.State, height int64, lastCommit *types.Commi } blockID := types.BlockID{Hash: block.Hash(), PartsHeader: types.PartSetHeader{Total: 3, Hash: tmrand.Bytes(32)}} - state, err := blockExec.ApplyBlock(state, blockID, block) + state, _, err := blockExec.ApplyBlock(state, blockID, block) if err != nil { return state, types.BlockID{}, err } diff --git a/state/services.go b/state/services.go index d83a410c9..a30956bdc 100644 --- a/state/services.go +++ b/state/services.go @@ -14,13 +14,17 @@ import ( // BlockStore defines the interface used by the ConsensusState. type BlockStore interface { + Base() int64 Height() int64 + Size() int64 LoadBlockMeta(height int64) *types.BlockMeta LoadBlock(height int64) *types.Block SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) + PruneBlocks(height int64) (uint64, error) + LoadBlockByHash(hash []byte) *types.Block LoadBlockPart(height int64, index int) *types.Part diff --git a/state/store.go b/state/store.go index efb7bddf0..08b695f8a 100644 --- a/state/store.go +++ b/state/store.go @@ -125,6 +125,102 @@ type ABCIResponses struct { BeginBlock *abci.ResponseBeginBlock `json:"begin_block"` } +// PruneStates deletes states between the given heights (including from, excluding to). It is not +// guaranteed to delete all states, since the last checkpointed state and states being pointed to by +// e.g. `LastHeightChanged` must remain. The state at to must also exist. +// +// The from parameter is necessary since we can't do a key scan in a performant way due to the key +// encoding not preserving ordering: https://github.com/tendermint/tendermint/issues/4567 +// This will cause some old states to be left behind when doing incremental partial prunes, +// specifically older checkpoints and LastHeightChanged targets. +func PruneStates(db dbm.DB, from int64, to int64) error { + if from <= 0 || to <= 0 { + return fmt.Errorf("from height %v and to height %v must be greater than 0", from, to) + } + if from >= to { + return fmt.Errorf("from height %v must be lower than to height %v", from, to) + } + valInfo := loadValidatorsInfo(db, to) + if valInfo == nil { + return fmt.Errorf("validators at height %v not found", to) + } + paramsInfo := loadConsensusParamsInfo(db, to) + if paramsInfo == nil { + return fmt.Errorf("consensus params at height %v not found", to) + } + + keepVals := make(map[int64]bool) + if valInfo.ValidatorSet == nil { + keepVals[valInfo.LastHeightChanged] = true + keepVals[lastStoredHeightFor(to, valInfo.LastHeightChanged)] = true // keep last checkpoint too + } + keepParams := make(map[int64]bool) + if paramsInfo.ConsensusParams.Equals(&types.ConsensusParams{}) { + keepParams[paramsInfo.LastHeightChanged] = true + } + + batch := db.NewBatch() + defer batch.Close() + pruned := uint64(0) + var err error + + // We have to delete in reverse order, to avoid deleting previous heights that have validator + // sets and consensus params that we may need to retrieve. + for h := to - 1; h >= from; h-- { + // For heights we keep, we must make sure they have the full validator set or consensus + // params, otherwise they will panic if they're retrieved directly (instead of + // indirectly via a LastHeightChanged pointer). + if keepVals[h] { + v := loadValidatorsInfo(db, h) + if v.ValidatorSet == nil { + v.ValidatorSet, err = LoadValidators(db, h) + if err != nil { + return err + } + v.LastHeightChanged = h + batch.Set(calcValidatorsKey(h), v.Bytes()) + } + } else { + batch.Delete(calcValidatorsKey(h)) + } + + if keepParams[h] { + p := loadConsensusParamsInfo(db, h) + if p.ConsensusParams.Equals(&types.ConsensusParams{}) { + p.ConsensusParams, err = LoadConsensusParams(db, h) + if err != nil { + return err + } + p.LastHeightChanged = h + batch.Set(calcConsensusParamsKey(h), p.Bytes()) + } + } else { + batch.Delete(calcConsensusParamsKey(h)) + } + + batch.Delete(calcABCIResponsesKey(h)) + pruned++ + + // avoid batches growing too large by flushing to database regularly + if pruned%1000 == 0 && pruned > 0 { + err := batch.Write() + if err != nil { + return err + } + batch.Close() + batch = db.NewBatch() + defer batch.Close() + } + } + + err = batch.WriteSync() + if err != nil { + return err + } + + return nil +} + // NewABCIResponses returns a new ABCIResponses func NewABCIResponses(block *types.Block) *ABCIResponses { resDeliverTxs := make([]*abci.ResponseDeliverTx, len(block.Data.Txs)) diff --git a/state/store_test.go b/state/store_test.go index 7661b631f..d46eaa9eb 100644 --- a/state/store_test.go +++ b/state/store_test.go @@ -65,3 +65,118 @@ func BenchmarkLoadValidators(b *testing.B) { }) } } + +func TestPruneStates(t *testing.T) { + testcases := map[string]struct { + makeHeights int64 + pruneFrom int64 + pruneTo int64 + expectErr bool + expectVals []int64 + expectParams []int64 + expectABCI []int64 + }{ + "error on pruning from 0": {100, 0, 5, true, nil, nil, nil}, + "error when from > to": {100, 3, 2, true, nil, nil, nil}, + "error when from == to": {100, 3, 3, true, nil, nil, nil}, + "error when to does not exist": {100, 1, 101, true, nil, nil, nil}, + "prune all": {100, 1, 100, false, []int64{93, 100}, []int64{95, 100}, []int64{100}}, + "prune some": {10, 2, 8, false, []int64{1, 3, 8, 9, 10}, []int64{1, 5, 8, 9, 10}, []int64{1, 8, 9, 10}}, + "prune across checkpoint": {100001, 1, 100001, false, []int64{99993, 100000, 100001}, []int64{99995, 100001}, []int64{100001}}, + } + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + db := dbm.NewMemDB() + + // Generate a bunch of state data. Validators change for heights ending with 3, and + // parameters when ending with 5. + validator := &types.Validator{Address: []byte{1, 2, 3}, VotingPower: 100} + validatorSet := &types.ValidatorSet{ + Validators: []*types.Validator{validator}, + Proposer: validator, + } + valsChanged := int64(0) + paramsChanged := int64(0) + + for h := int64(1); h <= tc.makeHeights; h++ { + if valsChanged == 0 || h%10 == 2 { + valsChanged = h + 1 // Have to add 1, since NextValidators is what's stored + } + if paramsChanged == 0 || h%10 == 5 { + paramsChanged = h + } + + sm.SaveState(db, sm.State{ + LastBlockHeight: h - 1, + Validators: validatorSet, + NextValidators: validatorSet, + ConsensusParams: types.ConsensusParams{ + Block: types.BlockParams{MaxBytes: 10e6}, + }, + LastHeightValidatorsChanged: valsChanged, + LastHeightConsensusParamsChanged: paramsChanged, + }) + sm.SaveABCIResponses(db, h, sm.NewABCIResponses(&types.Block{ + Header: types.Header{Height: h}, + Data: types.Data{ + Txs: types.Txs{ + []byte{1}, + []byte{2}, + []byte{3}, + }, + }, + })) + } + + // Test assertions + err := sm.PruneStates(db, tc.pruneFrom, tc.pruneTo) + if tc.expectErr { + require.Error(t, err) + return + } + require.NoError(t, err) + + expectVals := sliceToMap(tc.expectVals) + expectParams := sliceToMap(tc.expectParams) + expectABCI := sliceToMap(tc.expectABCI) + + for h := int64(1); h <= tc.makeHeights; h++ { + vals, err := sm.LoadValidators(db, h) + if expectVals[h] { + require.NoError(t, err, "validators height %v", h) + require.NotNil(t, vals) + } else { + require.Error(t, err, "validators height %v", h) + require.Equal(t, sm.ErrNoValSetForHeight{Height: h}, err) + } + + params, err := sm.LoadConsensusParams(db, h) + if expectParams[h] { + require.NoError(t, err, "params height %v", h) + require.False(t, params.Equals(&types.ConsensusParams{})) + } else { + require.Error(t, err, "params height %v", h) + require.Equal(t, sm.ErrNoConsensusParamsForHeight{Height: h}, err) + } + + abci, err := sm.LoadABCIResponses(db, h) + if expectABCI[h] { + require.NoError(t, err, "abci height %v", h) + require.NotNil(t, abci) + } else { + require.Error(t, err, "abci height %v", h) + require.Equal(t, sm.ErrNoABCIResponsesForHeight{Height: h}, err) + } + } + }) + } +} + +func sliceToMap(s []int64) map[int64]bool { + m := make(map[int64]bool, len(s)) + for _, i := range s { + m[i] = true + } + return m +} diff --git a/store/store.go b/store/store.go index 2f9ba93fd..38a53d590 100644 --- a/store/store.go +++ b/store/store.go @@ -7,6 +7,7 @@ import ( "github.com/pkg/errors" + db "github.com/tendermint/tm-db" dbm "github.com/tendermint/tm-db" "github.com/tendermint/tendermint/types" @@ -24,6 +25,8 @@ Currently the precommit signatures are duplicated in the Block parts as well as the Commit. In the future this may change, perhaps by moving the Commit data outside the Block. (TODO) +The store can be assumed to contain all contiguous blocks between base and height (inclusive). + // NOTE: BlockStore methods will panic if they encounter errors // deserializing loaded data, indicating probable corruption on disk. */ @@ -31,6 +34,7 @@ type BlockStore struct { db dbm.DB mtx sync.RWMutex + base int64 height int64 } @@ -39,18 +43,36 @@ type BlockStore struct { func NewBlockStore(db dbm.DB) *BlockStore { bsjson := LoadBlockStoreStateJSON(db) return &BlockStore{ + base: bsjson.Base, height: bsjson.Height, db: db, } } -// Height returns the last known contiguous block height. +// Base returns the first known contiguous block height, or 0 for empty block stores. +func (bs *BlockStore) Base() int64 { + bs.mtx.RLock() + defer bs.mtx.RUnlock() + return bs.base +} + +// Height returns the last known contiguous block height, or 0 for empty block stores. func (bs *BlockStore) Height() int64 { bs.mtx.RLock() defer bs.mtx.RUnlock() return bs.height } +// Size returns the number of blocks in the block store. +func (bs *BlockStore) Size() int64 { + bs.mtx.RLock() + defer bs.mtx.RUnlock() + if bs.height == 0 { + return 0 + } + return bs.height - bs.base + 1 +} + // LoadBlock returns the block with the given height. // If no block is found for that height, it returns nil. func (bs *BlockStore) LoadBlock(height int64) *types.Block { @@ -171,6 +193,74 @@ func (bs *BlockStore) LoadSeenCommit(height int64) *types.Commit { return commit } +// PruneBlocks removes block up to (but not including) a height. It returns number of blocks pruned. +func (bs *BlockStore) PruneBlocks(height int64) (uint64, error) { + if height <= 0 { + return 0, fmt.Errorf("height must be greater than 0") + } + bs.mtx.RLock() + if height > bs.height { + bs.mtx.RUnlock() + return 0, fmt.Errorf("cannot prune beyond the latest height %v", bs.height) + } + base := bs.base + bs.mtx.RUnlock() + if height < base { + return 0, fmt.Errorf("cannot prune to height %v, it is lower than base height %v", + height, base) + } + + pruned := uint64(0) + batch := bs.db.NewBatch() + defer batch.Close() + flush := func(batch db.Batch, base int64) error { + // We can't trust batches to be atomic, so update base first to make sure noone + // tries to access missing blocks. + bs.mtx.Lock() + bs.base = base + bs.mtx.Unlock() + bs.saveState() + + err := batch.WriteSync() + if err != nil { + return fmt.Errorf("failed to prune up to height %v: %w", base, err) + } + batch.Close() + return nil + } + + for h := base; h < height; h++ { + meta := bs.LoadBlockMeta(h) + if meta == nil { // assume already deleted + continue + } + batch.Delete(calcBlockMetaKey(h)) + batch.Delete(calcBlockHashKey(meta.BlockID.Hash)) + batch.Delete(calcBlockCommitKey(h)) + batch.Delete(calcSeenCommitKey(h)) + for p := 0; p < meta.BlockID.PartsHeader.Total; p++ { + batch.Delete(calcBlockPartKey(h, p)) + } + pruned++ + + // flush every 1000 blocks to avoid batches becoming too large + if pruned%1000 == 0 && pruned > 0 { + err := flush(batch, h) + if err != nil { + return 0, err + } + batch = bs.db.NewBatch() + defer batch.Close() + } + } + + err := flush(batch, height) + if err != nil { + return 0, err + } + return pruned, nil +} + // SaveBlock persists the given block, blockParts, and seenCommit to the underlying db. // blockParts: Must be parts of the block // seenCommit: The +2/3 precommits that were seen which committed at height. @@ -213,14 +303,17 @@ func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s seenCommitBytes := cdc.MustMarshalBinaryBare(seenCommit) bs.db.Set(calcSeenCommitKey(height), seenCommitBytes) - // Save new BlockStoreStateJSON descriptor - BlockStoreStateJSON{Height: height}.Save(bs.db) - // Done! bs.mtx.Lock() bs.height = height + if bs.base == 0 && height == 1 { + bs.base = 1 + } bs.mtx.Unlock() + // Save new BlockStoreStateJSON descriptor + bs.saveState() + // Flush bs.db.SetSync(nil, nil) } @@ -233,6 +326,16 @@ func (bs *BlockStore) saveBlockPart(height int64, index int, part *types.Part) { bs.db.Set(calcBlockPartKey(height, index), partBytes) } +func (bs *BlockStore) saveState() { + bs.mtx.RLock() + bsJSON := BlockStoreStateJSON{ + Base: bs.base, + Height: bs.height, + } + bs.mtx.RUnlock() + bsJSON.Save(bs.db) +} + //----------------------------------------------------------------------------- func calcBlockMetaKey(height int64) []byte { @@ -261,6 +364,7 @@ var blockStoreKey = []byte("blockStore") // BlockStoreStateJSON is the block store state JSON structure. type BlockStoreStateJSON struct { + Base int64 `json:"base"` Height int64 `json:"height"` } @@ -282,6 +386,7 @@ func LoadBlockStoreStateJSON(db dbm.DB) BlockStoreStateJSON { } if len(bytes) == 0 { return BlockStoreStateJSON{ + Base: 0, Height: 0, } } @@ -290,5 +395,9 @@ func LoadBlockStoreStateJSON(db dbm.DB) BlockStoreStateJSON { if err != nil { panic(fmt.Sprintf("Could not unmarshal bytes: %X", bytes)) } + // Backwards compatibility with persisted data from before Base existed. + if bsj.Height > 0 && bsj.Base == 0 { + bsj.Base = 1 + } return bsj } diff --git a/store/store_test.go b/store/store_test.go index 7fedf8606..16f52aa88 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -65,20 +65,39 @@ func makeStateAndBlockStore(logger log.Logger) (sm.State, *BlockStore, cleanupFu func TestLoadBlockStoreStateJSON(t *testing.T) { db := db.NewMemDB() + bsj := &BlockStoreStateJSON{Base: 100, Height: 1000} + bsj.Save(db) - bsj := &BlockStoreStateJSON{Height: 1000} + retrBSJ := LoadBlockStoreStateJSON(db) + assert.Equal(t, *bsj, retrBSJ, "expected the retrieved DBs to match") +} + +func TestLoadBlockStoreStateJSON_Empty(t *testing.T) { + db := db.NewMemDB() + + bsj := &BlockStoreStateJSON{} bsj.Save(db) retrBSJ := LoadBlockStoreStateJSON(db) + assert.Equal(t, BlockStoreStateJSON{}, retrBSJ, "expected the retrieved DBs to match") +} - assert.Equal(t, *bsj, retrBSJ, "expected the retrieved DBs to match") +func TestLoadBlockStoreStateJSON_NoBase(t *testing.T) { + db := db.NewMemDB() + + bsj := &BlockStoreStateJSON{Height: 1000} + bsj.Save(db) + + retrBSJ := LoadBlockStoreStateJSON(db) + assert.Equal(t, BlockStoreStateJSON{Base: 1, Height: 1000}, retrBSJ, "expected the retrieved DBs to match") } func TestNewBlockStore(t *testing.T) { db := db.NewMemDB() - err := db.Set(blockStoreKey, []byte(`{"height": "10000"}`)) + err := db.Set(blockStoreKey, []byte(`{"base": "100", "height": "10000"}`)) require.NoError(t, err) bs := NewBlockStore(db) + require.Equal(t, int64(100), bs.Base(), "failed to properly parse blockstore") require.Equal(t, int64(10000), bs.Height(), "failed to properly parse blockstore") panicCausers := []struct { @@ -140,6 +159,7 @@ func TestMain(m *testing.M) { func TestBlockStoreSaveLoadBlock(t *testing.T) { state, bs, cleanup := makeStateAndBlockStore(log.NewTMLogger(new(bytes.Buffer))) defer cleanup() + require.Equal(t, bs.Base(), int64(0), "initially the base should be zero") require.Equal(t, bs.Height(), int64(0), "initially the height should be zero") // check there are no blocks at various heights @@ -155,7 +175,8 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) { validPartSet := block.MakePartSet(2) seenCommit := makeTestCommit(10, tmtime.Now()) bs.SaveBlock(block, partSet, seenCommit) - require.Equal(t, bs.Height(), block.Header.Height, "expecting the new height to be changed") + require.EqualValues(t, 1, bs.Base(), "expecting the new height to be changed") + require.EqualValues(t, block.Header.Height, bs.Height(), "expecting the new height to be changed") incompletePartSet := types.NewPartSetFromHeader(types.PartSetHeader{Total: 2}) uncontiguousPartSet := types.NewPartSetFromHeader(types.PartSetHeader{Total: 0}) @@ -364,6 +385,92 @@ func TestLoadBlockPart(t *testing.T) { "expecting successful retrieval of previously saved block") } +func TestPruneBlocks(t *testing.T) { + config := cfg.ResetTestRoot("blockchain_reactor_test") + defer os.RemoveAll(config.RootDir) + state, err := sm.LoadStateFromDBOrGenesisFile(dbm.NewMemDB(), config.GenesisFile()) + require.NoError(t, err) + db := dbm.NewMemDB() + bs := NewBlockStore(db) + assert.EqualValues(t, 0, bs.Base()) + assert.EqualValues(t, 0, bs.Height()) + assert.EqualValues(t, 0, bs.Size()) + + // pruning an empty store should error, even when pruning to 0 + _, err = bs.PruneBlocks(1) + require.Error(t, err) + + _, err = bs.PruneBlocks(0) + require.Error(t, err) + + // make more than 1000 blocks, to test batch deletions + for h := int64(1); h <= 1500; h++ { + block := makeBlock(h, state, new(types.Commit)) + partSet := block.MakePartSet(2) + seenCommit := makeTestCommit(h, tmtime.Now()) + bs.SaveBlock(block, partSet, seenCommit) + } + + assert.EqualValues(t, 1, bs.Base()) + assert.EqualValues(t, 1500, bs.Height()) + assert.EqualValues(t, 1500, bs.Size()) + + prunedBlock := bs.LoadBlock(1199) + + // Check that basic pruning works + pruned, err := bs.PruneBlocks(1200) + require.NoError(t, err) + assert.EqualValues(t, 1199, pruned) + assert.EqualValues(t, 1200, bs.Base()) + assert.EqualValues(t, 1500, bs.Height()) + assert.EqualValues(t, 301, bs.Size()) + assert.EqualValues(t, BlockStoreStateJSON{ + Base: 1200, + Height: 1500, + }, LoadBlockStoreStateJSON(db)) + + require.NotNil(t, bs.LoadBlock(1200)) + require.Nil(t, bs.LoadBlock(1199)) + require.Nil(t, bs.LoadBlockByHash(prunedBlock.Hash())) + require.Nil(t, bs.LoadBlockCommit(1199)) + require.Nil(t, bs.LoadBlockMeta(1199)) + require.Nil(t, bs.LoadBlockPart(1199, 1)) + + for i := int64(1); i < 1200; i++ { + require.Nil(t, bs.LoadBlock(i)) + } + for i := int64(1200); i <= 1500; i++ { + require.NotNil(t, bs.LoadBlock(i)) + } + + // Pruning below the current base should error + _, err = bs.PruneBlocks(1199) + require.Error(t, err) + + // Pruning to the current base should work + pruned, err = bs.PruneBlocks(1200) + require.NoError(t, err) + assert.EqualValues(t, 0, pruned) + + // Pruning again should work + pruned, err = bs.PruneBlocks(1300) + require.NoError(t, err) + assert.EqualValues(t, 100, pruned) + assert.EqualValues(t, 1300, bs.Base()) + + // Pruning beyond the current height should error + _, err = bs.PruneBlocks(1501) + require.Error(t, err) + + // Pruning to the current height should work + pruned, err = bs.PruneBlocks(1500) + require.NoError(t, err) + assert.EqualValues(t, 200, pruned) + assert.Nil(t, bs.LoadBlock(1499)) + assert.NotNil(t, bs.LoadBlock(1500)) + assert.Nil(t, bs.LoadBlock(1501)) +} + func TestLoadBlockMeta(t *testing.T) { bs, db := freshBlockStore() height := int64(10)