From 2d8287d0f70962cc1d7bda0250def18962abf282 Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Wed, 29 Sep 2021 16:17:32 +0200 Subject: [PATCH] e2e: allow running of single node using the e2e app (backport) (#7024) --- CHANGELOG_PENDING.md | 2 ++ cmd/tendermint/commands/run_node.go | 4 +-- proxy/client.go | 7 ++++ test/e2e/Makefile | 6 ++-- test/e2e/README.md | 34 ++++++++++++++++++- test/e2e/app/app.go | 51 ++++++++++++++++++++++++++++- test/e2e/app/snapshots.go | 2 +- test/e2e/app/state.go | 2 +- test/e2e/docker/Dockerfile | 2 +- test/e2e/node/built-in.toml | 4 +++ test/e2e/{app => node}/config.go | 14 ++++++++ test/e2e/{app => node}/main.go | 7 ++-- test/e2e/node/socket.toml | 5 +++ 13 files changed, 126 insertions(+), 14 deletions(-) create mode 100644 test/e2e/node/built-in.toml rename test/e2e/{app => node}/config.go (80%) rename test/e2e/{app => node}/main.go (98%) create mode 100644 test/e2e/node/socket.toml diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 3854d3343..9b1da5bb4 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -20,6 +20,8 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi ### FEATURES +- [\#6982](https://github.com/tendermint/tendermint/pull/6982) tendermint binary has built-in suppport for running the end to end application (with state sync support) (@cmwaters). + ### IMPROVEMENTS ### BUG FIXES diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index acc79920a..fd4408c5a 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -46,9 +46,7 @@ func AddNodeFlags(cmd *cobra.Command) { "proxy_app", config.ProxyApp, "proxy app address, or one of: 'kvstore',"+ - " 'persistent_kvstore',"+ - " 'counter',"+ - " 'counter_serial' or 'noop' for local testing.") + " 'persistent_kvstore', 'counter', 'e2e' or 'noop' for local testing.") cmd.Flags().String("abci", config.ABCI, "specify abci transport (socket | grpc)") // rpc flags diff --git a/proxy/client.go b/proxy/client.go index 27baa9738..0504b09f3 100644 --- a/proxy/client.go +++ b/proxy/client.go @@ -8,6 +8,7 @@ import ( "github.com/tendermint/tendermint/abci/example/kvstore" "github.com/tendermint/tendermint/abci/types" tmsync "github.com/tendermint/tendermint/libs/sync" + e2e "github.com/tendermint/tendermint/test/e2e/app" ) // ClientCreator creates new ABCI clients. @@ -79,6 +80,12 @@ func DefaultClientCreator(addr, transport, dbDir string) ClientCreator { return NewLocalClientCreator(kvstore.NewApplication()) case "persistent_kvstore": return NewLocalClientCreator(kvstore.NewPersistentKVStoreApplication(dbDir)) + case "e2e": + app, err := e2e.NewApplication(e2e.DefaultConfig(dbDir)) + if err != nil { + panic(err) + } + return NewLocalClientCreator(app) case "noop": return NewLocalClientCreator(types.NewBaseApplication()) default: diff --git a/test/e2e/Makefile b/test/e2e/Makefile index c9eb8bc19..c751aab89 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -6,8 +6,8 @@ docker: # We need to build support for database backends into the app in # order to build a binary with a Tendermint node in it (for built-in # ABCI testing). -app: - go build -o build/app -tags badgerdb,boltdb,cleveldb,rocksdb ./app +node: + go build -o build/node -tags badgerdb,boltdb,cleveldb,rocksdb ./node # To be used primarily by the e2e docker instance. If you want to produce this binary # elsewhere, then run go build in the maverick directory. @@ -20,4 +20,4 @@ generator: runner: go build -o build/runner ./runner -.PHONY: all app docker generator maverick runner +.PHONY: all node docker generator maverick runner diff --git a/test/e2e/README.md b/test/e2e/README.md index fb8e42158..09b6d554e 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -122,10 +122,42 @@ Docker does not enable IPv6 by default. To do so, enter the following in } ``` -## Benchmarking testnets +## Benchmarking Testnets It is also possible to run a simple benchmark on a testnet. This is done through the `benchmark` command. This manages the entire process: setting up the environment, starting the test net, waiting for a considerable amount of blocks to be used (currently 100), and then returning the following metrics from the sample of the blockchain: - Average time to produce a block - Standard deviation of producing a block - Minimum and maximum time to produce a block + +## Running Individual Nodes + +The E2E test harness is designed to run several nodes of varying configurations within docker. It is also possible to run a single node in the case of running larger, geographically-dispersed testnets. To run a single node you can either run: + +**Built-in** + +```bash +make node +tendermint init validator +TMHOME=$HOME/.tendermint ./build/node ./node/built-in.toml +``` + +To make things simpler the e2e application can also be run in the tendermint binary +by running + +```bash +tendermint start --proxy-app e2e +``` + +However this won't offer the same level of configurability of the application. + +**Socket** + +```bash +make node +tendermint init validator +tendermint start +./build/node ./node.socket.toml +``` + +Check `node/config.go` to see how the settings of the test application can be tweaked. \ No newline at end of file diff --git a/test/e2e/app/app.go b/test/e2e/app/app.go index daec08ed0..c1c7b8c9a 100644 --- a/test/e2e/app/app.go +++ b/test/e2e/app/app.go @@ -1,4 +1,4 @@ -package main +package app import ( "bytes" @@ -28,6 +28,55 @@ type Application struct { restoreChunks [][]byte } +// Config allows for the setting of high level parameters for running the e2e Application +// KeyType and ValidatorUpdates must be the same for all nodes running the same application. +type Config struct { + // The directory with which state.json will be persisted in. Usually $HOME/.tendermint/data + Dir string `toml:"dir"` + + // SnapshotInterval specifies the height interval at which the application + // will take state sync snapshots. Defaults to 0 (disabled). + SnapshotInterval uint64 `toml:"snapshot_interval"` + + // RetainBlocks specifies the number of recent blocks to retain. Defaults to + // 0, which retains all blocks. Must be greater that PersistInterval, + // SnapshotInterval and EvidenceAgeHeight. + RetainBlocks uint64 `toml:"retain_blocks"` + + // KeyType sets the curve that will be used by validators. + // Options are ed25519 & secp256k1 + KeyType string `toml:"key_type"` + + // PersistInterval specifies the height interval at which the application + // will persist state to disk. Defaults to 1 (every height), setting this to + // 0 disables state persistence. + PersistInterval uint64 `toml:"persist_interval"` + + // ValidatorUpdates is a map of heights to validator names and their power, + // and will be returned by the ABCI application. For example, the following + // changes the power of validator01 and validator02 at height 1000: + // + // [validator_update.1000] + // validator01 = 20 + // validator02 = 10 + // + // Specifying height 0 returns the validator update during InitChain. The + // application returns the validator updates as-is, i.e. removing a + // validator must be done by returning it with power 0, and any validators + // not specified are not changed. + // + // height <-> pubkey <-> voting power + ValidatorUpdates map[string]map[string]uint8 `toml:"validator_update"` +} + +func DefaultConfig(dir string) *Config { + return &Config{ + PersistInterval: 1, + SnapshotInterval: 100, + Dir: dir, + } +} + // NewApplication creates the application. func NewApplication(cfg *Config) (*Application, error) { state, err := NewState(filepath.Join(cfg.Dir, "state.json"), cfg.PersistInterval) diff --git a/test/e2e/app/snapshots.go b/test/e2e/app/snapshots.go index 4ddb7ecdc..4ef20375f 100644 --- a/test/e2e/app/snapshots.go +++ b/test/e2e/app/snapshots.go @@ -1,5 +1,5 @@ // nolint: gosec -package main +package app import ( "encoding/json" diff --git a/test/e2e/app/state.go b/test/e2e/app/state.go index ad9960105..33a124caa 100644 --- a/test/e2e/app/state.go +++ b/test/e2e/app/state.go @@ -1,5 +1,5 @@ //nolint: gosec -package main +package app import ( "crypto/sha256" diff --git a/test/e2e/docker/Dockerfile b/test/e2e/docker/Dockerfile index 807b9f572..efe2183e9 100644 --- a/test/e2e/docker/Dockerfile +++ b/test/e2e/docker/Dockerfile @@ -19,7 +19,7 @@ COPY . . RUN make build && cp build/tendermint /usr/bin/tendermint COPY test/e2e/docker/entrypoint* /usr/bin/ RUN cd test/e2e && make maverick && cp build/maverick /usr/bin/maverick -RUN cd test/e2e && make app && cp build/app /usr/bin/app +RUN cd test/e2e && make node && cp build/node /usr/bin/app # Set up runtime directory. We don't use a separate runtime image since we need # e.g. leveldb and rocksdb which are already installed in the build image. diff --git a/test/e2e/node/built-in.toml b/test/e2e/node/built-in.toml new file mode 100644 index 000000000..0a2146a58 --- /dev/null +++ b/test/e2e/node/built-in.toml @@ -0,0 +1,4 @@ +snapshot_interval = 100 +persist_interval = 1 +chain_id = "test-chain" +protocol = "builtin" diff --git a/test/e2e/app/config.go b/test/e2e/node/config.go similarity index 80% rename from test/e2e/app/config.go rename to test/e2e/node/config.go index d7e776538..7efb4e822 100644 --- a/test/e2e/app/config.go +++ b/test/e2e/node/config.go @@ -6,6 +6,8 @@ import ( "fmt" "github.com/BurntSushi/toml" + + "github.com/tendermint/tendermint/test/e2e/app" ) // Config is the application configuration. @@ -26,6 +28,18 @@ type Config struct { KeyType string `toml:"key_type"` } +// App extracts out the application specific configuration parameters +func (cfg *Config) App() *app.Config { + return &app.Config{ + Dir: cfg.Dir, + SnapshotInterval: cfg.SnapshotInterval, + RetainBlocks: cfg.RetainBlocks, + KeyType: cfg.KeyType, + ValidatorUpdates: cfg.ValidatorUpdates, + PersistInterval: cfg.PersistInterval, + } +} + // LoadConfig loads the configuration from disk. func LoadConfig(file string) (*Config, error) { cfg := &Config{ diff --git a/test/e2e/app/main.go b/test/e2e/node/main.go similarity index 98% rename from test/e2e/app/main.go rename to test/e2e/node/main.go index fe81c361e..b7dbd96e9 100644 --- a/test/e2e/app/main.go +++ b/test/e2e/node/main.go @@ -28,6 +28,7 @@ import ( "github.com/tendermint/tendermint/privval" "github.com/tendermint/tendermint/proxy" rpcserver "github.com/tendermint/tendermint/rpc/jsonrpc/server" + "github.com/tendermint/tendermint/test/e2e/app" e2e "github.com/tendermint/tendermint/test/e2e/pkg" mcs "github.com/tendermint/tendermint/test/maverick/consensus" maverick "github.com/tendermint/tendermint/test/maverick/node" @@ -98,7 +99,7 @@ func run(configFile string) error { // startApp starts the application server, listening for connections from Tendermint. func startApp(cfg *Config) error { - app, err := NewApplication(cfg) + app, err := app.NewApplication(cfg.App()) if err != nil { return err } @@ -119,7 +120,7 @@ func startApp(cfg *Config) error { // // FIXME There is no way to simply load the configuration from a file, so we need to pull in Viper. func startNode(cfg *Config) error { - app, err := NewApplication(cfg) + app, err := app.NewApplication(cfg.App()) if err != nil { return err } @@ -205,7 +206,7 @@ func startLightClient(cfg *Config) error { // startMaverick starts a Maverick node that runs the application directly. It assumes the Tendermint // configuration is in $TMHOME/config/tendermint.toml. func startMaverick(cfg *Config) error { - app, err := NewApplication(cfg) + app, err := app.NewApplication(cfg.App()) if err != nil { return err } diff --git a/test/e2e/node/socket.toml b/test/e2e/node/socket.toml new file mode 100644 index 000000000..2f7913e62 --- /dev/null +++ b/test/e2e/node/socket.toml @@ -0,0 +1,5 @@ +snapshot_interval = 100 +persist_interval = 1 +chain_id = "test-chain" +protocol = "socket" +listen = "tcp://127.0.0.1:26658"