diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index fe9d4fdc1..e432b9293 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -51,6 +51,8 @@ Friendly reminder: We have a [bug bounty program](https://hackerone.com/tendermi ### FEATURES +- [config] Add `--mode` flag and config variable. See [ADR-52](https://github.com/tendermint/tendermint/blob/master/docs/architecture/adr-052-tendermint-mode.md) @dongsam + ### IMPROVEMENTS - [crypto/ed25519] \#5632 Adopt zip215 `ed25519` verification. (@marbar3778) diff --git a/UPGRADING.md b/UPGRADING.md index aa34d37d6..ce8c2a2a6 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -17,7 +17,10 @@ This guide provides instructions for upgrading to specific versions of Tendermin * `fast_sync = "v1"` is no longer supported. Please use `v2` instead. * All config parameters are now hyphen-case (also known as kebab-case) instead of snake_case. Before restarting the node make sure - you have updated all the variables in your `config.toml` file. + you have updated all the variables in your `config.toml` file. + +* Added `--mode` flag and `mode` config variable on `config.toml` for setting Mode of the Node: `full` | `validator` | `seed` (default: `full`) + [ADR-52](https://github.com/tendermint/tendermint/blob/master/docs/architecture/adr-052-tendermint-mode.md) ### CLI Changes @@ -30,7 +33,7 @@ This guide provides instructions for upgrading to specific versions of Tendermin $ tendermint gen_node_key > $TMHOME/config/node_key.json ``` -* CLI commands and flags are all now hyphen-case instead of snake_case. +* CLI commands and flags are all now hyphen-case instead of snake_case. Make sure to adjust any scripts that calls a cli command with snake_casing ## v0.34.0 diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index f33ccdf46..7703ff203 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -24,6 +24,9 @@ func AddNodeFlags(cmd *cobra.Command) { // bind flags cmd.Flags().String("moniker", config.Moniker, "node name") + // mode flags + cmd.Flags().String("mode", config.Mode, "node mode (full | validator | seed)") + // priv val flags cmd.Flags().String( "priv-validator-laddr", @@ -71,7 +74,6 @@ func AddNodeFlags(cmd *cobra.Command) { config.P2P.UnconditionalPeerIDs, "comma-delimited IDs of unconditional peers") cmd.Flags().Bool("p2p.upnp", config.P2P.UPNP, "enable/disable UPNP port forwarding") cmd.Flags().Bool("p2p.pex", config.P2P.PexReactor, "enable/disable Peer-Exchange") - cmd.Flags().Bool("p2p.seed-mode", config.P2P.SeedMode, "enable/disable seed mode") cmd.Flags().String("p2p.private-peer-ids", config.P2P.PrivatePeerIDs, "comma-delimited private peer IDs") // consensus flags diff --git a/cmd/tendermint/commands/testnet.go b/cmd/tendermint/commands/testnet.go index ff52536ce..6dcf17e1e 100644 --- a/cmd/tendermint/commands/testnet.go +++ b/cmd/tendermint/commands/testnet.go @@ -105,7 +105,9 @@ func testnetFiles(cmd *cobra.Command, args []string) error { ) } + // set mode to validator for testnet config := cfg.DefaultConfig() + config.Mode = cfg.ModeValidator // overwrite default config if set and valid if configFile != "" { diff --git a/config/config.go b/config/config.go index 2a56117e8..8e43e2bb5 100644 --- a/config/config.go +++ b/config/config.go @@ -23,6 +23,13 @@ const ( // DefaultLogLevel defines a default log level as INFO. DefaultLogLevel = "info" + + ModeFull = "full" + ModeValidator = "validator" + ModeSeed = "seed" + + BlockchainV0 = "v0" + BlockchainV2 = "v2" ) // NOTE: Most of the structs & relevant comments + the @@ -39,6 +46,7 @@ var ( defaultConfigFileName = "config.toml" defaultGenesisJSONName = "genesis.json" + defaultMode = ModeFull defaultPrivValKeyName = "priv_validator_key.json" defaultPrivValStateName = "priv_validator_state.json" @@ -159,6 +167,18 @@ type BaseConfig struct { //nolint: maligned // A custom human readable name for this node Moniker string `mapstructure:"moniker"` + // Mode of Node: full | validator | seed (default: "full") + // * full (default) + // - all reactors + // - No priv_validator_key.json, priv_validator_state.json + // * validator + // - all reactors + // - with priv_validator_key.json, priv_validator_state.json + // * seed + // - only P2P, PEX Reactor + // - No priv_validator_key.json, priv_validator_state.json + Mode string `mapstructure:"mode"` + // If this node is many blocks behind the tip of the chain, FastSync // allows them to catchup quickly by downloading blocks in parallel // and verifying their commits @@ -235,6 +255,7 @@ func DefaultBaseConfig() BaseConfig { PrivValidatorKey: defaultPrivValKeyPath, PrivValidatorState: defaultPrivValStatePath, NodeKey: defaultNodeKeyPath, + Mode: defaultMode, Moniker: defaultMoniker, ProxyApp: "tcp://127.0.0.1:26658", ABCI: "socket", @@ -251,6 +272,7 @@ func DefaultBaseConfig() BaseConfig { func TestBaseConfig() BaseConfig { cfg := DefaultBaseConfig() cfg.chainID = "tendermint_test" + cfg.Mode = ModeValidator cfg.ProxyApp = "kvstore" cfg.FastSyncMode = false cfg.DBBackend = "memdb" @@ -322,6 +344,11 @@ func (cfg BaseConfig) ValidateBasic() error { default: return errors.New("unknown log format (must be 'plain' or 'json')") } + switch cfg.Mode { + case ModeFull, ModeValidator, ModeSeed: + default: + return fmt.Errorf("unknown mode: %v", cfg.Mode) + } return nil } @@ -557,12 +584,6 @@ type P2PConfig struct { //nolint: maligned // Set true to enable the peer-exchange reactor PexReactor bool `mapstructure:"pex"` - // Seed mode, in which node constantly crawls the network and looks for - // peers. If another node asks it for addresses, it responds and disconnects. - // - // Does not work if the peer-exchange reactor is disabled. - SeedMode bool `mapstructure:"seed-mode"` - // Comma separated list of peer IDs to keep private (will not be gossiped to // other peers) PrivatePeerIDs string `mapstructure:"private-peer-ids"` @@ -600,7 +621,6 @@ func DefaultP2PConfig() *P2PConfig { SendRate: 5120000, // 5 mB/s RecvRate: 5120000, // 5 mB/s PexReactor: true, - SeedMode: false, AllowDuplicateIP: false, HandshakeTimeout: 20 * time.Second, DialTimeout: 3 * time.Second, @@ -807,7 +827,7 @@ type FastSyncConfig struct { // DefaultFastSyncConfig returns a default configuration for the fast sync service func DefaultFastSyncConfig() *FastSyncConfig { return &FastSyncConfig{ - Version: "v0", + Version: BlockchainV0, } } @@ -819,9 +839,9 @@ func TestFastSyncConfig() *FastSyncConfig { // ValidateBasic performs basic validation. func (cfg *FastSyncConfig) ValidateBasic() error { switch cfg.Version { - case "v0": + case BlockchainV0: return nil - case "v2": + case BlockchainV2: return nil default: return fmt.Errorf("unknown fastsync version %s", cfg.Version) diff --git a/config/toml.go b/config/toml.go index efa1152dc..f98a35198 100644 --- a/config/toml.go +++ b/config/toml.go @@ -88,6 +88,19 @@ proxy-app = "{{ .BaseConfig.ProxyApp }}" # A custom human readable name for this node moniker = "{{ .BaseConfig.Moniker }}" +# Mode of Node: full | validator | seed (default: "full") +# You will need to set it to "validator" if you want to run the node as a validator +# * full node (default) +# - all reactors +# - No priv_validator_key.json, priv_validator_state.json +# * validator node +# - all reactors +# - with priv_validator_key.json, priv_validator_state.json +# * seed node +# - only P2P, PEX Reactor +# - No priv_validator_key.json, priv_validator_state.json +mode = "{{ .BaseConfig.Mode }}" + # If this node is many blocks behind the tip of the chain, FastSync # allows them to catchup quickly by downloading blocks in parallel # and verifying their commits @@ -305,12 +318,6 @@ recv-rate = {{ .P2P.RecvRate }} # Set true to enable the peer-exchange reactor pex = {{ .P2P.PexReactor }} -# Seed mode, in which node constantly crawls the network and looks for -# peers. If another node asks it for addresses, it responds and disconnects. -# -# Does not work if the peer-exchange reactor is disabled. -seed-mode = {{ .P2P.SeedMode }} - # Comma separated list of peer IDs to keep private (will not be gossiped to other peers) private-peer-ids = "{{ .P2P.PrivatePeerIDs }}" diff --git a/consensus/wal_generator.go b/consensus/wal_generator.go index b3dced8f4..e82bfe8a2 100644 --- a/consensus/wal_generator.go +++ b/consensus/wal_generator.go @@ -89,7 +89,7 @@ func WALGenerateNBlocks(t *testing.T, wr io.Writer, numBlocks int) (err error) { consensusState := NewState(config.Consensus, state.Copy(), blockExec, blockStore, mempool, evpool) consensusState.SetLogger(logger) consensusState.SetEventBus(eventBus) - if privValidator != nil { + if privValidator != nil && privValidator != (*privval.FilePV)(nil) { consensusState.SetPrivValidator(privValidator) } // END OF COPY PASTE diff --git a/docs/architecture/adr-052-tendermint-mode.md b/docs/architecture/adr-052-tendermint-mode.md index acd5028b4..344c68a5b 100644 --- a/docs/architecture/adr-052-tendermint-mode.md +++ b/docs/architecture/adr-052-tendermint-mode.md @@ -7,16 +7,16 @@ ## Context -- Fullnode mode: fullnode mode does not have the capability to become a validator. +- Full mode: full mode does not have the capability to become a validator. - Validator mode : this mode is exactly same as existing state machine behavior. sync without voting on consensus, and participate consensus when fully synced -- Seed mode : lightweight seed mode maintaining an address book, p2p like [TenderSeed](https://gitlab.com/polychainlabs/tenderseed) +- Seed mode : lightweight seed node maintaining an address book, p2p like [TenderSeed](https://gitlab.com/polychainlabs/tenderseed) ## Decision We would like to suggest a simple Tendermint mode abstraction. These modes will live under one binary, and when initializing a node the user will be able to specify which node they would like to create. - Which reactor, component to include for each node - - fullnode *(default)* + - full *(default)* - switch, transport - reactors - mempool @@ -24,6 +24,7 @@ We would like to suggest a simple Tendermint mode abstraction. These modes will - evidence - blockchain - p2p/pex + - statesync - rpc (safe connections only) - *~~no privValidator(priv_validator_key.json, priv_validator_state.json)~~* - validator @@ -33,7 +34,8 @@ We would like to suggest a simple Tendermint mode abstraction. These modes will - consensus - evidence - blockchain -  - p2p/pex + - p2p/pex + - statesync - rpc (safe connections only) - with privValidator(priv_validator_key.json, priv_validator_state.json) - seed @@ -44,17 +46,17 @@ We would like to suggest a simple Tendermint mode abstraction. These modes will - We would like to suggest by introducing `mode` parameter in `config.toml` and cli - `mode = "{{ .BaseConfig.Mode }}"` in `config.toml` - `tendermint node --mode validator` in cli - - fullnode | validator | seed (default: "fullnode") + - full | validator | seednode (default: "full") - RPC modification - `host:26657/status` - - return empty `validator_info` when fullnode mode - - no rpc server in seed mode + - return empty `validator_info` when in full mode + - no rpc server in seednode - Where to modify in codebase - Add switch for `config.Mode` on `node/node.go:DefaultNewNode` - If `config.Mode==validator`, call default `NewNode` (current logic) - - If `config.Mode==fullnode`, call `NewNode` with `nil` `privValidator` (do not load or generation) + - If `config.Mode==full`, call `NewNode` with `nil` `privValidator` (do not load or generation) - Need to add exception routine for `nil` `privValidator` to related functions - - If `config.Mode==seed`, call `NewSeedNode` (seed version of `node/node.go:NewNode`) + - If `config.Mode==seed`, call `NewSeedNode` (seed node version of `node/node.go:NewNode`) - Need to add exception routine for `nil` `reactor`, `component` to related functions ## Status diff --git a/docs/nodes/configuration.md b/docs/nodes/configuration.md index 30d22234f..eaf448c7e 100644 --- a/docs/nodes/configuration.md +++ b/docs/nodes/configuration.md @@ -41,6 +41,19 @@ moniker = "anonymous" # and verifying their commits fast-sync = true +# Mode of Node: full | validator | seed +# You will need to set it to "validator" if you want to run the node as a validator +# * full node (default) +# - all reactors +# - No priv_validator_key.json, priv_validator_state.json +# * validator node +# - all reactors +# - with priv_validator_key.json, priv_validator_state.json +# * seed node +# - only P2P, PEX Reactor +# - No priv_validator_key.json, priv_validator_state.json +mode = "full" + # Database backend: goleveldb | cleveldb | boltdb | rocksdb | badgerdb # * goleveldb (github.com/syndtr/goleveldb - most popular implementation) # - pure go @@ -242,12 +255,6 @@ recv-rate = 5120000 # Set true to enable the peer-exchange reactor pex = true -# Seed mode, in which node constantly crawls the network and looks for -# peers. If another node asks it for addresses, it responds and disconnects. -# -# Does not work if the peer-exchange reactor is disabled. -seed-mode = false - # Comma separated list of peer IDs to keep private (will not be gossiped to other peers) private-peer-ids = "" @@ -494,5 +501,4 @@ This section will cover settings within the p2p section of the `config.toml`. - `max-num-outbound-peers` = is the maximum number of peers you will initiate outbound connects to at one time (where you dial their address and initiate the connection). - `unconditional-peer-ids` = is similar to `persistent-peers` except that these peers will be connected to even if you are already connected to the maximum number of peers. This can be a validator node ID on your sentry node. - `pex` = turns the peer exchange reactor on or off. Validator node will want the `pex` turned off so it would not begin gossiping to unknown peers on the network. PeX can also be turned off for statically configured networks with fixed network connectivity. For full nodes on open, dynamic networks, it should be turned on. -- `seed-mode` = is used for when node operators want to run their node as a seed node. Seed node's run a variation of the PeX protocol that disconnects from peers after sending them a list of peers to connect to. To minimize the servers usage, it is recommended to set the mempool's size to 0. - `private-peer-ids` = is a comma-separated list of node ids that will _not_ be exposed to other peers (i.e., you will not tell other peers about the ids in this list). This can be filled with a validator's node id. diff --git a/docs/nodes/validators.md b/docs/nodes/validators.md index 191a84cb2..b787fa8a4 100644 --- a/docs/nodes/validators.md +++ b/docs/nodes/validators.md @@ -56,6 +56,7 @@ The validator will only talk to the sentry that are provided, the sentry nodes w When initializing nodes there are five parameters in the `config.toml` that may need to be altered. +- `mode:` (full | validator | seed) Mode of node (default: 'full'). If you want to run the node as validator, change it to 'validator'. - `pex:` boolean. This turns the peer exchange reactor on or off for a node. When `pex=false`, only the `persistent-peers` list is available for connection. - `persistent-peers:` a comma separated list of `nodeID@ip:port` values that define a list of peers that are expected to be online at all times. This is necessary at first startup because by setting `pex=false` the node will not be able to join the network. - `unconditional-peer-ids:` comma separated list of nodeID's. These nodes will be connected to no matter the limits of inbound and outbound peers. This is useful for when sentry nodes have full address books. @@ -67,6 +68,7 @@ When initializing nodes there are five parameters in the `config.toml` that may | Config Option | Setting | | ------------------------ | -------------------------- | +| mode | validator | | pex | false | | persistent-peers | list of sentry nodes | | private-peer-ids | none | @@ -74,12 +76,13 @@ When initializing nodes there are five parameters in the `config.toml` that may | addr-book-strict | false | | double-sign-check-height | 10 | -The validator node should have `pex=false` so it does not gossip to the entire network. The persistent peers will be your sentry nodes. Private peers can be left empty as the validator is not trying to hide who it is communicating with. Setting unconditional peers is optional for a validator because they will not have a full address books. +To run the node as validator ensure `mode=validator`. The validator node should have `pex=false` so it does not gossip to the entire network. The persistent peers will be your sentry nodes. Private peers can be left empty as the validator is not trying to hide who it is communicating with. Setting unconditional peers is optional for a validator because they will not have a full address books. #### Sentry Node Configuration | Config Option | Setting | | ---------------------- | --------------------------------------------- | +| mode | full | | pex | true | | persistent-peers | validator node, optionally other sentry nodes | | private-peer-ids | validator node ID | diff --git a/networks/remote/integration.sh b/networks/remote/integration.sh index ae9320757..4c7132865 100644 --- a/networks/remote/integration.sh +++ b/networks/remote/integration.sh @@ -124,7 +124,7 @@ Restart=on-failure User={{service}} Group={{service}} PermissionsStartOnly=true -ExecStart=/usr/bin/tendermint node --proxy-app=kvstore --p2p.persistent-peers=$id0@$ip0:26656,$id1@$ip1:26656,$id2@$ip2:26656,$id3@$ip3:26656 +ExecStart=/usr/bin/tendermint node --mode validator --proxy-app=kvstore --p2p.persistent-peers=$id0@$ip0:26656,$id1@$ip1:26656,$id2@$ip2:26656,$id3@$ip3:26656 ExecReload=/bin/kill -HUP \$MAINPID KillSignal=SIGTERM diff --git a/node/node.go b/node/node.go index 752b036b2..4f7901e3e 100644 --- a/node/node.go +++ b/node/node.go @@ -100,12 +100,23 @@ func DefaultNewNode(config *cfg.Config, logger log.Logger) (*Node, error) { if err != nil { return nil, fmt.Errorf("failed to load or gen node key %s: %w", config.NodeKeyFile(), err) } - - pval, err := privval.LoadOrGenFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile()) - if err != nil { - return nil, err + if config.Mode == cfg.ModeSeed { + return NewSeedNode(config, + nodeKey, + DefaultGenesisDocProviderFunc(config), + logger, + ) } + var pval *privval.FilePV + if config.Mode == cfg.ModeValidator { + pval, err = privval.LoadOrGenFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile()) + if err != nil { + return nil, err + } + } else { + pval = nil + } return NewNode(config, pval, nodeKey, @@ -298,12 +309,13 @@ func doHandshake( return nil } -func logNodeStartupInfo(state sm.State, pubKey crypto.PubKey, logger, consensusLogger log.Logger) { +func logNodeStartupInfo(state sm.State, pubKey crypto.PubKey, logger, consensusLogger log.Logger, mode string) { // Log the version info. logger.Info("Version info", "software", version.TMCoreSemVer, "block", version.BlockProtocol, "p2p", version.P2PProtocol, + "mode", mode, ) // If the state and software differ in block version, at least log it. @@ -313,13 +325,18 @@ func logNodeStartupInfo(state sm.State, pubKey crypto.PubKey, logger, consensusL "state", state.Version.Consensus.Block, ) } - - addr := pubKey.Address() - // Log whether this node is a validator or an observer - if state.Validators.HasAddress(addr) { - consensusLogger.Info("This node is a validator", "addr", addr, "pubKey", pubKey) - } else { - consensusLogger.Info("This node is not a validator", "addr", addr, "pubKey", pubKey) + switch { + case mode == cfg.ModeFull: + consensusLogger.Info("This node is a fullnode") + case mode == cfg.ModeValidator: + addr := pubKey.Address() + // Log whether this node is a validator or an observer + if state.Validators.HasAddress(addr) { + consensusLogger.Info("This node is a validator", "addr", addr, "pubKey", pubKey.Bytes()) + } else { + consensusLogger.Info("This node is a validator (NOT in the active validator set)", + "addr", addr, "pubKey", pubKey.Bytes()) + } } } @@ -328,7 +345,7 @@ func onlyValidatorIsUs(state sm.State, pubKey crypto.PubKey) bool { return false } addr, _ := state.Validators.GetByIndex(0) - return bytes.Equal(pubKey.Address(), addr) + return pubKey != nil && bytes.Equal(pubKey.Address(), addr) } func createMempoolReactor( @@ -445,7 +462,7 @@ func createBlockchainReactor( logger = logger.With("module", "blockchain") switch config.FastSync.Version { - case "v0": + case cfg.BlockchainV0: reactorShim := p2p.NewReactorShim(logger, "BlockchainShim", bcv0.ChannelShims) var ( @@ -471,7 +488,7 @@ func createBlockchainReactor( return reactorShim, reactor, nil - case "v2": + case cfg.BlockchainV2: reactor := bcv2.NewBlockchainReactor(state.Copy(), blockExec, blockStore, fastSync) reactor.SetLogger(logger) @@ -507,10 +524,8 @@ func createConsensusReactor( evidencePool, cs.StateMetrics(csMetrics), ) - consensusState.SetLogger(logger) - - if privValidator != nil { + if privValidator != nil && config.Mode == cfg.ModeValidator { consensusState.SetPrivValidator(privValidator) } @@ -677,11 +692,13 @@ func createSwitch(config *cfg.Config, ) sw.SetLogger(p2pLogger) - sw.AddReactor("MEMPOOL", mempoolReactor) - sw.AddReactor("BLOCKCHAIN", bcReactor) - sw.AddReactor("CONSENSUS", consensusReactor) - sw.AddReactor("EVIDENCE", evidenceReactor) - sw.AddReactor("STATESYNC", stateSyncReactor) + if config.Mode != cfg.ModeSeed { + sw.AddReactor("MEMPOOL", mempoolReactor) + sw.AddReactor("BLOCKCHAIN", bcReactor) + sw.AddReactor("CONSENSUS", consensusReactor) + sw.AddReactor("EVIDENCE", evidenceReactor) + sw.AddReactor("STATESYNC", stateSyncReactor) + } sw.SetNodeInfo(nodeInfo) sw.SetNodeKey(nodeKey) @@ -723,19 +740,19 @@ func createAddrBookAndSetOnSwitch(config *cfg.Config, sw *p2p.Switch, func createPEXReactorAndAddToSwitch(addrBook pex.AddrBook, config *cfg.Config, sw *p2p.Switch, logger log.Logger) *pex.Reactor { + reactorConfig := &pex.ReactorConfig{ + Seeds: splitAndTrimEmpty(config.P2P.Seeds, ",", " "), + SeedMode: config.Mode == cfg.ModeSeed, + // See consensus/reactor.go: blocksToContributeToBecomeGoodPeer 10000 + // blocks assuming 10s blocks ~ 28 hours. + // TODO (melekes): make it dynamic based on the actual block latencies + // from the live network. + // https://github.com/tendermint/tendermint/issues/3523 + SeedDisconnectWaitPeriod: 28 * time.Hour, + PersistentPeersMaxDialPeriod: config.P2P.PersistentPeersMaxDialPeriod, + } // TODO persistent peers ? so we can have their DNS addrs saved - pexReactor := pex.NewReactor(addrBook, - &pex.ReactorConfig{ - Seeds: splitAndTrimEmpty(config.P2P.Seeds, ",", " "), - SeedMode: config.P2P.SeedMode, - // See consensus/reactor.go: blocksToContributeToBecomeGoodPeer 10000 - // blocks assuming 10s blocks ~ 28 hours. - // TODO (melekes): make it dynamic based on the actual block latencies - // from the live network. - // https://github.com/tendermint/tendermint/issues/3523 - SeedDisconnectWaitPeriod: 28 * time.Hour, - PersistentPeersMaxDialPeriod: config.P2P.PersistentPeersMaxDialPeriod, - }) + pexReactor := pex.NewReactor(addrBook, reactorConfig) pexReactor.SetLogger(logger.With("module", "pex")) sw.AddReactor("PEX", pexReactor) return pexReactor @@ -813,6 +830,100 @@ func startStateSync(ssR *statesync.Reactor, bcR fastSyncReactor, conR *cs.Reacto return nil } +// NewSeedNode returns a new seed node, containing only p2p, pex reactor +func NewSeedNode(config *cfg.Config, + nodeKey p2p.NodeKey, + genesisDocProvider GenesisDocProvider, + logger log.Logger, + options ...Option) (*Node, error) { + + genDoc, err := genesisDocProvider() + if err != nil { + return nil, err + } + + state, err := sm.MakeGenesisState(genDoc) + if err != nil { + return nil, err + } + + nodeInfo, err := makeSeedNodeInfo(config, nodeKey, genDoc, state) + if err != nil { + return nil, err + } + + // Setup Transport and Switch. + p2pMetrics := p2p.PrometheusMetrics(config.Instrumentation.Namespace, "chain_id", genDoc.ChainID) + p2pLogger := logger.With("module", "p2p") + transport := createTransport(p2pLogger, config) + sw := createSwitch( + config, transport, p2pMetrics, nil, nil, + nil, nil, nil, nil, nodeInfo, nodeKey, p2pLogger, + ) + + err = sw.AddPersistentPeers(splitAndTrimEmpty(config.P2P.PersistentPeers, ",", " ")) + if err != nil { + return nil, fmt.Errorf("could not add peers from persistent_peers field: %w", err) + } + + err = sw.AddUnconditionalPeerIDs(splitAndTrimEmpty(config.P2P.UnconditionalPeerIDs, ",", " ")) + if err != nil { + return nil, fmt.Errorf("could not add peer ids from unconditional_peer_ids field: %w", err) + } + + addrBook, err := createAddrBookAndSetOnSwitch(config, sw, p2pLogger, nodeKey) + if err != nil { + return nil, fmt.Errorf("could not create addrbook: %w", err) + } + + peerManager, err := createPeerManager(config, p2pLogger, nodeKey.ID) + if err != nil { + return nil, fmt.Errorf("failed to create peer manager: %w", err) + } + + router, err := createRouter(p2pLogger, nodeInfo, nodeKey.PrivKey, peerManager, transport) + if err != nil { + return nil, fmt.Errorf("failed to create router: %w", err) + } + + // start the pex reactor + pexReactor := createPEXReactorAndAddToSwitch(addrBook, config, sw, logger) + pexReactorV2, err := createPEXReactorV2(config, logger, peerManager, router) + if err != nil { + return nil, err + } + + if config.RPC.PprofListenAddress != "" { + go func() { + logger.Info("Starting pprof server", "laddr", config.RPC.PprofListenAddress) + logger.Error("pprof server error", "err", http.ListenAndServe(config.RPC.PprofListenAddress, nil)) + }() + } + + node := &Node{ + config: config, + genesisDoc: genDoc, + + transport: transport, + sw: sw, + addrBook: addrBook, + nodeInfo: nodeInfo, + nodeKey: nodeKey, + peerManager: peerManager, + router: router, + + pexReactor: pexReactor, + pexReactorV2: pexReactorV2, + } + node.BaseService = *service.NewBaseService(logger, "SeedNode", node) + + for _, option := range options { + option(node) + } + + return node, nil +} + // NewNode returns a new, ready to go, Tendermint Node. func NewNode(config *cfg.Config, privValidator types.PrivValidator, @@ -875,10 +986,15 @@ func NewNode(config *cfg.Config, } } } - - pubKey, err := privValidator.GetPubKey(context.TODO()) - if err != nil { - return nil, fmt.Errorf("can't get pubkey: %w", err) + var pubKey crypto.PubKey + if config.Mode == cfg.ModeValidator { + pubKey, err = privValidator.GetPubKey(context.TODO()) + if err != nil { + return nil, fmt.Errorf("can't get pubkey: %w", err) + } + if pubKey == nil { + return nil, errors.New("could not retrieve public key from private validator") + } } // Determine whether we should attempt state sync. @@ -909,7 +1025,7 @@ func NewNode(config *cfg.Config, // app may modify the validator set, specifying ourself as the only validator. fastSync := config.FastSyncMode && !onlyValidatorIsUs(state, pubKey) - logNodeStartupInfo(state, pubKey, logger, consensusLogger) + logNodeStartupInfo(state, pubKey, logger, consensusLogger, config.Mode) // TODO: Fetch and provide real options and do proper p2p bootstrapping. // TODO: Use a persistent peer database. @@ -989,13 +1105,16 @@ func NewNode(config *cfg.Config, // FIXME The way we do phased startups (e.g. replay -> fast sync -> consensus) is very messy, // we should clean this whole thing up. See: // https://github.com/tendermint/tendermint/issues/4644 - stateSyncReactorShim := p2p.NewReactorShim(logger.With("module", "statesync"), "StateSyncShim", statesync.ChannelShims) - var ( + stateSyncReactor *statesync.Reactor + stateSyncReactorShim *p2p.ReactorShim + channels map[p2p.ChannelID]*p2p.Channel peerUpdates *p2p.PeerUpdates ) + stateSyncReactorShim = p2p.NewReactorShim(logger.With("module", "statesync"), "StateSyncShim", statesync.ChannelShims) + if useLegacyP2P { channels = getChannelsFromShim(stateSyncReactorShim) peerUpdates = stateSyncReactorShim.PeerUpdates @@ -1004,7 +1123,7 @@ func NewNode(config *cfg.Config, peerUpdates = peerManager.Subscribe() } - stateSyncReactor := statesync.NewReactor( + stateSyncReactor = statesync.NewReactor( stateSyncReactorShim.Logger, proxyApp.Snapshot(), proxyApp.Query(), @@ -1121,7 +1240,7 @@ func (n *Node) OnStart() error { // Start the RPC server before the P2P server // so we can eg. receive txs for the first block - if n.config.RPC.ListenAddress != "" { + if n.config.RPC.ListenAddress != "" && n.config.Mode != cfg.ModeSeed { listeners, err := n.startRPC() if err != nil { return err @@ -1164,31 +1283,33 @@ func (n *Node) OnStart() error { return err } - if n.config.FastSync.Version == "v0" { - // Start the real blockchain reactor separately since the switch uses the shim. - if err := n.bcReactor.Start(); err != nil { - return err + if n.config.Mode != cfg.ModeSeed { + if n.config.FastSync.Version == cfg.BlockchainV0 { + // Start the real blockchain reactor separately since the switch uses the shim. + if err := n.bcReactor.Start(); err != nil { + return err + } } - } - // Start the real consensus reactor separately since the switch uses the shim. - if err := n.consensusReactor.Start(); err != nil { - return err - } + // Start the real consensus reactor separately since the switch uses the shim. + if err := n.consensusReactor.Start(); err != nil { + return err + } - // Start the real state sync reactor separately since the switch uses the shim. - if err := n.stateSyncReactor.Start(); err != nil { - return err - } + // Start the real state sync reactor separately since the switch uses the shim. + if err := n.stateSyncReactor.Start(); err != nil { + return err + } - // Start the real mempool reactor separately since the switch uses the shim. - if err := n.mempoolReactor.Start(); err != nil { - return err - } + // Start the real mempool reactor separately since the switch uses the shim. + if err := n.mempoolReactor.Start(); err != nil { + return err + } - // Start the real evidence reactor separately since the switch uses the shim. - if err := n.evidenceReactor.Start(); err != nil { - return err + // Start the real evidence reactor separately since the switch uses the shim. + if err := n.evidenceReactor.Start(); err != nil { + return err + } } if !useLegacyP2P && n.pexReactorV2 != nil { @@ -1233,32 +1354,35 @@ func (n *Node) OnStop() { n.Logger.Error("Error closing indexerService", "err", err) } - // now stop the reactors - if n.config.FastSync.Version == "v0" { - // Stop the real blockchain reactor separately since the switch uses the shim. - if err := n.bcReactor.Stop(); err != nil { - n.Logger.Error("failed to stop the blockchain reactor", "err", err) + if n.config.Mode != cfg.ModeSeed { + + // now stop the reactors + if n.config.FastSync.Version == "v0" { + // Stop the real blockchain reactor separately since the switch uses the shim. + if err := n.bcReactor.Stop(); err != nil { + n.Logger.Error("failed to stop the blockchain reactor", "err", err) + } } - } - // Stop the real consensus reactor separately since the switch uses the shim. - if err := n.consensusReactor.Stop(); err != nil { - n.Logger.Error("failed to stop the consensus reactor", "err", err) - } + // Stop the real consensus reactor separately since the switch uses the shim. + if err := n.consensusReactor.Stop(); err != nil { + n.Logger.Error("failed to stop the consensus reactor", "err", err) + } - // Stop the real state sync reactor separately since the switch uses the shim. - if err := n.stateSyncReactor.Stop(); err != nil { - n.Logger.Error("failed to stop the state sync reactor", "err", err) - } + // Stop the real state sync reactor separately since the switch uses the shim. + if err := n.stateSyncReactor.Stop(); err != nil { + n.Logger.Error("failed to stop the state sync reactor", "err", err) + } - // Stop the real mempool reactor separately since the switch uses the shim. - if err := n.mempoolReactor.Stop(); err != nil { - n.Logger.Error("failed to stop the mempool reactor", "err", err) - } + // Stop the real mempool reactor separately since the switch uses the shim. + if err := n.mempoolReactor.Stop(); err != nil { + n.Logger.Error("failed to stop the mempool reactor", "err", err) + } - // Stop the real evidence reactor separately since the switch uses the shim. - if err := n.evidenceReactor.Stop(); err != nil { - n.Logger.Error("failed to stop the evidence reactor", "err", err) + // Stop the real evidence reactor separately since the switch uses the shim. + if err := n.evidenceReactor.Stop(); err != nil { + n.Logger.Error("failed to stop the evidence reactor", "err", err) + } } if !useLegacyP2P && n.pexReactorV2 != nil { @@ -1312,11 +1436,7 @@ func (n *Node) OnStop() { // ConfigureRPC makes sure RPC has all the objects it needs to operate. func (n *Node) ConfigureRPC() error { - pubKey, err := n.privValidator.GetPubKey(context.TODO()) - if err != nil { - return fmt.Errorf("can't get pubkey: %w", err) - } - rpccore.SetEnvironment(&rpccore.Environment{ + rpcCoreEnv := rpccore.Environment{ ProxyAppQuery: n.proxyApp.Query(), ProxyAppMempool: n.proxyApp.Mempool(), @@ -1327,7 +1447,6 @@ func (n *Node) ConfigureRPC() error { P2PPeers: n.sw, P2PTransport: n, - PubKey: pubKey, GenDoc: n.genesisDoc, TxIndexer: n.txIndexer, ConsensusReactor: n.consensusReactor, @@ -1337,7 +1456,15 @@ func (n *Node) ConfigureRPC() error { Logger: n.Logger.With("module", "rpc"), Config: *n.config.RPC, - }) + } + if n.config.Mode == cfg.ModeValidator { + pubKey, err := n.privValidator.GetPubKey(context.TODO()) + if pubKey == nil || err != nil { + return fmt.Errorf("can't get pubkey: %w", err) + } + rpcCoreEnv.PubKey = pubKey + } + rpccore.SetEnvironment(&rpcCoreEnv) return nil } @@ -1577,10 +1704,10 @@ func makeNodeInfo( var bcChannel byte switch config.FastSync.Version { - case "v0": + case cfg.BlockchainV0: bcChannel = byte(bcv0.BlockchainChannel) - case "v2": + case cfg.BlockchainV2: bcChannel = bcv2.BlockchainChannel default: @@ -1630,6 +1757,45 @@ func makeNodeInfo( return nodeInfo, err } +func makeSeedNodeInfo( + config *cfg.Config, + nodeKey p2p.NodeKey, + genDoc *types.GenesisDoc, + state sm.State, +) (p2p.NodeInfo, error) { + nodeInfo := p2p.NodeInfo{ + ProtocolVersion: p2p.NewProtocolVersion( + version.P2PProtocol, // global + state.Version.Consensus.Block, + state.Version.Consensus.App, + ), + NodeID: nodeKey.ID, + Network: genDoc.ChainID, + Version: version.TMCoreSemVer, + Channels: []byte{}, + Moniker: config.Moniker, + Other: p2p.NodeInfoOther{ + TxIndex: "off", + RPCAddress: config.RPC.ListenAddress, + }, + } + + if config.P2P.PexReactor { + nodeInfo.Channels = append(nodeInfo.Channels, pex.PexChannel) + } + + lAddr := config.P2P.ExternalAddress + + if lAddr == "" { + lAddr = config.P2P.ListenAddress + } + + nodeInfo.ListenAddr = lAddr + + err := nodeInfo.Validate() + return nodeInfo, err +} + //------------------------------------------------------------------------------ var ( diff --git a/node/node_test.go b/node/node_test.go index 5d2b21535..d55a8d280 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -522,6 +522,27 @@ func TestNodeNewNodeCustomReactors(t *testing.T) { assert.Equal(t, customBlockchainReactor, n.Switch().Reactor("BLOCKCHAIN")) } +func TestNodeNewSeedNode(t *testing.T) { + config := cfg.ResetTestRoot("node_new_node_custom_reactors_test") + config.Mode = cfg.ModeSeed + defer os.RemoveAll(config.RootDir) + + nodeKey, err := p2p.LoadOrGenNodeKey(config.NodeKeyFile()) + require.NoError(t, err) + + n, err := NewSeedNode(config, + nodeKey, + DefaultGenesisDocProviderFunc(config), + log.TestingLogger(), + ) + require.NoError(t, err) + + err = n.Start() + require.NoError(t, err) + + assert.True(t, n.pexReactor.IsRunning()) +} + func state(nVals int, height int64) (sm.State, dbm.DB, []types.PrivValidator) { privVals := make([]types.PrivValidator, nVals) vals := make([]types.GenesisValidator, nVals) diff --git a/rpc/core/status.go b/rpc/core/status.go index 8052ed97b..8b09dc5da 100644 --- a/rpc/core/status.go +++ b/rpc/core/status.go @@ -1,6 +1,7 @@ package core import ( + "bytes" "time" tmbytes "github.com/tendermint/tendermint/libs/bytes" @@ -49,7 +50,14 @@ func Status(ctx *rpctypes.Context) (*ctypes.ResultStatus, error) { if val := validatorAtHeight(latestUncommittedHeight()); val != nil { votingPower = val.VotingPower } - + validatorInfo := ctypes.ValidatorInfo{} + if env.PubKey != nil { + validatorInfo = ctypes.ValidatorInfo{ + Address: env.PubKey.Address(), + PubKey: env.PubKey, + VotingPower: votingPower, + } + } result := &ctypes.ResultStatus{ NodeInfo: env.P2PTransport.NodeInfo(), SyncInfo: ctypes.SyncInfo{ @@ -63,22 +71,32 @@ func Status(ctx *rpctypes.Context) (*ctypes.ResultStatus, error) { EarliestBlockTime: time.Unix(0, earliestBlockTimeNano), CatchingUp: env.ConsensusReactor.WaitSync(), }, - ValidatorInfo: ctypes.ValidatorInfo{ - Address: env.PubKey.Address(), - PubKey: env.PubKey, - VotingPower: votingPower, - }, + ValidatorInfo: validatorInfo, } return result, nil } func validatorAtHeight(h int64) *types.Validator { - vals, err := env.StateStore.LoadValidators(h) + valsWithH, err := env.StateStore.LoadValidators(h) if err != nil { return nil } + if env.PubKey == nil { + return nil + } privValAddress := env.PubKey.Address() - _, val := vals.GetByAddress(privValAddress) + + // If we're still at height h, search in the current validator set. + lastBlockHeight, vals := env.ConsensusState.GetValidators() + if lastBlockHeight == h { + for _, val := range vals { + if bytes.Equal(val.Address, privValAddress) { + return val + } + } + } + + _, val := valsWithH.GetByAddress(privValAddress) return val } diff --git a/test/app/test.sh b/test/app/test.sh index 710aae80b..d415bc10e 100755 --- a/test/app/test.sh +++ b/test/app/test.sh @@ -17,7 +17,7 @@ function kvstore_over_socket(){ echo "Starting kvstore_over_socket" abci-cli kvstore > /dev/null & pid_kvstore=$! - tendermint start > tendermint.log & + tendermint start --mode validator > tendermint.log & pid_tendermint=$! sleep 5 @@ -32,7 +32,7 @@ function kvstore_over_socket_reorder(){ rm -rf $TMHOME tendermint init echo "Starting kvstore_over_socket_reorder (ie. start tendermint first)" - tendermint start > tendermint.log & + tendermint start --mode validator > tendermint.log & pid_tendermint=$! sleep 2 abci-cli kvstore > /dev/null & @@ -52,7 +52,7 @@ function counter_over_socket() { echo "Starting counter_over_socket" abci-cli counter --serial > /dev/null & pid_counter=$! - tendermint start > tendermint.log & + tendermint start --mode validator > tendermint.log & pid_tendermint=$! sleep 5 @@ -68,7 +68,7 @@ function counter_over_grpc() { echo "Starting counter_over_grpc" abci-cli counter --serial --abci grpc > /dev/null & pid_counter=$! - tendermint start --abci grpc > tendermint.log & + tendermint start --mode validator --abci grpc > tendermint.log & pid_tendermint=$! sleep 5 @@ -86,7 +86,7 @@ function counter_over_grpc_grpc() { pid_counter=$! sleep 1 GRPC_PORT=36656 - tendermint start --abci grpc --rpc.grpc-laddr tcp://localhost:$GRPC_PORT > tendermint.log & + tendermint start --mode validator --abci grpc --rpc.grpc-laddr tcp://localhost:$GRPC_PORT > tendermint.log & pid_tendermint=$! sleep 5 diff --git a/test/e2e/app/main.go b/test/e2e/app/main.go index a465bb786..5141750ed 100644 --- a/test/e2e/app/main.go +++ b/test/e2e/app/main.go @@ -75,9 +75,12 @@ func run(configFile string) error { case "socket", "grpc": err = startApp(cfg) case "builtin": - if cfg.Mode == string(e2e.ModeLight) { - err = startLightClient(cfg) - } else { + switch cfg.Mode { + case string(e2e.ModeLight): + err = startLightNode(cfg) + case string(e2e.ModeSeed): + err = startSeedNode(cfg) + default: err = startNode(cfg) } // FIXME: Temporarily remove maverick until it is redesigned @@ -149,7 +152,25 @@ func startNode(cfg *Config) error { return n.Start() } -func startLightClient(cfg *Config) error { +func startSeedNode(cfg *Config) error { + tmcfg, nodeLogger, nodeKey, err := setupNode() + if err != nil { + return fmt.Errorf("failed to setup config: %w", err) + } + + n, err := node.NewSeedNode( + tmcfg, + *nodeKey, + node.DefaultGenesisDocProviderFunc(tmcfg), + nodeLogger, + ) + if err != nil { + return err + } + return n.Start() +} + +func startLightNode(cfg *Config) error { tmcfg, nodeLogger, _, err := setupNode() if err != nil { return err diff --git a/test/e2e/runner/rpc.go b/test/e2e/runner/rpc.go index f526e00d0..6df7ba87b 100644 --- a/test/e2e/runner/rpc.go +++ b/test/e2e/runner/rpc.go @@ -67,6 +67,9 @@ func waitForHeight(testnet *e2e.Testnet, height int64) (*types.Block, *types.Blo // waitForNode waits for a node to become available and catch up to the given block height. func waitForNode(node *e2e.Node, height int64, timeout time.Duration) (*rpctypes.ResultStatus, error) { + if node.Mode == e2e.ModeSeed { + return nil, nil + } client, err := node.Client() if err != nil { return nil, err diff --git a/test/e2e/runner/setup.go b/test/e2e/runner/setup.go index 2f4d704b3..69c9067ad 100644 --- a/test/e2e/runner/setup.go +++ b/test/e2e/runner/setup.go @@ -256,6 +256,9 @@ func MakeConfig(node *e2e.Node) (*config.Config, error) { cfg.P2P.AddrBookStrict = false cfg.DBBackend = node.Database cfg.StateSync.DiscoveryTime = 5 * time.Second + if node.Mode != e2e.ModeLight { + cfg.Mode = string(node.Mode) + } switch node.ABCIProtocol { case e2e.ProtocolUNIX: @@ -296,7 +299,6 @@ func MakeConfig(node *e2e.Node) (*config.Config, error) { return nil, fmt.Errorf("invalid privval protocol setting %q", node.PrivvalProtocol) } case e2e.ModeSeed: - cfg.P2P.SeedMode = true cfg.P2P.PexReactor = true case e2e.ModeFull, e2e.ModeLight: // Don't need to do anything, since we're using a dummy privval key by default.