diff --git a/cmd/tendermint/commands/light.go b/cmd/tendermint/commands/light.go index 0264d3725..0ba594fd6 100644 --- a/cmd/tendermint/commands/light.go +++ b/cmd/tendermint/commands/light.go @@ -8,7 +8,6 @@ import ( "net/http" "os" "path/filepath" - "regexp" "strings" "time" @@ -16,7 +15,6 @@ import ( dbm "github.com/tendermint/tm-db" - "github.com/tendermint/tendermint/crypto/merkle" "github.com/tendermint/tendermint/libs/log" tmmath "github.com/tendermint/tendermint/libs/math" tmos "github.com/tendermint/tendermint/libs/os" @@ -24,7 +22,6 @@ import ( lproxy "github.com/tendermint/tendermint/light/proxy" lrpc "github.com/tendermint/tendermint/light/rpc" dbs "github.com/tendermint/tendermint/light/store/db" - rpchttp "github.com/tendermint/tendermint/rpc/client/http" rpcserver "github.com/tendermint/tendermint/rpc/jsonrpc/server" ) @@ -217,17 +214,11 @@ func runProxy(cmd *cobra.Command, args []string) error { cfg.WriteTimeout = config.RPC.TimeoutBroadcastTxCommit + 1*time.Second } - rpcClient, err := rpchttp.NewWithTimeout(primaryAddr, cfg.WriteTimeout) + p, err := lproxy.NewProxy(c, listenAddr, primaryAddr, cfg, logger, lrpc.KeyPathFn(lrpc.DefaultMerkleKeyPathFn())) if err != nil { - return fmt.Errorf("failed to create http client for %s: %w", primaryAddr, err) + return err } - p := lproxy.Proxy{ - Addr: listenAddr, - Config: cfg, - Client: lrpc.NewClient(rpcClient, c, lrpc.KeyPathFn(defaultMerkleKeyPathFn())), - Logger: logger, - } // Stop upon receiving SIGTERM or CTRL-C. tmos.TrapSignal(logger, func() { p.Listener.Close() @@ -266,21 +257,3 @@ func saveProviders(db dbm.DB, primaryAddr, witnessesAddrs string) error { } return nil } - -func defaultMerkleKeyPathFn() lrpc.KeyPathFunc { - // regexp for extracting store name from /abci_query path - storeNameRegexp := regexp.MustCompile(`\/store\/(.+)\/key`) - - return func(path string, key []byte) (merkle.KeyPath, error) { - matches := storeNameRegexp.FindStringSubmatch(path) - if len(matches) != 2 { - return nil, fmt.Errorf("can't find store name in %s using %s", path, storeNameRegexp) - } - storeName := matches[1] - - kp := merkle.KeyPath{} - kp = kp.AppendKey([]byte(storeName), merkle.KeyEncodingURL) - kp = kp.AppendKey(key, merkle.KeyEncodingURL) - return kp, nil - } -} diff --git a/light/client.go b/light/client.go index 08dae678d..86f257157 100644 --- a/light/client.go +++ b/light/client.go @@ -861,6 +861,8 @@ func (c *Client) cleanupAfter(height int64) error { } func (c *Client) updateTrustedLightBlock(l *types.LightBlock) error { + c.logger.Debug("updating trusted light block", "light_block", l) + if err := c.trustedStore.SaveLightBlock(l); err != nil { return fmt.Errorf("failed to save trusted header: %w", err) } diff --git a/light/proxy/proxy.go b/light/proxy/proxy.go index c5b71b0ad..8f1e7bf87 100644 --- a/light/proxy/proxy.go +++ b/light/proxy/proxy.go @@ -8,7 +8,9 @@ import ( "github.com/tendermint/tendermint/libs/log" tmpubsub "github.com/tendermint/tendermint/libs/pubsub" + "github.com/tendermint/tendermint/light" lrpc "github.com/tendermint/tendermint/light/rpc" + rpchttp "github.com/tendermint/tendermint/rpc/client/http" rpcserver "github.com/tendermint/tendermint/rpc/jsonrpc/server" ) @@ -21,6 +23,28 @@ type Proxy struct { Listener net.Listener } +// NewProxy creates the struct used to run an HTTP server for serving light +// client rpc requests. +func NewProxy( + lightClient *light.Client, + listenAddr, providerAddr string, + config *rpcserver.Config, + logger log.Logger, + opts ...lrpc.Option, +) (*Proxy, error) { + rpcClient, err := rpchttp.NewWithTimeout(providerAddr, config.WriteTimeout) + if err != nil { + return nil, fmt.Errorf("failed to create http client for %s: %w", providerAddr, err) + } + + return &Proxy{ + Addr: listenAddr, + Config: config, + Client: lrpc.NewClient(rpcClient, lightClient, opts...), + Logger: logger, + }, nil +} + // ListenAndServe configures the rpcserver.WebsocketManager, sets up the RPC // routes to proxy via Client, and starts up an HTTP server on the TCP network // address p.Addr. diff --git a/light/rpc/client.go b/light/rpc/client.go index 440a85f22..5c14619c8 100644 --- a/light/rpc/client.go +++ b/light/rpc/client.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "regexp" "time" "github.com/gogo/protobuf/proto" @@ -59,6 +60,27 @@ func KeyPathFn(fn KeyPathFunc) Option { } } +// DefaultMerkleKeyPathFn creates a function used to generate merkle key paths +// from a path string and a key. This is the default used by the cosmos SDK. +// This merkle key paths are required when verifying /abci_query calls +func DefaultMerkleKeyPathFn() KeyPathFunc { + // regexp for extracting store name from /abci_query path + storeNameRegexp := regexp.MustCompile(`\/store\/(.+)\/key`) + + return func(path string, key []byte) (merkle.KeyPath, error) { + matches := storeNameRegexp.FindStringSubmatch(path) + if len(matches) != 2 { + return nil, fmt.Errorf("can't find store name in %s using %s", path, storeNameRegexp) + } + storeName := matches[1] + + kp := merkle.KeyPath{} + kp = kp.AppendKey([]byte(storeName), merkle.KeyEncodingURL) + kp = kp.AppendKey(key, merkle.KeyEncodingURL) + return kp, nil + } +} + // NewClient returns a new client. func NewClient(next rpcclient.Client, lc LightClient, opts ...Option) *Client { c := &Client{ diff --git a/test/e2e/app/config.go b/test/e2e/app/config.go index 38c967916..d7e776538 100644 --- a/test/e2e/app/config.go +++ b/test/e2e/app/config.go @@ -14,6 +14,7 @@ type Config struct { Listen string Protocol string Dir string + Mode string `toml:"mode"` PersistInterval uint64 `toml:"persist_interval"` SnapshotInterval uint64 `toml:"snapshot_interval"` RetainBlocks uint64 `toml:"retain_blocks"` diff --git a/test/e2e/app/main.go b/test/e2e/app/main.go index 743b97b3a..25660ec44 100644 --- a/test/e2e/app/main.go +++ b/test/e2e/app/main.go @@ -1,11 +1,14 @@ package main import ( + "context" "errors" "fmt" "net" + "net/http" "os" "path/filepath" + "strings" "time" "github.com/spf13/viper" @@ -17,12 +20,18 @@ import ( tmflags "github.com/tendermint/tendermint/libs/cli/flags" "github.com/tendermint/tendermint/libs/log" tmnet "github.com/tendermint/tendermint/libs/net" + "github.com/tendermint/tendermint/light" + lproxy "github.com/tendermint/tendermint/light/proxy" + lrpc "github.com/tendermint/tendermint/light/rpc" + dbs "github.com/tendermint/tendermint/light/store/db" "github.com/tendermint/tendermint/node" "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/privval" grpcprivval "github.com/tendermint/tendermint/privval/grpc" privvalproto "github.com/tendermint/tendermint/proto/tendermint/privval" "github.com/tendermint/tendermint/proxy" + rpcserver "github.com/tendermint/tendermint/rpc/jsonrpc/server" + e2e "github.com/tendermint/tendermint/test/e2e/pkg" ) var logger = log.NewTMLogger(log.NewSyncWriter(os.Stdout)) @@ -66,10 +75,13 @@ func run(configFile string) error { case "socket", "grpc": err = startApp(cfg) case "builtin": + if cfg.Mode == string(e2e.ModeLight) { + err = startLightClient(cfg) + } else { + err = startNode(cfg) + } // FIXME: Temporarily remove maverick until it is redesigned // if len(cfg.Misbehaviors) == 0 { - err = startNode(cfg) - // } else { // err = startMaverick(cfg) // } default: @@ -137,6 +149,63 @@ func startNode(cfg *Config) error { return n.Start() } +func startLightClient(cfg *Config) error { + tmcfg, nodeLogger, _, err := setupNode() + if err != nil { + return err + } + + dbContext := &node.DBContext{ID: "light", Config: tmcfg} + lightDB, err := node.DefaultDBProvider(dbContext) + if err != nil { + return err + } + + providers := rpcEndpoints(tmcfg.P2P.PersistentPeers) + + c, err := light.NewHTTPClient( + context.Background(), + cfg.ChainID, + light.TrustOptions{ + Period: tmcfg.StateSync.TrustPeriod, + Height: tmcfg.StateSync.TrustHeight, + Hash: tmcfg.StateSync.TrustHashBytes(), + }, + providers[0], + providers[1:], + dbs.New(lightDB), + light.Logger(nodeLogger), + ) + if err != nil { + return err + } + + rpccfg := rpcserver.DefaultConfig() + rpccfg.MaxBodyBytes = tmcfg.RPC.MaxBodyBytes + rpccfg.MaxHeaderBytes = tmcfg.RPC.MaxHeaderBytes + rpccfg.MaxOpenConnections = tmcfg.RPC.MaxOpenConnections + // If necessary adjust global WriteTimeout to ensure it's greater than + // TimeoutBroadcastTxCommit. + // See https://github.com/tendermint/tendermint/issues/3435 + if rpccfg.WriteTimeout <= tmcfg.RPC.TimeoutBroadcastTxCommit { + rpccfg.WriteTimeout = tmcfg.RPC.TimeoutBroadcastTxCommit + 1*time.Second + } + + p, err := lproxy.NewProxy(c, tmcfg.RPC.ListenAddress, providers[0], rpccfg, nodeLogger, + lrpc.KeyPathFn(lrpc.DefaultMerkleKeyPathFn())) + if err != nil { + return err + } + + logger.Info("Starting proxy...", "laddr", tmcfg.RPC.ListenAddress) + if err := p.ListenAndServe(); err != http.ErrServerClosed { + // Error starting or closing listener: + logger.Error("proxy ListenAndServe", "err", err) + } + + return nil +} + // FIXME: Temporarily disconnected maverick until it is redesigned // startMaverick starts a Maverick node that runs the application directly. It assumes the Tendermint // configuration is in $TMHOME/config/tendermint.toml. @@ -267,3 +336,21 @@ func setupNode() (*config.Config, log.Logger, *p2p.NodeKey, error) { return tmcfg, nodeLogger, &nodeKey, nil } + +// rpcEndpoints takes a list of persistent peers and splits them into a list of rpc endpoints +// using 26657 as the port number +func rpcEndpoints(peers string) []string { + arr := strings.Split(peers, ",") + endpoints := make([]string, len(arr)) + for i, v := range arr { + addr, err := p2p.ParseNodeAddress(v) + if err != nil { + panic(err) + } + // use RPC port instead + addr.Port = 26657 + rpcEndpoint := "http://" + addr.Hostname + ":" + fmt.Sprint(addr.Port) + endpoints[i] = rpcEndpoint + } + return endpoints +} diff --git a/test/e2e/generator/generate.go b/test/e2e/generator/generate.go index 42cb3d1a5..85646bd74 100644 --- a/test/e2e/generator/generate.go +++ b/test/e2e/generator/generate.go @@ -77,7 +77,7 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er KeyType: opt["keyType"].(string), } - var numSeeds, numValidators, numFulls int + var numSeeds, numValidators, numFulls, numLightClients int switch opt["topology"].(string) { case "single": numValidators = 1 @@ -85,7 +85,8 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er numValidators = 4 case "large": // FIXME Networks are kept small since large ones use too much CPU. - numSeeds = r.Intn(4) + numSeeds = r.Intn(3) + numLightClients = r.Intn(3) numValidators = 4 + r.Intn(7) numFulls = r.Intn(5) default: @@ -143,6 +144,13 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er r, e2e.ModeFull, startAt, manifest.InitialHeight, false) } + for i := 1; i <= numLightClients; i++ { + startAt := manifest.InitialHeight + 5 + manifest.Nodes[fmt.Sprintf("light%02d", i)] = generateNode( + r, e2e.ModeLight, startAt+(5*int64(i)), manifest.InitialHeight, false, + ) + } + // We now set up peer discovery for nodes. Seed nodes are fully meshed with // each other, while non-seed nodes either use a set of random seeds or a // set of random peers that start before themselves. @@ -175,6 +183,10 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er } }) for i, name := range peerNames { + // we skip over light clients - they connect to all peers initially + if manifest.Nodes[name].Mode == string(e2e.ModeLight) { + continue + } if len(seedNames) > 0 && (i == 0 || r.Float64() >= 0.5) { manifest.Nodes[name].Seeds = uniformSetChoice(seedNames).Choose(r) } else if i > 0 { @@ -213,7 +225,7 @@ func generateNode( node.SnapshotInterval = 3 } - if node.Mode == "validator" { + if node.Mode == string(e2e.ModeValidator) { misbehaveAt := startAt + 5 + int64(r.Intn(10)) if startAt == 0 { misbehaveAt += initialHeight - 1 @@ -224,6 +236,11 @@ func generateNode( } } + if node.Mode == string(e2e.ModeLight) { + node.ABCIProtocol = "builtin" + node.StateSync = false + } + // If a node which does not persist state also does not retain blocks, randomly // choose to either persist state or retain all blocks. if node.PersistInterval != nil && *node.PersistInterval == 0 && node.RetainBlocks > 0 { diff --git a/test/e2e/networks/ci.toml b/test/e2e/networks/ci.toml index 93b5a18bd..fbf6ef067 100644 --- a/test/e2e/networks/ci.toml +++ b/test/e2e/networks/ci.toml @@ -90,3 +90,13 @@ fast_sync = "v0" state_sync = true seeds = ["seed01"] perturb = ["restart"] + +[node.light01] +mode= "light" +start_at= 1005 +persistent_peers = ["validator01", "validator02", "validator03"] + +[node.light02] +mode= "light" +start_at= 1015 +persistent_peers = ["validator04", "full01", "validator05"] \ No newline at end of file diff --git a/test/e2e/networks/simple.toml b/test/e2e/networks/simple.toml index 37f711a91..05cda1819 100644 --- a/test/e2e/networks/simple.toml +++ b/test/e2e/networks/simple.toml @@ -2,4 +2,4 @@ [node.validator02] [node.validator03] [node.validator04] - + diff --git a/test/e2e/pkg/manifest.go b/test/e2e/pkg/manifest.go index c8c0d5c3c..8fbaea185 100644 --- a/test/e2e/pkg/manifest.go +++ b/test/e2e/pkg/manifest.go @@ -58,9 +58,9 @@ type Manifest struct { // ManifestNode represents a node in a testnet manifest. type ManifestNode struct { - // Mode specifies the type of node: "validator", "full", or "seed". Defaults to - // "validator". Full nodes do not get a signing key (a dummy key is generated), - // and seed nodes run in seed mode with the PEX reactor enabled. + // Mode specifies the type of node: "validator", "full", "light" or "seed". + // Defaults to "validator". Full nodes do not get a signing key (a dummy key + // is generated), and seed nodes run in seed mode with the PEX reactor enabled. Mode string `toml:"mode"` // Seeds is the list of node names to use as P2P seed nodes. Defaults to none. @@ -68,7 +68,8 @@ type ManifestNode struct { // PersistentPeers is a list of node names to maintain persistent P2P // connections to. If neither seeds nor persistent peers are specified, - // this defaults to all other nodes in the network. + // this defaults to all other nodes in the network. For light clients, + // this relates to the providers the light client is connected to. PersistentPeers []string `toml:"persistent_peers"` // Database specifies the database backend: "goleveldb", "cleveldb", diff --git a/test/e2e/pkg/testnet.go b/test/e2e/pkg/testnet.go index 52ff83ba3..b773a158a 100644 --- a/test/e2e/pkg/testnet.go +++ b/test/e2e/pkg/testnet.go @@ -33,6 +33,7 @@ type Perturbation string const ( ModeValidator Mode = "validator" ModeFull Mode = "full" + ModeLight Mode = "light" ModeSeed Mode = "seed" ProtocolBuiltin Protocol = "builtin" @@ -151,7 +152,7 @@ func LoadTestnet(file string) (*Testnet, error) { ProxyPort: proxyPortGen.Next(), Mode: ModeValidator, Database: "goleveldb", - ABCIProtocol: ProtocolUNIX, + ABCIProtocol: ProtocolBuiltin, PrivvalProtocol: ProtocolFile, StartAt: nodeManifest.StartAt, FastSync: nodeManifest.FastSync, @@ -327,6 +328,9 @@ func (n Node) Validate(testnet Testnet) error { default: return fmt.Errorf("invalid ABCI protocol setting %q", n.ABCIProtocol) } + if n.Mode == ModeLight && n.ABCIProtocol != ProtocolBuiltin { + return errors.New("light client must use builtin protocol") + } switch n.PrivvalProtocol { case ProtocolFile, ProtocolTCP, ProtocolGRPC, ProtocolUNIX: default: @@ -402,7 +406,7 @@ func (t Testnet) LookupNode(name string) *Node { func (t Testnet) ArchiveNodes() []*Node { nodes := []*Node{} for _, node := range t.Nodes { - if node.Mode != ModeSeed && node.StartAt == 0 && node.RetainBlocks == 0 { + if !node.Stateless() && node.StartAt == 0 && node.RetainBlocks == 0 { nodes = append(nodes, node) } } @@ -476,6 +480,11 @@ func (n Node) Client() (*rpchttp.HTTP, error) { return rpchttp.New(fmt.Sprintf("http://127.0.0.1:%v", n.ProxyPort)) } +// Stateless returns true if the node is either a seed node or a light node +func (n Node) Stateless() bool { + return n.Mode == ModeLight || n.Mode == ModeSeed +} + // keyGenerator generates pseudorandom Ed25519 keys based on a seed. type keyGenerator struct { random *rand.Rand diff --git a/test/e2e/runner/setup.go b/test/e2e/runner/setup.go index 3528d67cb..2f4d704b3 100644 --- a/test/e2e/runner/setup.go +++ b/test/e2e/runner/setup.go @@ -65,23 +65,23 @@ func Setup(testnet *e2e.Testnet) error { for _, node := range testnet.Nodes { nodeDir := filepath.Join(testnet.Dir, node.Name) + dirs := []string{ filepath.Join(nodeDir, "config"), filepath.Join(nodeDir, "data"), filepath.Join(nodeDir, "data", "app"), } for _, dir := range dirs { + // light clients don't need an app directory + if node.Mode == e2e.ModeLight && strings.Contains(dir, "app") { + continue + } err := os.MkdirAll(dir, 0755) if err != nil { return err } } - err = genesis.SaveAs(filepath.Join(nodeDir, "config", "genesis.json")) - if err != nil { - return err - } - cfg, err := MakeConfig(node) if err != nil { return err @@ -97,6 +97,16 @@ func Setup(testnet *e2e.Testnet) error { return err } + if node.Mode == e2e.ModeLight { + // stop early if a light client + continue + } + + err = genesis.SaveAs(filepath.Join(nodeDir, "config", "genesis.json")) + if err != nil { + return err + } + err = (&p2p.NodeKey{PrivKey: node.NodeKey}).SaveAs(filepath.Join(nodeDir, "config", "node_key.json")) if err != nil { return err @@ -168,8 +178,7 @@ services: image: tendermint/e2e-node {{- if eq .ABCIProtocol "builtin" }} entrypoint: /usr/bin/entrypoint-builtin -{{- end }} -{{- if ne .ABCIProtocol "builtin"}} +{{- else }} command: {{ startCommands .Misbehaviors .LogLevel }} {{- end }} init: true @@ -289,7 +298,7 @@ func MakeConfig(node *e2e.Node) (*config.Config, error) { case e2e.ModeSeed: cfg.P2P.SeedMode = true cfg.P2P.PexReactor = true - case e2e.ModeFull: + case e2e.ModeFull, e2e.ModeLight: // Don't need to do anything, since we're using a dummy privval key by default. default: return nil, fmt.Errorf("unexpected mode %q", node.Mode) @@ -338,6 +347,8 @@ func MakeAppConfig(node *e2e.Node) ([]byte, error) { "chain_id": node.Testnet.Name, "dir": "data/app", "listen": AppAddressUNIX, + "mode": node.Mode, + "proxy_port": node.ProxyPort, "protocol": "socket", "persist_interval": node.PersistInterval, "snapshot_interval": node.SnapshotInterval, diff --git a/test/e2e/runner/start.go b/test/e2e/runner/start.go index 53acdd821..5a1768b32 100644 --- a/test/e2e/runner/start.go +++ b/test/e2e/runner/start.go @@ -58,7 +58,7 @@ func Start(testnet *e2e.Testnet) error { // Update any state sync nodes with a trusted height and hash for _, node := range nodeQueue { - if node.StateSync { + if node.StateSync || node.Mode == e2e.ModeLight { err = UpdateConfigStateSync(node, block.Height, blockID.Hash.Bytes()) if err != nil { return err diff --git a/test/e2e/tests/app_test.go b/test/e2e/tests/app_test.go index 82e788ebd..b2a84e5fa 100644 --- a/test/e2e/tests/app_test.go +++ b/test/e2e/tests/app_test.go @@ -16,7 +16,7 @@ import ( // Tests that any initial state given in genesis has made it into the app. func TestApp_InitialState(t *testing.T) { testNode(t, func(t *testing.T, node e2e.Node) { - if node.Mode == e2e.ModeSeed { + if node.Stateless() { return } if len(node.Testnet.InitialState) == 0 { @@ -84,9 +84,24 @@ func TestApp_Tx(t *testing.T) { _, err = client.BroadcastTxCommit(ctx, tx) require.NoError(t, err) - resp, err := client.ABCIQuery(ctx, "", []byte(key)) + // wait for the tx to go through + time.Sleep(1 * time.Second) + + hash := tx.Hash() + txResp, err := client.Tx(ctx, hash, false) require.NoError(t, err) - assert.Equal(t, key, string(resp.Response.Key)) - assert.Equal(t, value, string(resp.Response.Value)) + assert.Equal(t, txResp.Tx, tx) + + if node.Mode == e2e.ModeLight { + return + } + + abciResp, err := client.ABCIQuery(ctx, "", []byte(key)) + require.NoError(t, err) + assert.Equal(t, key, string(abciResp.Response.Key)) + assert.Equal(t, value, string(abciResp.Response.Value)) + + assert.Equal(t, txResp.Height, abciResp.Response.Height) + }) }