diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 7ee898323..a3126d1f5 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -47,6 +47,7 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi - [crypto/ed25519] \#5632 Adopt zip215 `ed25519` verification. (@marbar3778) - [privval] \#5603 Add `--key` to `init`, `gen_validator`, `testnet` & `unsafe_reset_priv_validator` for use in generating `secp256k1` keys. +- [privval] \#5725 add gRPC support to private validator. - [abci/client] \#5673 `Async` requests return an error if queue is full (@melekes) - [mempool] \#5673 Cancel `CheckTx` requests if RPC client disconnects or times out (@melekes) - [abci] \#5706 Added `AbciVersion` to `RequestInfo` allowing applications to check ABCI version when connecting to Tendermint. (@marbar3778) diff --git a/Makefile b/Makefile index 34d1f7da2..90d651dbf 100644 --- a/Makefile +++ b/Makefile @@ -63,7 +63,7 @@ include tools/Makefile include test/Makefile ############################################################################### -### Build Tendermint ### +### Build Tendermint ### ############################################################################### build: $(BUILDDIR)/ @@ -127,6 +127,27 @@ install_abci: @go install -mod=readonly ./abci/cmd/... .PHONY: install_abci +############################################################################### +### Privval Server ### +############################################################################### + +build_privval_server: + @go build -mod=readonly -o $(BUILDDIR)/ -i ./cmd/priv_val_server/... +.PHONY: build_privval_server + +generate_test_cert: + # generate self signing ceritificate authority + @certstrap init --common-name "root CA" --expires "20 years" + # generate server cerificate + @certstrap request-cert -cn server -ip 127.0.0.1 + # self-sign server cerificate with rootCA + @certstrap sign server --CA "root CA" + # generate client cerificate + @certstrap request-cert -cn client -ip 127.0.0.1 + # self-sign client cerificate with rootCA + @certstrap sign client --CA "root CA" +.PHONY: generate_test_cert + ############################################################################### ### Distribution ### ############################################################################### diff --git a/cmd/priv_val_server/main.go b/cmd/priv_val_server/main.go index 813ce6b14..03d375693 100644 --- a/cmd/priv_val_server/main.go +++ b/cmd/priv_val_server/main.go @@ -1,24 +1,50 @@ package main import ( + "context" + "crypto/tls" + "crypto/x509" "flag" + "fmt" + "io/ioutil" + "net" + "net/http" "os" "time" - "github.com/tendermint/tendermint/crypto/ed25519" + grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "github.com/tendermint/tendermint/libs/log" tmnet "github.com/tendermint/tendermint/libs/net" tmos "github.com/tendermint/tendermint/libs/os" - "github.com/tendermint/tendermint/privval" + grpcprivval "github.com/tendermint/tendermint/privval/grpc" + privvalproto "github.com/tendermint/tendermint/proto/tendermint/privval" +) + +var ( + // Create a metrics registry. + reg = prometheus.NewRegistry() + + // Create some standard server metrics. + grpcMetrics = grpc_prometheus.NewServerMetrics() ) func main() { var ( - addr = flag.String("addr", ":26659", "Address of client to connect to") + addr = flag.String("addr", "127.0.0.1:26659", "Address to listen on (host:port)") chainID = flag.String("chain-id", "mychain", "chain id") privValKeyPath = flag.String("priv-key", "", "priv val key file path") privValStatePath = flag.String("priv-state", "", "priv val state file path") + insecure = flag.Bool("insecure", false, "allow server to run insecurely (no TLS)") + certFile = flag.String("certfile", "", "absolute path to server certificate") + keyFile = flag.String("keyfile", "", "absolute path to server key") + rootCA = flag.String("rootcafile", "", "absolute path to root CA") + prometheusAddr = flag.String("prometheus-addr", "", "address for prometheus endpoint (host:port)") logger = log.NewTMLogger( log.NewSyncWriter(os.Stdout), @@ -32,39 +58,106 @@ func main() { "chainID", *chainID, "privKeyPath", *privValKeyPath, "privStatePath", *privValStatePath, + "insecure", *insecure, + "certFile", *certFile, + "keyFile", *keyFile, + "rootCA", *rootCA, ) pv := privval.LoadFilePV(*privValKeyPath, *privValStatePath) - var dialer privval.SocketDialer + opts := []grpc.ServerOption{} + if !*insecure { + certificate, err := tls.LoadX509KeyPair(*certFile, *keyFile) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to load X509 key pair: %v", err) + os.Exit(1) + } + + certPool := x509.NewCertPool() + bs, err := ioutil.ReadFile(*rootCA) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to read client ca cert: %s", err) + os.Exit(1) + } + + if ok := certPool.AppendCertsFromPEM(bs); !ok { + fmt.Fprintf(os.Stderr, "failed to append client certs") + os.Exit(1) + } + + tlsConfig := &tls.Config{ + ClientAuth: tls.RequireAndVerifyClientCert, + Certificates: []tls.Certificate{certificate}, + ClientCAs: certPool, + MinVersion: tls.VersionTLS13, + } + + creds := grpc.Creds(credentials.NewTLS(tlsConfig)) + opts = append(opts, creds) + logger.Info("SignerServer: Creating security credentials") + } else { + logger.Info("SignerServer: You are using an insecure gRPC connection!") + } + + // add prometheus metrics for unary RPC calls + opts = append(opts, grpc.UnaryInterceptor(grpc_prometheus.UnaryServerInterceptor)) + + ss := grpcprivval.NewSignerServer(*chainID, pv, logger) + protocol, address := tmnet.ProtocolAndAddress(*addr) - switch protocol { - case "unix": - dialer = privval.DialUnixFn(address) - case "tcp": - connTimeout := 3 * time.Second // TODO - dialer = privval.DialTCPFn(address, connTimeout, ed25519.GenPrivKey()) - default: - logger.Error("Unknown protocol", "protocol", protocol) + + lis, err := net.Listen(protocol, address) + if err != nil { + fmt.Fprintf(os.Stderr, "SignerServer: Failed to listen %v", err) os.Exit(1) } - sd := privval.NewSignerDialerEndpoint(logger, dialer) - ss := privval.NewSignerServer(sd, *chainID, pv) + s := grpc.NewServer(opts...) - err := ss.Start() - if err != nil { - panic(err) + privvalproto.RegisterPrivValidatorAPIServer(s, ss) + + var httpSrv *http.Server + if *prometheusAddr != "" { + httpSrv = registerPrometheus(*prometheusAddr, s) + } + + logger.Info("SignerServer: Starting grpc server") + if err := s.Serve(lis); err != nil { + fmt.Fprintf(os.Stderr, "Unable to listen on port %s: %v", *addr, err) + os.Exit(1) } // Stop upon receiving SIGTERM or CTRL-C. tmos.TrapSignal(logger, func() { - err := ss.Stop() - if err != nil { - panic(err) + logger.Debug("SignerServer: calling Close") + if *prometheusAddr != "" { + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + if err := httpSrv.Shutdown(ctx); err != nil { + fmt.Fprintf(os.Stderr, "Unable to stop http server: %v", err) + os.Exit(1) + } } + s.GracefulStop() }) // Run forever. select {} } + +func registerPrometheus(addr string, s *grpc.Server) *http.Server { + // Initialize all metrics. + grpcMetrics.InitializeMetrics(s) + // create http server to serve prometheus + httpServer := &http.Server{Handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), Addr: addr} + + go func() { + if err := httpServer.ListenAndServe(); err != nil { + fmt.Fprintf(os.Stderr, "Unable to start a http server: %v", err) + os.Exit(1) + } + }() + + return httpServer +} diff --git a/config/config.go b/config/config.go index fd8b6669d..f66312127 100644 --- a/config/config.go +++ b/config/config.go @@ -204,6 +204,16 @@ type BaseConfig struct { //nolint: maligned // connections from an external PrivValidator process PrivValidatorListenAddr string `mapstructure:"priv-validator-laddr"` + // Client certificate generated while creating needed files for secure connection. + // If a remote validator address is provided but no certificate, the connection will be insecure + PrivValidatorClientCertificate string `mapstructure:"priv-validator-client-certificate-file"` + + // Client key generated while creating certificates for secure connection + PrivValidatorClientKey string `mapstructure:"priv-validator-client-key-file"` + + // Path Root Certificate Authority used to sign both client and server certificates + PrivValidatorRootCA string `mapstructure:"priv-validator-root-ca-file"` + // A JSON file containing the private key to use for p2p authenticated encryption NodeKey string `mapstructure:"node-key-file"` @@ -253,6 +263,21 @@ func (cfg BaseConfig) GenesisFile() string { return rootify(cfg.Genesis, cfg.RootDir) } +// PrivValidatorClientKeyFile returns the full path to the priv_validator_key.json file +func (cfg BaseConfig) PrivValidatorClientKeyFile() string { + return rootify(cfg.PrivValidatorClientKey, cfg.RootDir) +} + +// PrivValidatorClientCertificateFile returns the full path to the priv_validator_key.json file +func (cfg BaseConfig) PrivValidatorClientCertificateFile() string { + return rootify(cfg.PrivValidatorClientCertificate, cfg.RootDir) +} + +// PrivValidatorCertificateAuthorityFile returns the full path to the priv_validator_key.json file +func (cfg BaseConfig) PrivValidatorRootCAFile() string { + return rootify(cfg.PrivValidatorRootCA, cfg.RootDir) +} + // PrivValidatorKeyFile returns the full path to the priv_validator_key.json file func (cfg BaseConfig) PrivValidatorKeyFile() string { return rootify(cfg.PrivValidatorKey, cfg.RootDir) @@ -273,6 +298,19 @@ func (cfg BaseConfig) DBDir() string { return rootify(cfg.DBPath, cfg.RootDir) } +func (cfg *BaseConfig) ArePrivValidatorClientSecurityOptionsPresent() bool { + switch { + case cfg.PrivValidatorRootCA == "": + return false + case cfg.PrivValidatorClientKey == "": + return false + case cfg.PrivValidatorClientCertificate == "": + return false + default: + return true + } +} + // ValidateBasic performs basic validation (checking param bounds, etc.) and // returns an error if any check fails. func (cfg BaseConfig) ValidateBasic() error { diff --git a/config/toml.go b/config/toml.go index ed29ab46c..34152acca 100644 --- a/config/toml.go +++ b/config/toml.go @@ -136,8 +136,19 @@ priv-validator-state-file = "{{ js .BaseConfig.PrivValidatorState }}" # TCP or UNIX socket address for Tendermint to listen on for # connections from an external PrivValidator process +# when the listenAddr is prefixed with grpc instead of tcp it will use the gRPC Client priv-validator-laddr = "{{ .BaseConfig.PrivValidatorListenAddr }}" +# Client certificate generated while creating needed files for secure connection. +# If a remote validator address is provided but no certificate, the connection will be insecure +priv-validator-client-certificate-file = "{{ js .BaseConfig.PrivValidatorClientCertificate }}" + +# Client key generated while creating certificates for secure connection +priv-validator-client-key-file = "{{ js .BaseConfig.PrivValidatorClientKey }}" + +# Path Root Certificate Authority used to sign both client and server certificates +priv-validator-certificate-authority = "{{ js .BaseConfig.PrivValidatorRootCA }}" + # Path to the JSON file containing the private key to use for node authentication in the p2p protocol node-key-file = "{{ js .BaseConfig.NodeKey }}" diff --git a/docs/nodes/remote_signer.md b/docs/nodes/remote_signer.md new file mode 100644 index 000000000..e7dfccacd --- /dev/null +++ b/docs/nodes/remote_signer.md @@ -0,0 +1,70 @@ +--- +order: 7 +--- + +# Remote signer + +Tendermint provides a remote signer option for validators. A remote signer enables the operator to store the validator key on a different machine minimizing the attack surface if a server were to be compromised. + +The remote signer protocol implements a [client and server architecture](https://en.wikipedia.org/wiki/Client%E2%80%93server_model). When Tendermint requires the public key or signature for a proposal or vote it requests it from the remote signer. + +To run a secure validator and remote signer system it is recommended to use a VPC (virtual private cloud) or a private connection. + +There are two different configurations that can be used: Raw or gRPC. + +## Raw + +While both options use tcp or unix sockets the raw option uses tcp or unix sockets without http. The raw protocol sets up Tendermint as the server and the remote signer as the client. This aids in not exposing the remote signer to public network. + +> Warning: Raw will be deprecated in a future major release, we recommend implementing your key management server against the gRPC configuration. + +## gRPC + +[gRPC](https://grpc.io/) is an RPC framework built with [HTTP/2](https://en.wikipedia.org/wiki/HTTP/2), uses [Protocol Buffers](https://developers.google.com/protocol-buffers) to define services and has been standardized within the cloud infrastructure community. gRPC provides a language agnostic way to implement services. This aids developers in the writing key management servers in various different languages. + +GRPC utilizes [TLS](https://en.wikipedia.org/wiki/Transport_Layer_Security), another widely standardized protocol, to secure connections. There are two forms of TLS to secure a connection, one-way and two-way. One way is when the client identifies the server but the server allows anyone to connect to it. Two-way is when the client identifies the server and the server identifies the client, prohibiting connections from unknown parties. + +When using gRPC Tendermint is setup as the client. Tendermint will make calls to the remote signer. We recommend not exposing the remote signer to the public network with the use of virtual private cloud. + +Securing your remote signers connection is highly recommended, but we provide the option to run it with a insecure connection. + +### Generating Certificates + +To run a secure connection with gRPC we need to generate certificates and keys. We will walkthrough how to self sign certificates for two-way TLS. + +There are two ways to generate certificates, [openssl](https://www.openssl.org/) and [certstarp](https://github.com/square/certstrap). Both of these options can be used but we will be covering `certstrap` because it provides a simpler process then openssl. + +- Install `Certstrap`: + +```sh + go get github.com/square/certstrap@v1.2.0 +``` + +- Create certificate authority for self signing. + +```sh + # generate self signing ceritificate authority + certstrap init --common-name "" --expires "20 years" +``` + +- Request a certificate for the server. + - For generalization purposes we set the ip to `127.0.0.1`, but for your node please use the servers IP. +- Sign the servers certificate with your certificate authority + +```sh + # generate server cerificate + certstrap request-cert -cn server -ip 127.0.0.1 + # self-sign server cerificate with rootCA + certstrap sign server --CA "" 127.0.0.1 + ``` + +- Request a certificate for the client. + - For generalization purposes we set the ip to `127.0.0.1`, but for your node please use the clients IP. +- Sign the clients certificate with your certificate authority + +```sh +# generate client cerificate + certstrap request-cert -cn client -ip 127.0.0.1 +# self-sign client cerificate with rootCA + certstrap sign client --CA "" 127.0.0.1 +``` diff --git a/go.mod b/go.mod index 08fc9efc4..90a93e7ab 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,8 @@ require ( github.com/golang/protobuf v1.4.3 github.com/google/orderedcode v0.0.1 github.com/gorilla/websocket v1.4.2 + github.com/grpc-ecosystem/go-grpc-middleware v1.2.2 + github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/gtank/merlin v0.1.1 github.com/hdevalence/ed25519consensus v0.0.0-20201207055737-7fde80a9d5ff github.com/libp2p/go-buffer-pool v0.0.2 diff --git a/go.sum b/go.sum index d110a35e2..baf3477ef 100644 --- a/go.sum +++ b/go.sum @@ -32,9 +32,11 @@ github.com/Workiva/go-datastructures v1.0.52/go.mod h1:Z+F2Rca0qCsVYDS8z7bAGm8f3 github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= @@ -243,7 +245,9 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.2.1/go.mod h1:EaizFBKfUKtMIF5iaDEhniwNedqGo9FuLFzppDr3uwI= +github.com/grpc-ecosystem/go-grpc-middleware v1.2.2 h1:FlFbCRLd5Jr4iYXZufAvgWN6Ao0JrI5chLINnUXDDr0= github.com/grpc-ecosystem/go-grpc-middleware v1.2.2/go.mod h1:EaizFBKfUKtMIF5iaDEhniwNedqGo9FuLFzppDr3uwI= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= @@ -481,6 +485,7 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -812,6 +817,7 @@ google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEG google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/libs/net/net_test.go b/libs/net/net_test.go index 38cd58f6a..5ce5965c2 100644 --- a/libs/net/net_test.go +++ b/libs/net/net_test.go @@ -18,6 +18,11 @@ func TestProtocolAndAddress(t *testing.T) { "tcp", "mydomain:80", }, + { + "grpc://mydomain:80", + "grpc", + "mydomain:80", + }, { "mydomain:80", "tcp", diff --git a/node/node.go b/node/node.go index b79aa1d6c..d9e504758 100644 --- a/node/node.go +++ b/node/node.go @@ -8,12 +8,13 @@ import ( "net" "net/http" _ "net/http/pprof" // nolint: gosec // securely exposed on separate, optional port - "strings" "time" + grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/rs/cors" + "google.golang.org/grpc" dbm "github.com/tendermint/tm-db" @@ -26,6 +27,7 @@ import ( "github.com/tendermint/tendermint/evidence" tmjson "github.com/tendermint/tendermint/libs/json" "github.com/tendermint/tendermint/libs/log" + tmnet "github.com/tendermint/tendermint/libs/net" tmpubsub "github.com/tendermint/tendermint/libs/pubsub" "github.com/tendermint/tendermint/libs/service" "github.com/tendermint/tendermint/light" @@ -33,6 +35,7 @@ import ( "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/p2p/pex" "github.com/tendermint/tendermint/privval" + tmgrpc "github.com/tendermint/tendermint/privval/grpc" "github.com/tendermint/tendermint/proxy" rpccore "github.com/tendermint/tendermint/rpc/core" grpccore "github.com/tendermint/tendermint/rpc/grpc" @@ -675,10 +678,19 @@ func NewNode(config *cfg.Config, // If an address is provided, listen on the socket for a connection from an // external signing process. if config.PrivValidatorListenAddr != "" { + protocol, address := tmnet.ProtocolAndAddress(config.PrivValidatorListenAddr) // FIXME: we should start services inside OnStart - privValidator, err = createAndStartPrivValidatorSocketClient(config.PrivValidatorListenAddr, genDoc.ChainID, logger) - if err != nil { - return nil, fmt.Errorf("error with private validator socket client: %w", err) + switch protocol { + case "grpc": + privValidator, err = createAndStartPrivValidatorGRPCClient(config, address, genDoc.ChainID, logger) + if err != nil { + return nil, fmt.Errorf("error with private validator grpc client: %w", err) + } + default: + privValidator, err = createAndStartPrivValidatorSocketClient(config.PrivValidatorListenAddr, genDoc.ChainID, logger) + if err != nil { + return nil, fmt.Errorf("error with private validator socket client: %w", err) + } } } @@ -1395,6 +1407,7 @@ func createAndStartPrivValidatorSocketClient( chainID string, logger log.Logger, ) (types.PrivValidator, error) { + pve, err := privval.NewSignerListener(listenAddr, logger) if err != nil { return nil, fmt.Errorf("failed to start private validator: %w", err) @@ -1420,23 +1433,44 @@ func createAndStartPrivValidatorSocketClient( return pvscWithRetries, nil } -// splitAndTrimEmpty slices s into all subslices separated by sep and returns a -// slice of the string s with all leading and trailing Unicode code points -// contained in cutset removed. If sep is empty, SplitAndTrim splits after each -// UTF-8 sequence. First part is equivalent to strings.SplitN with a count of -// -1. also filter out empty strings, only return non-empty strings. -func splitAndTrimEmpty(s, sep, cutset string) []string { - if s == "" { - return []string{} - } - - spl := strings.Split(s, sep) - nonEmptyStrings := make([]string, 0, len(spl)) - for i := 0; i < len(spl); i++ { - element := strings.Trim(spl[i], cutset) - if element != "" { - nonEmptyStrings = append(nonEmptyStrings, element) - } +func createAndStartPrivValidatorGRPCClient( + config *cfg.Config, + address, + chainID string, + logger log.Logger, +) (types.PrivValidator, error) { + var transportSecurity grpc.DialOption + if config.BaseConfig.ArePrivValidatorClientSecurityOptionsPresent() { + transportSecurity = tmgrpc.GenerateTLS(config.PrivValidatorClientCertificateFile(), + config.PrivValidatorClientKeyFile(), config.PrivValidatorRootCAFile(), logger) + } else { + transportSecurity = grpc.WithInsecure() + logger.Info("Using an insecure gRPC connection!") + } + dialOptions := tmgrpc.DefaultDialOptions() + if config.Instrumentation.Prometheus { + grpcMetrics := grpc_prometheus.DefaultClientMetrics + dialOptions = append(dialOptions, grpc.WithUnaryInterceptor(grpcMetrics.UnaryClientInterceptor())) + } + + dialOptions = append(dialOptions, transportSecurity) + + ctx := context.Background() + + conn, err := grpc.DialContext(ctx, address, dialOptions...) + if err != nil { + logger.Error("unable to connect to server", "target", address, "err", err) + } + pvsc, err := tmgrpc.NewSignerClient(conn, chainID, logger) + if err != nil { + return nil, fmt.Errorf("failed to start private validator: %w", err) } - return nonEmptyStrings + + // try to get a pubkey from private validate first time + _, err = pvsc.GetPubKey() + if err != nil { + return nil, fmt.Errorf("can't get pubkey: %w", err) + } + + return pvsc, nil } diff --git a/node/utils.go b/node/utils.go new file mode 100644 index 000000000..a96dd457f --- /dev/null +++ b/node/utils.go @@ -0,0 +1,26 @@ +package node + +import ( + "strings" +) + +// splitAndTrimEmpty slices s into all subslices separated by sep and returns a +// slice of the string s with all leading and trailing Unicode code points +// contained in cutset removed. If sep is empty, SplitAndTrim splits after each +// UTF-8 sequence. First part is equivalent to strings.SplitN with a count of +// -1. also filter out empty strings, only return non-empty strings. +func splitAndTrimEmpty(s, sep, cutset string) []string { + if s == "" { + return []string{} + } + + spl := strings.Split(s, sep) + nonEmptyStrings := make([]string, 0, len(spl)) + for i := 0; i < len(spl); i++ { + element := strings.Trim(spl[i], cutset) + if element != "" { + nonEmptyStrings = append(nonEmptyStrings, element) + } + } + return nonEmptyStrings +} diff --git a/privval/grpc/client.go b/privval/grpc/client.go new file mode 100644 index 000000000..5b44973e8 --- /dev/null +++ b/privval/grpc/client.go @@ -0,0 +1,108 @@ +package grpc + +import ( + "context" + "time" + + grpc "google.golang.org/grpc" + "google.golang.org/grpc/status" + + "github.com/tendermint/tendermint/crypto" + cryptoenc "github.com/tendermint/tendermint/crypto/encoding" + "github.com/tendermint/tendermint/libs/log" + privvalproto "github.com/tendermint/tendermint/proto/tendermint/privval" + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" + "github.com/tendermint/tendermint/types" +) + +// SignerClient implements PrivValidator. +// Handles remote validator connections that provide signing services +type SignerClient struct { + logger log.Logger + + client privvalproto.PrivValidatorAPIClient + conn *grpc.ClientConn + chainID string +} + +var _ types.PrivValidator = (*SignerClient)(nil) + +// NewSignerClient returns an instance of SignerClient. +// it will start the endpoint (if not already started) +func NewSignerClient(conn *grpc.ClientConn, + chainID string, log log.Logger) (*SignerClient, error) { + + sc := &SignerClient{ + logger: log, + chainID: chainID, + client: privvalproto.NewPrivValidatorAPIClient(conn), // Create the Private Validator Client + } + + return sc, nil +} + +// Close closes the underlying connection +func (sc *SignerClient) Close() error { + sc.logger.Info("Stopping service") + if sc.conn != nil { + return sc.conn.Close() + } + return nil +} + +//-------------------------------------------------------- +// Implement PrivValidator + +// GetPubKey retrieves a public key from a remote signer +// returns an error if client is not able to provide the key +func (sc *SignerClient) GetPubKey() (crypto.PubKey, error) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) // Todo: should this be configurable? + defer cancel() + resp, err := sc.client.GetPubKey(ctx, &privvalproto.PubKeyRequest{ChainId: sc.chainID}) + if err != nil { + errStatus, _ := status.FromError(err) + sc.logger.Error("SignerClient::GetPubKey", "err", errStatus.Message()) + return nil, errStatus.Err() + } + + pk, err := cryptoenc.PubKeyFromProto(resp.PubKey) + if err != nil { + return nil, err + } + + return pk, nil +} + +// SignVote requests a remote signer to sign a vote +func (sc *SignerClient) SignVote(chainID string, vote *tmproto.Vote) error { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + resp, err := sc.client.SignVote(ctx, &privvalproto.SignVoteRequest{ChainId: sc.chainID, Vote: vote}) + if err != nil { + errStatus, _ := status.FromError(err) + sc.logger.Error("Client SignVote", "err", errStatus.Message()) + return errStatus.Err() + } + + *vote = resp.Vote + + return nil +} + +// SignProposal requests a remote signer to sign a proposal +func (sc *SignerClient) SignProposal(chainID string, proposal *tmproto.Proposal) error { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + resp, err := sc.client.SignProposal( + ctx, &privvalproto.SignProposalRequest{ChainId: chainID, Proposal: proposal}) + + if err != nil { + errStatus, _ := status.FromError(err) + sc.logger.Error("SignerClient::SignProposal", "err", errStatus.Message()) + return errStatus.Err() + } + + *proposal = resp.Proposal + + return nil +} diff --git a/privval/grpc/client_test.go b/privval/grpc/client_test.go new file mode 100644 index 000000000..09feea731 --- /dev/null +++ b/privval/grpc/client_test.go @@ -0,0 +1,168 @@ +package grpc_test + +import ( + "context" + "net" + "testing" + "time" + + grpc "google.golang.org/grpc" + "google.golang.org/grpc/test/bufconn" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/crypto/tmhash" + "github.com/tendermint/tendermint/libs/log" + tmrand "github.com/tendermint/tendermint/libs/rand" + tmgrpc "github.com/tendermint/tendermint/privval/grpc" + privvalproto "github.com/tendermint/tendermint/proto/tendermint/privval" + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" + "github.com/tendermint/tendermint/types" +) + +const chainID = "chain-id" + +func dialer(pv types.PrivValidator, logger log.Logger) (*grpc.Server, func(context.Context, string) (net.Conn, error)) { + listener := bufconn.Listen(1024 * 1024) + + server := grpc.NewServer() + + s := tmgrpc.NewSignerServer(chainID, pv, logger) + + privvalproto.RegisterPrivValidatorAPIServer(server, s) + + go func() { + if err := server.Serve(listener); err != nil { + panic(err) + } + }() + + return server, func(context.Context, string) (net.Conn, error) { + return listener.Dial() + } +} + +func TestSignerClient_GetPubKey(t *testing.T) { + + ctx := context.Background() + mockPV := types.NewMockPV() + logger := log.TestingLogger() + srv, dialer := dialer(mockPV, logger) + defer srv.Stop() + + conn, err := grpc.DialContext(ctx, "", grpc.WithInsecure(), grpc.WithContextDialer(dialer)) + if err != nil { + panic(err) + } + defer conn.Close() + + client, err := tmgrpc.NewSignerClient(conn, chainID, logger) + require.NoError(t, err) + + pk, err := client.GetPubKey() + require.NoError(t, err) + assert.Equal(t, mockPV.PrivKey.PubKey(), pk) +} + +func TestSignerClient_SignVote(t *testing.T) { + + ctx := context.Background() + mockPV := types.NewMockPV() + logger := log.TestingLogger() + srv, dialer := dialer(mockPV, logger) + defer srv.Stop() + + conn, err := grpc.DialContext(ctx, "", grpc.WithInsecure(), grpc.WithContextDialer(dialer)) + if err != nil { + panic(err) + } + defer conn.Close() + + client, err := tmgrpc.NewSignerClient(conn, chainID, logger) + require.NoError(t, err) + + ts := time.Now() + hash := tmrand.Bytes(tmhash.Size) + valAddr := tmrand.Bytes(crypto.AddressSize) + + want := &types.Vote{ + Type: tmproto.PrecommitType, + Height: 1, + Round: 2, + BlockID: types.BlockID{Hash: hash, PartSetHeader: types.PartSetHeader{Hash: hash, Total: 2}}, + Timestamp: ts, + ValidatorAddress: valAddr, + ValidatorIndex: 1, + } + + have := &types.Vote{ + Type: tmproto.PrecommitType, + Height: 1, + Round: 2, + BlockID: types.BlockID{Hash: hash, PartSetHeader: types.PartSetHeader{Hash: hash, Total: 2}}, + Timestamp: ts, + ValidatorAddress: valAddr, + ValidatorIndex: 1, + } + + pbHave := have.ToProto() + + err = client.SignVote(chainID, pbHave) + require.NoError(t, err) + + pbWant := want.ToProto() + + require.NoError(t, mockPV.SignVote(chainID, pbWant)) + + assert.Equal(t, pbWant.Signature, pbHave.Signature) +} + +func TestSignerClient_SignProposal(t *testing.T) { + + ctx := context.Background() + mockPV := types.NewMockPV() + logger := log.TestingLogger() + srv, dialer := dialer(mockPV, logger) + defer srv.Stop() + + conn, err := grpc.DialContext(ctx, "", grpc.WithInsecure(), grpc.WithContextDialer(dialer)) + if err != nil { + panic(err) + } + defer conn.Close() + + client, err := tmgrpc.NewSignerClient(conn, chainID, logger) + require.NoError(t, err) + + ts := time.Now() + hash := tmrand.Bytes(tmhash.Size) + + have := &types.Proposal{ + Type: tmproto.ProposalType, + Height: 1, + Round: 2, + POLRound: 2, + BlockID: types.BlockID{Hash: hash, PartSetHeader: types.PartSetHeader{Hash: hash, Total: 2}}, + Timestamp: ts, + } + want := &types.Proposal{ + Type: tmproto.ProposalType, + Height: 1, + Round: 2, + POLRound: 2, + BlockID: types.BlockID{Hash: hash, PartSetHeader: types.PartSetHeader{Hash: hash, Total: 2}}, + Timestamp: ts, + } + + pbHave := have.ToProto() + + err = client.SignProposal(chainID, pbHave) + require.NoError(t, err) + + pbWant := want.ToProto() + + require.NoError(t, mockPV.SignProposal(chainID, pbWant)) + + assert.Equal(t, pbWant.Signature, pbHave.Signature) +} diff --git a/privval/grpc/server.go b/privval/grpc/server.go new file mode 100644 index 000000000..763e50aaa --- /dev/null +++ b/privval/grpc/server.go @@ -0,0 +1,87 @@ +package grpc + +import ( + context "context" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/tendermint/tendermint/crypto" + cryptoenc "github.com/tendermint/tendermint/crypto/encoding" + "github.com/tendermint/tendermint/libs/log" + privvalproto "github.com/tendermint/tendermint/proto/tendermint/privval" + "github.com/tendermint/tendermint/types" +) + +// SignerServer implements PrivValidatorAPIServer 9generated via protobuf services) +// Handles remote validator connections that provide signing services +type SignerServer struct { + logger log.Logger + chainID string + privVal types.PrivValidator +} + +func NewSignerServer(chainID string, + privVal types.PrivValidator, log log.Logger) *SignerServer { + + return &SignerServer{ + logger: log, + chainID: chainID, + privVal: privVal, + } +} + +var _ privvalproto.PrivValidatorAPIServer = (*SignerServer)(nil) + +// PubKey receives a request for the pubkey +// returns the pubkey on success and error on failure +func (ss *SignerServer) GetPubKey(ctx context.Context, req *privvalproto.PubKeyRequest) ( + *privvalproto.PubKeyResponse, error) { + var pubKey crypto.PubKey + + pubKey, err := ss.privVal.GetPubKey() + if err != nil { + return nil, status.Errorf(codes.NotFound, "error getting pubkey: %v", err) + } + + pk, err := cryptoenc.PubKeyToProto(pubKey) + if err != nil { + return nil, status.Errorf(codes.Internal, "error transitioning pubkey to proto: %v", err) + } + + ss.logger.Info("SignerServer: GetPubKey Success") + + return &privvalproto.PubKeyResponse{PubKey: pk}, nil +} + +// SignVote receives a vote sign requests, attempts to sign it +// returns SignedVoteResponse on success and error on failure +func (ss *SignerServer) SignVote(ctx context.Context, req *privvalproto.SignVoteRequest) ( + *privvalproto.SignedVoteResponse, error) { + vote := req.Vote + + err := ss.privVal.SignVote(req.ChainId, vote) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "error signing vote: %v", err) + } + + ss.logger.Info("SignerServer: SignVote Success") + + return &privvalproto.SignedVoteResponse{Vote: *vote}, nil +} + +// SignProposal receives a proposal sign requests, attempts to sign it +// returns SignedProposalResponse on success and error on failure +func (ss *SignerServer) SignProposal(ctx context.Context, req *privvalproto.SignProposalRequest) ( + *privvalproto.SignedProposalResponse, error) { + proposal := req.Proposal + + err := ss.privVal.SignProposal(req.ChainId, proposal) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "error signing proposal: %v", err) + } + + ss.logger.Info("SignerServer: SignProposal Success") + + return &privvalproto.SignedProposalResponse{Proposal: *proposal}, nil +} diff --git a/privval/grpc/server_test.go b/privval/grpc/server_test.go new file mode 100644 index 000000000..ab728fdad --- /dev/null +++ b/privval/grpc/server_test.go @@ -0,0 +1,187 @@ +package grpc_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/crypto/tmhash" + "github.com/tendermint/tendermint/libs/log" + tmrand "github.com/tendermint/tendermint/libs/rand" + tmgrpc "github.com/tendermint/tendermint/privval/grpc" + privvalproto "github.com/tendermint/tendermint/proto/tendermint/privval" + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" + "github.com/tendermint/tendermint/types" +) + +const ChainID = "123" + +func TestGetPubKey(t *testing.T) { + + testCases := []struct { + name string + pv types.PrivValidator + err bool + }{ + {name: "valid", pv: types.NewMockPV(), err: false}, + {name: "error on pubkey", pv: types.NewErroringMockPV(), err: true}, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + s := tmgrpc.NewSignerServer(ChainID, tc.pv, log.TestingLogger()) + + req := &privvalproto.PubKeyRequest{ChainId: ChainID} + resp, err := s.GetPubKey(context.Background(), req) + if tc.err { + require.Error(t, err) + } else { + pk, err := tc.pv.GetPubKey() + require.NoError(t, err) + assert.Equal(t, resp.PubKey.GetEd25519(), pk.Bytes()) + } + }) + } + +} + +func TestSignVote(t *testing.T) { + + ts := time.Now() + hash := tmrand.Bytes(tmhash.Size) + valAddr := tmrand.Bytes(crypto.AddressSize) + + testCases := []struct { + name string + pv types.PrivValidator + have, want *types.Vote + err bool + }{ + {name: "valid", pv: types.NewMockPV(), have: &types.Vote{ + Type: tmproto.PrecommitType, + Height: 1, + Round: 2, + BlockID: types.BlockID{Hash: hash, PartSetHeader: types.PartSetHeader{Hash: hash, Total: 2}}, + Timestamp: ts, + ValidatorAddress: valAddr, + ValidatorIndex: 1, + }, want: &types.Vote{ + Type: tmproto.PrecommitType, + Height: 1, + Round: 2, + BlockID: types.BlockID{Hash: hash, PartSetHeader: types.PartSetHeader{Hash: hash, Total: 2}}, + Timestamp: ts, + ValidatorAddress: valAddr, + ValidatorIndex: 1, + }, + err: false}, + {name: "invalid vote", pv: types.NewErroringMockPV(), have: &types.Vote{ + Type: tmproto.PrecommitType, + Height: 1, + Round: 2, + BlockID: types.BlockID{Hash: hash, PartSetHeader: types.PartSetHeader{Hash: hash, Total: 2}}, + Timestamp: ts, + ValidatorAddress: valAddr, + ValidatorIndex: 1, + Signature: []byte("signed"), + }, want: &types.Vote{ + Type: tmproto.PrecommitType, + Height: 1, + Round: 2, + BlockID: types.BlockID{Hash: hash, PartSetHeader: types.PartSetHeader{Hash: hash, Total: 2}}, + Timestamp: ts, + ValidatorAddress: valAddr, + ValidatorIndex: 1, + Signature: []byte("signed"), + }, + err: true}, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + s := tmgrpc.NewSignerServer(ChainID, tc.pv, log.TestingLogger()) + + req := &privvalproto.SignVoteRequest{ChainId: ChainID, Vote: tc.have.ToProto()} + resp, err := s.SignVote(context.Background(), req) + if tc.err { + require.Error(t, err) + } else { + pbVote := tc.want.ToProto() + + require.NoError(t, tc.pv.SignVote(ChainID, pbVote)) + assert.Equal(t, pbVote.Signature, resp.Vote.Signature) + } + }) + } +} + +func TestSignProposal(t *testing.T) { + + ts := time.Now() + hash := tmrand.Bytes(tmhash.Size) + + testCases := []struct { + name string + pv types.PrivValidator + have, want *types.Proposal + err bool + }{ + {name: "valid", pv: types.NewMockPV(), have: &types.Proposal{ + Type: tmproto.ProposalType, + Height: 1, + Round: 2, + POLRound: 2, + BlockID: types.BlockID{Hash: hash, PartSetHeader: types.PartSetHeader{Hash: hash, Total: 2}}, + Timestamp: ts, + }, want: &types.Proposal{ + Type: tmproto.ProposalType, + Height: 1, + Round: 2, + POLRound: 2, + BlockID: types.BlockID{Hash: hash, PartSetHeader: types.PartSetHeader{Hash: hash, Total: 2}}, + Timestamp: ts, + }, + err: false}, + {name: "invalid proposal", pv: types.NewErroringMockPV(), have: &types.Proposal{ + Type: tmproto.ProposalType, + Height: 1, + Round: 2, + POLRound: 2, + BlockID: types.BlockID{Hash: hash, PartSetHeader: types.PartSetHeader{Hash: hash, Total: 2}}, + Timestamp: ts, + Signature: []byte("signed"), + }, want: &types.Proposal{ + Type: tmproto.ProposalType, + Height: 1, + Round: 2, + POLRound: 2, + BlockID: types.BlockID{Hash: hash, PartSetHeader: types.PartSetHeader{Hash: hash, Total: 2}}, + Timestamp: ts, + Signature: []byte("signed"), + }, + err: true}, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + s := tmgrpc.NewSignerServer(ChainID, tc.pv, log.TestingLogger()) + + req := &privvalproto.SignProposalRequest{ChainId: ChainID, Proposal: tc.have.ToProto()} + resp, err := s.SignProposal(context.Background(), req) + if tc.err { + require.Error(t, err) + } else { + pbProposal := tc.want.ToProto() + require.NoError(t, tc.pv.SignProposal(ChainID, pbProposal)) + assert.Equal(t, pbProposal.Signature, resp.Proposal.Signature) + } + }) + } +} diff --git a/privval/grpc/util.go b/privval/grpc/util.go new file mode 100644 index 000000000..66b5397b6 --- /dev/null +++ b/privval/grpc/util.go @@ -0,0 +1,82 @@ +package grpc + +import ( + "crypto/tls" + "crypto/x509" + "io/ioutil" + "os" + "time" + + grpc_retry "github.com/grpc-ecosystem/go-grpc-middleware/retry" + "github.com/tendermint/tendermint/libs/log" + grpc "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/keepalive" +) + +// DefaultDialOptions constructs a list of grpc dial options +func DefaultDialOptions( + extraOpts ...grpc.DialOption, +) []grpc.DialOption { + const ( + retries = 50 // 50 * 100ms = 5s total + timeout = 1 * time.Second + maxCallRecvMsgSize = 1 << 20 // Default 5Mb + ) + + var kacp = keepalive.ClientParameters{ + Time: 10 * time.Second, // send pings every 10 seconds if there is no activity + Timeout: 2 * time.Second, // wait 2 seconds for ping ack before considering the connection dead + } + + opts := []grpc_retry.CallOption{ + grpc_retry.WithBackoff(grpc_retry.BackoffExponential(timeout)), + } + + dialOpts := []grpc.DialOption{ + grpc.WithKeepaliveParams(kacp), + grpc.WithDefaultCallOptions( + grpc.MaxCallRecvMsgSize(maxCallRecvMsgSize), + grpc_retry.WithMax(retries), + ), + grpc.WithUnaryInterceptor( + grpc_retry.UnaryClientInterceptor(opts...), + ), + } + + dialOpts = append(dialOpts, extraOpts...) + + return dialOpts +} + +func GenerateTLS(certPath, keyPath, ca string, log log.Logger) grpc.DialOption { + certificate, err := tls.LoadX509KeyPair( + certPath, + keyPath, + ) + if err != nil { + log.Error("error", err) + os.Exit(1) + } + + certPool := x509.NewCertPool() + bs, err := ioutil.ReadFile(ca) + if err != nil { + log.Error("failed to read ca cert:", "error", err) + os.Exit(1) + } + + ok := certPool.AppendCertsFromPEM(bs) + if !ok { + log.Error("failed to append certs") + os.Exit(1) + } + + transportCreds := credentials.NewTLS(&tls.Config{ + Certificates: []tls.Certificate{certificate}, + RootCAs: certPool, + MinVersion: tls.VersionTLS13, + }) + + return grpc.WithTransportCredentials(transportCreds) +} diff --git a/proto/tendermint/privval/service.pb.go b/proto/tendermint/privval/service.pb.go new file mode 100644 index 000000000..a9ecd76ed --- /dev/null +++ b/proto/tendermint/privval/service.pb.go @@ -0,0 +1,199 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: tendermint/privval/service.proto + +package privval + +import ( + context "context" + fmt "fmt" + proto "github.com/gogo/protobuf/proto" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" + math "math" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package + +func init() { proto.RegisterFile("tendermint/privval/service.proto", fileDescriptor_7afe74f9f46d3dc9) } + +var fileDescriptor_7afe74f9f46d3dc9 = []byte{ + // 251 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0x28, 0x49, 0xcd, 0x4b, + 0x49, 0x2d, 0xca, 0xcd, 0xcc, 0x2b, 0xd1, 0x2f, 0x28, 0xca, 0x2c, 0x2b, 0x4b, 0xcc, 0xd1, 0x2f, + 0x4e, 0x2d, 0x2a, 0xcb, 0x4c, 0x4e, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x12, 0x42, 0xa8, + 0xd0, 0x83, 0xaa, 0x90, 0x92, 0xc3, 0xa2, 0xab, 0xa4, 0xb2, 0x20, 0xb5, 0x18, 0xa2, 0xc7, 0x68, + 0x09, 0x13, 0x97, 0x40, 0x40, 0x51, 0x66, 0x59, 0x58, 0x62, 0x4e, 0x66, 0x4a, 0x62, 0x49, 0x7e, + 0x91, 0x63, 0x80, 0xa7, 0x50, 0x10, 0x17, 0xa7, 0x7b, 0x6a, 0x49, 0x40, 0x69, 0x92, 0x77, 0x6a, + 0xa5, 0x90, 0xa2, 0x1e, 0xa6, 0xb1, 0x7a, 0x10, 0xb9, 0xa0, 0xd4, 0xc2, 0xd2, 0xd4, 0xe2, 0x12, + 0x29, 0x25, 0x7c, 0x4a, 0x8a, 0x0b, 0xf2, 0xf3, 0x8a, 0x53, 0x85, 0xc2, 0xb9, 0x38, 0x82, 0x33, + 0xd3, 0xf3, 0xc2, 0xf2, 0x4b, 0x52, 0x85, 0x94, 0xb1, 0xa9, 0x87, 0xc9, 0xc2, 0x0c, 0x55, 0xc3, + 0xa5, 0x28, 0x35, 0x05, 0xa2, 0x0c, 0x6a, 0x70, 0x32, 0x17, 0x0f, 0x48, 0x34, 0xa0, 0x28, 0xbf, + 0x20, 0xbf, 0x38, 0x31, 0x47, 0x48, 0x1d, 0x97, 0x3e, 0x98, 0x0a, 0x98, 0x05, 0x5a, 0xb8, 0x2d, + 0x40, 0x28, 0x85, 0x58, 0xe2, 0x14, 0x7c, 0xe2, 0x91, 0x1c, 0xe3, 0x85, 0x47, 0x72, 0x8c, 0x0f, + 0x1e, 0xc9, 0x31, 0x4e, 0x78, 0x2c, 0xc7, 0x70, 0xe1, 0xb1, 0x1c, 0xc3, 0x8d, 0xc7, 0x72, 0x0c, + 0x51, 0x96, 0xe9, 0x99, 0x25, 0x19, 0xa5, 0x49, 0x7a, 0xc9, 0xf9, 0xb9, 0xfa, 0x48, 0x61, 0x8d, + 0x12, 0xec, 0xf9, 0x25, 0xf9, 0xfa, 0x98, 0xf1, 0x90, 0xc4, 0x06, 0x96, 0x31, 0x06, 0x04, 0x00, + 0x00, 0xff, 0xff, 0x42, 0x60, 0x24, 0x48, 0xda, 0x01, 0x00, 0x00, +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// PrivValidatorAPIClient is the client API for PrivValidatorAPI service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type PrivValidatorAPIClient interface { + GetPubKey(ctx context.Context, in *PubKeyRequest, opts ...grpc.CallOption) (*PubKeyResponse, error) + SignVote(ctx context.Context, in *SignVoteRequest, opts ...grpc.CallOption) (*SignedVoteResponse, error) + SignProposal(ctx context.Context, in *SignProposalRequest, opts ...grpc.CallOption) (*SignedProposalResponse, error) +} + +type privValidatorAPIClient struct { + cc *grpc.ClientConn +} + +func NewPrivValidatorAPIClient(cc *grpc.ClientConn) PrivValidatorAPIClient { + return &privValidatorAPIClient{cc} +} + +func (c *privValidatorAPIClient) GetPubKey(ctx context.Context, in *PubKeyRequest, opts ...grpc.CallOption) (*PubKeyResponse, error) { + out := new(PubKeyResponse) + err := c.cc.Invoke(ctx, "/tendermint.privval.PrivValidatorAPI/GetPubKey", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *privValidatorAPIClient) SignVote(ctx context.Context, in *SignVoteRequest, opts ...grpc.CallOption) (*SignedVoteResponse, error) { + out := new(SignedVoteResponse) + err := c.cc.Invoke(ctx, "/tendermint.privval.PrivValidatorAPI/SignVote", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *privValidatorAPIClient) SignProposal(ctx context.Context, in *SignProposalRequest, opts ...grpc.CallOption) (*SignedProposalResponse, error) { + out := new(SignedProposalResponse) + err := c.cc.Invoke(ctx, "/tendermint.privval.PrivValidatorAPI/SignProposal", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// PrivValidatorAPIServer is the server API for PrivValidatorAPI service. +type PrivValidatorAPIServer interface { + GetPubKey(context.Context, *PubKeyRequest) (*PubKeyResponse, error) + SignVote(context.Context, *SignVoteRequest) (*SignedVoteResponse, error) + SignProposal(context.Context, *SignProposalRequest) (*SignedProposalResponse, error) +} + +// UnimplementedPrivValidatorAPIServer can be embedded to have forward compatible implementations. +type UnimplementedPrivValidatorAPIServer struct { +} + +func (*UnimplementedPrivValidatorAPIServer) GetPubKey(ctx context.Context, req *PubKeyRequest) (*PubKeyResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetPubKey not implemented") +} +func (*UnimplementedPrivValidatorAPIServer) SignVote(ctx context.Context, req *SignVoteRequest) (*SignedVoteResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method SignVote not implemented") +} +func (*UnimplementedPrivValidatorAPIServer) SignProposal(ctx context.Context, req *SignProposalRequest) (*SignedProposalResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method SignProposal not implemented") +} + +func RegisterPrivValidatorAPIServer(s *grpc.Server, srv PrivValidatorAPIServer) { + s.RegisterService(&_PrivValidatorAPI_serviceDesc, srv) +} + +func _PrivValidatorAPI_GetPubKey_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(PubKeyRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(PrivValidatorAPIServer).GetPubKey(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/tendermint.privval.PrivValidatorAPI/GetPubKey", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(PrivValidatorAPIServer).GetPubKey(ctx, req.(*PubKeyRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _PrivValidatorAPI_SignVote_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SignVoteRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(PrivValidatorAPIServer).SignVote(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/tendermint.privval.PrivValidatorAPI/SignVote", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(PrivValidatorAPIServer).SignVote(ctx, req.(*SignVoteRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _PrivValidatorAPI_SignProposal_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SignProposalRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(PrivValidatorAPIServer).SignProposal(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/tendermint.privval.PrivValidatorAPI/SignProposal", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(PrivValidatorAPIServer).SignProposal(ctx, req.(*SignProposalRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _PrivValidatorAPI_serviceDesc = grpc.ServiceDesc{ + ServiceName: "tendermint.privval.PrivValidatorAPI", + HandlerType: (*PrivValidatorAPIServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetPubKey", + Handler: _PrivValidatorAPI_GetPubKey_Handler, + }, + { + MethodName: "SignVote", + Handler: _PrivValidatorAPI_SignVote_Handler, + }, + { + MethodName: "SignProposal", + Handler: _PrivValidatorAPI_SignProposal_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "tendermint/privval/service.proto", +} diff --git a/proto/tendermint/privval/service.proto b/proto/tendermint/privval/service.proto new file mode 100644 index 000000000..63e9afca7 --- /dev/null +++ b/proto/tendermint/privval/service.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; +package tendermint.privval; +option go_package = "github.com/tendermint/tendermint/proto/tendermint/privval"; + +import "tendermint/privval/types.proto"; + +//---------------------------------------- +// Service Definition + +service PrivValidatorAPI { + rpc GetPubKey(PubKeyRequest) returns (PubKeyResponse); + rpc SignVote(SignVoteRequest) returns (SignedVoteResponse); + rpc SignProposal(SignProposalRequest) returns (SignedProposalResponse); +} diff --git a/test/e2e/app/main.go b/test/e2e/app/main.go index 34d99b909..e641f0e8c 100644 --- a/test/e2e/app/main.go +++ b/test/e2e/app/main.go @@ -3,12 +3,14 @@ package main import ( "errors" "fmt" + "net" "os" "path/filepath" "strconv" "time" "github.com/spf13/viper" + "google.golang.org/grpc" "github.com/tendermint/tendermint/abci/server" "github.com/tendermint/tendermint/config" @@ -19,6 +21,8 @@ import ( "github.com/tendermint/tendermint/node" "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/privval" + grpcprivval "github.com/tendermint/tendermint/privval/grpc" + privvalproto "github.com/tendermint/tendermint/proto/tendermint/privval" "github.com/tendermint/tendermint/proxy" mcs "github.com/tendermint/tendermint/test/maverick/consensus" maverick "github.com/tendermint/tendermint/test/maverick/node" @@ -182,6 +186,24 @@ func startSigner(cfg *Config) error { dialFn = privval.DialTCPFn(address, 3*time.Second, ed25519.GenPrivKey()) case "unix": dialFn = privval.DialUnixFn(address) + case "grpc": + lis, err := net.Listen("tcp", address) + if err != nil { + return err + } + ss := grpcprivval.NewSignerServer(cfg.ChainID, filePV, logger) + + s := grpc.NewServer() + + privvalproto.RegisterPrivValidatorAPIServer(s, ss) + + go func() { // no need to clean up since we remove docker containers + if err := s.Serve(lis); err != nil { + panic(err) + } + }() + + return nil default: return fmt.Errorf("invalid privval protocol %q", protocol) } @@ -193,6 +215,7 @@ func startSigner(cfg *Config) error { if err != nil { return err } + logger.Info(fmt.Sprintf("Remote signer connecting to %v", cfg.PrivValServer)) return nil } diff --git a/test/e2e/generator/generate.go b/test/e2e/generator/generate.go index de6aea731..42cb3d1a5 100644 --- a/test/e2e/generator/generate.go +++ b/test/e2e/generator/generate.go @@ -30,7 +30,7 @@ var ( nodeDatabases = uniformChoice{"goleveldb", "cleveldb", "rocksdb", "boltdb", "badgerdb"} // FIXME: grpc disabled due to https://github.com/tendermint/tendermint/issues/5439 nodeABCIProtocols = uniformChoice{"unix", "tcp", "builtin"} // "grpc" - nodePrivvalProtocols = uniformChoice{"file", "unix", "tcp"} + nodePrivvalProtocols = uniformChoice{"file", "unix", "tcp", "grpc"} // FIXME: v2 disabled due to flake nodeFastSyncs = uniformChoice{"", "v0"} // "v2" nodeStateSyncs = uniformChoice{false, true} diff --git a/test/e2e/networks/ci.toml b/test/e2e/networks/ci.toml index d0dc9db6d..e27092040 100644 --- a/test/e2e/networks/ci.toml +++ b/test/e2e/networks/ci.toml @@ -51,7 +51,7 @@ seeds = ["seed01"] database = "badgerdb" # FIXME: should be grpc, disabled due to https://github.com/tendermint/tendermint/issues/5439 #abci_protocol = "grpc" -privval_protocol = "unix" +privval_protocol = "grpc" persist_interval = 3 retain_blocks = 3 perturb = ["kill"] diff --git a/test/e2e/pkg/manifest.go b/test/e2e/pkg/manifest.go index e94fd0790..492e362d8 100644 --- a/test/e2e/pkg/manifest.go +++ b/test/e2e/pkg/manifest.go @@ -78,8 +78,9 @@ type ManifestNode struct { ABCIProtocol string `toml:"abci_protocol"` // PrivvalProtocol specifies the protocol used to sign consensus messages: - // "file", "unix", or "tcp". Defaults to "file". For unix and tcp, the ABCI + // "file", "unix", "tcp", or "grpc". Defaults to "file". For tcp and unix, the ABCI // application will launch a remote signer client in a separate goroutine. + // For grpc the ABCI application will launch a remote signer server. // Only nodes with mode=validator will actually make use of this. PrivvalProtocol string `toml:"privval_protocol"` diff --git a/test/e2e/pkg/testnet.go b/test/e2e/pkg/testnet.go index eaae577b1..da0890b00 100644 --- a/test/e2e/pkg/testnet.go +++ b/test/e2e/pkg/testnet.go @@ -322,7 +322,7 @@ func (n Node) Validate(testnet Testnet) error { return fmt.Errorf("invalid ABCI protocol setting %q", n.ABCIProtocol) } switch n.PrivvalProtocol { - case ProtocolFile, ProtocolUNIX, ProtocolTCP: + case ProtocolFile, ProtocolTCP, ProtocolGRPC, ProtocolUNIX: default: return fmt.Errorf("invalid privval protocol setting %q", n.PrivvalProtocol) } diff --git a/test/e2e/runner/setup.go b/test/e2e/runner/setup.go index c0dd893d7..ceb7f29d9 100644 --- a/test/e2e/runner/setup.go +++ b/test/e2e/runner/setup.go @@ -32,6 +32,7 @@ const ( AppAddressUNIX = "unix:///var/run/app.sock" PrivvalAddressTCP = "tcp://0.0.0.0:27559" + PrivvalAddressGRPC = "grpc://0.0.0.0:27559" PrivvalAddressUNIX = "unix:///var/run/privval.sock" PrivvalKeyFile = "config/priv_validator_key.json" PrivvalStateFile = "data/priv_validator_state.json" @@ -265,6 +266,8 @@ func MakeConfig(node *e2e.Node) (*config.Config, error) { cfg.PrivValidatorListenAddr = PrivvalAddressUNIX case e2e.ProtocolTCP: cfg.PrivValidatorListenAddr = PrivvalAddressTCP + case e2e.ProtocolGRPC: + cfg.PrivValidatorListenAddr = PrivvalAddressGRPC default: return nil, fmt.Errorf("invalid privval protocol setting %q", node.PrivvalProtocol) } @@ -351,6 +354,10 @@ func MakeAppConfig(node *e2e.Node) ([]byte, error) { cfg["privval_server"] = PrivvalAddressUNIX cfg["privval_key"] = PrivvalKeyFile cfg["privval_state"] = PrivvalStateFile + case e2e.ProtocolGRPC: + cfg["privval_server"] = PrivvalAddressGRPC + cfg["privval_key"] = PrivvalKeyFile + cfg["privval_state"] = PrivvalStateFile default: return nil, fmt.Errorf("unexpected privval protocol setting %q", node.PrivvalProtocol) } diff --git a/types/priv_validator.go b/types/priv_validator.go index 49211773a..3ce02511a 100644 --- a/types/priv_validator.go +++ b/types/priv_validator.go @@ -128,6 +128,11 @@ type ErroringMockPV struct { var ErroringMockPVErr = errors.New("erroringMockPV always returns an error") +// Implements PrivValidator. +func (pv *ErroringMockPV) GetPubKey() (crypto.PubKey, error) { + return nil, ErroringMockPVErr +} + // Implements PrivValidator. func (pv *ErroringMockPV) SignVote(chainID string, vote *tmproto.Vote) error { return ErroringMockPVErr