From 36a1acff520b9f30925de9ad8f27189653709ea4 Mon Sep 17 00:00:00 2001 From: William Banfield <4561443+williambanfield@users.noreply.github.com> Date: Wed, 13 Oct 2021 16:52:25 -0400 Subject: [PATCH] internal/proxy: add initial set of abci metrics (#7115) This PR adds an initial set of metrics for use ABCI. The initial metrics enable the calculation of timing histograms and call counts for each of the ABCI methods. The metrics are also labeled as either 'sync' or 'async' to determine if the method call was performed using ABCI's `*Async` methods. An example of these metrics is included here for reference: ``` tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.0001"} 0 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.0004"} 5 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.002"} 12 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.009"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.02"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.1"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.65"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="2"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="6"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="25"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="+Inf"} 13 tendermint_abci_connection_method_timing_sum{chain_id="ci",method="commit",type="sync"} 0.007802058000000001 tendermint_abci_connection_method_timing_count{chain_id="ci",method="commit",type="sync"} 13 ``` These metrics can easily be graphed using prometheus's `histogram_quantile(...)` method to pick out a particular quantile to graph or examine. I chose buckets that were somewhat of an estimate of expected range of times for ABCI operations. They start at .0001 seconds and range to 25 seconds. The hope is that this range captures enough possible times to be useful for us and operators. --- internal/blocksync/v0/reactor_test.go | 2 +- internal/consensus/metrics.go | 9 +++++ internal/consensus/replay_file.go | 2 +- internal/consensus/replay_stubs.go | 2 +- internal/consensus/replay_test.go | 12 +++---- internal/consensus/wal_generator.go | 2 +- internal/proxy/app_conn.go | 43 +++++++++++++++++++++--- internal/proxy/metrics.go | 47 +++++++++++++++++++++++++++ internal/proxy/multi_app_conn.go | 16 +++++---- internal/proxy/multi_app_conn_test.go | 4 +-- internal/state/execution_test.go | 10 +++--- internal/state/helpers_test.go | 2 +- node/node.go | 9 +++-- node/node_test.go | 6 ++-- node/setup.go | 5 +-- 15 files changed, 134 insertions(+), 37 deletions(-) create mode 100644 internal/proxy/metrics.go diff --git a/internal/blocksync/v0/reactor_test.go b/internal/blocksync/v0/reactor_test.go index e947581aa..312b1cb39 100644 --- a/internal/blocksync/v0/reactor_test.go +++ b/internal/blocksync/v0/reactor_test.go @@ -98,7 +98,7 @@ func (rts *reactorTestSuite) addNode(t *testing.T, t.Helper() rts.nodes = append(rts.nodes, nodeID) - rts.app[nodeID] = proxy.NewAppConns(abciclient.NewLocalCreator(&abci.BaseApplication{})) + rts.app[nodeID] = proxy.NewAppConns(abciclient.NewLocalCreator(&abci.BaseApplication{}), proxy.NopMetrics()) require.NoError(t, rts.app[nodeID].Start()) blockDB := dbm.NewMemDB() diff --git a/internal/consensus/metrics.go b/internal/consensus/metrics.go index bceac4942..a75f1505c 100644 --- a/internal/consensus/metrics.go +++ b/internal/consensus/metrics.go @@ -61,6 +61,9 @@ type Metrics struct { // Number of blockparts transmitted by peer. BlockParts metrics.Counter + + // Histogram of time taken per step annotated with reason that the step proceeded. + StepTime metrics.Histogram } // PrometheusMetrics returns Metrics build using Prometheus client library. @@ -187,6 +190,12 @@ func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics { Name: "block_parts", Help: "Number of blockparts transmitted by peer.", }, append(labels, "peer_id")).With(labelsAndValues...), + StepTime: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: MetricsSubsystem, + Name: "step_time", + Help: "Time spent per step.", + }, append(labels, "step", "reason")).With(labelsAndValues...), } } diff --git a/internal/consensus/replay_file.go b/internal/consensus/replay_file.go index d009533ba..f60dff531 100644 --- a/internal/consensus/replay_file.go +++ b/internal/consensus/replay_file.go @@ -312,7 +312,7 @@ func newConsensusStateForReplay(cfg config.BaseConfig, csConfig *config.Consensu // Create proxyAppConn connection (consensus, mempool, query) clientCreator, _ := proxy.DefaultClientCreator(cfg.ProxyApp, cfg.ABCI, cfg.DBDir()) - proxyApp := proxy.NewAppConns(clientCreator) + proxyApp := proxy.NewAppConns(clientCreator, proxy.NopMetrics()) err = proxyApp.Start() if err != nil { tmos.Exit(fmt.Sprintf("Error starting proxy app conns: %v", err)) diff --git a/internal/consensus/replay_stubs.go b/internal/consensus/replay_stubs.go index bc8c11cd9..1235baccb 100644 --- a/internal/consensus/replay_stubs.go +++ b/internal/consensus/replay_stubs.go @@ -64,7 +64,7 @@ func newMockProxyApp(appHash []byte, abciResponses *tmstate.ABCIResponses) proxy if err != nil { panic(err) } - return proxy.NewAppConnConsensus(cli) + return proxy.NewAppConnConsensus(cli, proxy.NopMetrics()) } type mockProxyApp struct { diff --git a/internal/consensus/replay_test.go b/internal/consensus/replay_test.go index 97017985d..0d0ae36e8 100644 --- a/internal/consensus/replay_test.go +++ b/internal/consensus/replay_test.go @@ -745,7 +745,7 @@ func testHandshakeReplay(t *testing.T, sim *simulatorTestSuite, nBlocks int, mod if nBlocks > 0 { // run nBlocks against a new client to build up the app state. // use a throwaway tendermint state - proxyApp := proxy.NewAppConns(clientCreator2) + proxyApp := proxy.NewAppConns(clientCreator2, proxy.NopMetrics()) stateDB1 := dbm.NewMemDB() stateStore := sm.NewStore(stateDB1) err := stateStore.Save(genesisState) @@ -765,7 +765,7 @@ func testHandshakeReplay(t *testing.T, sim *simulatorTestSuite, nBlocks int, mod // now start the app using the handshake - it should sync genDoc, _ := sm.MakeGenesisDocFromFile(cfg.GenesisFile()) handshaker := NewHandshaker(stateStore, state, store, genDoc) - proxyApp := proxy.NewAppConns(clientCreator2) + proxyApp := proxy.NewAppConns(clientCreator2, proxy.NopMetrics()) if err := proxyApp.Start(); err != nil { t.Fatalf("Error starting proxy app connections: %v", err) } @@ -893,7 +893,7 @@ func buildTMStateFromChain( defer kvstoreApp.Close() clientCreator := abciclient.NewLocalCreator(kvstoreApp) - proxyApp := proxy.NewAppConns(clientCreator) + proxyApp := proxy.NewAppConns(clientCreator, proxy.NopMetrics()) if err := proxyApp.Start(); err != nil { panic(err) } @@ -960,7 +960,7 @@ func TestHandshakePanicsIfAppReturnsWrongAppHash(t *testing.T) { { app := &badApp{numBlocks: 3, allHashesAreWrong: true} clientCreator := abciclient.NewLocalCreator(app) - proxyApp := proxy.NewAppConns(clientCreator) + proxyApp := proxy.NewAppConns(clientCreator, proxy.NopMetrics()) err := proxyApp.Start() require.NoError(t, err) t.Cleanup(func() { @@ -984,7 +984,7 @@ func TestHandshakePanicsIfAppReturnsWrongAppHash(t *testing.T) { { app := &badApp{numBlocks: 3, onlyLastHashIsWrong: true} clientCreator := abciclient.NewLocalCreator(app) - proxyApp := proxy.NewAppConns(clientCreator) + proxyApp := proxy.NewAppConns(clientCreator, proxy.NopMetrics()) err := proxyApp.Start() require.NoError(t, err) t.Cleanup(func() { @@ -1243,7 +1243,7 @@ func TestHandshakeUpdatesValidators(t *testing.T) { // now start the app using the handshake - it should sync genDoc, _ := sm.MakeGenesisDocFromFile(cfg.GenesisFile()) handshaker := NewHandshaker(stateStore, state, store, genDoc) - proxyApp := proxy.NewAppConns(clientCreator) + proxyApp := proxy.NewAppConns(clientCreator, proxy.NopMetrics()) if err := proxyApp.Start(); err != nil { t.Fatalf("Error starting proxy app connections: %v", err) } diff --git a/internal/consensus/wal_generator.go b/internal/consensus/wal_generator.go index 7e31188fa..4b4375498 100644 --- a/internal/consensus/wal_generator.go +++ b/internal/consensus/wal_generator.go @@ -65,7 +65,7 @@ func WALGenerateNBlocks(t *testing.T, wr io.Writer, numBlocks int) (err error) { blockStore := store.NewBlockStore(blockStoreDB) - proxyApp := proxy.NewAppConns(abciclient.NewLocalCreator(app)) + proxyApp := proxy.NewAppConns(abciclient.NewLocalCreator(app), proxy.NopMetrics()) proxyApp.SetLogger(logger.With("module", "proxy")) if err := proxyApp.Start(); err != nil { return fmt.Errorf("failed to start proxy app connections: %w", err) diff --git a/internal/proxy/app_conn.go b/internal/proxy/app_conn.go index ca2c7c109..803875646 100644 --- a/internal/proxy/app_conn.go +++ b/internal/proxy/app_conn.go @@ -2,7 +2,9 @@ package proxy import ( "context" + "time" + "github.com/go-kit/kit/metrics" abciclient "github.com/tendermint/tendermint/abci/client" "github.com/tendermint/tendermint/abci/types" ) @@ -56,11 +58,13 @@ type AppConnSnapshot interface { // Implements AppConnConsensus (subset of abciclient.Client) type appConnConsensus struct { + metrics *Metrics appConn abciclient.Client } -func NewAppConnConsensus(appConn abciclient.Client) AppConnConsensus { +func NewAppConnConsensus(appConn abciclient.Client, metrics *Metrics) AppConnConsensus { return &appConnConsensus{ + metrics: metrics, appConn: appConn, } } @@ -77,6 +81,7 @@ func (app *appConnConsensus) InitChainSync( ctx context.Context, req types.RequestInitChain, ) (*types.ResponseInitChain, error) { + defer addTimeSample(app.metrics.MethodTiming.With("method", "init_chain", "type", "sync"))() return app.appConn.InitChainSync(ctx, req) } @@ -84,6 +89,7 @@ func (app *appConnConsensus) BeginBlockSync( ctx context.Context, req types.RequestBeginBlock, ) (*types.ResponseBeginBlock, error) { + defer addTimeSample(app.metrics.MethodTiming.With("method", "begin_block", "type", "sync"))() return app.appConn.BeginBlockSync(ctx, req) } @@ -91,6 +97,7 @@ func (app *appConnConsensus) DeliverTxAsync( ctx context.Context, req types.RequestDeliverTx, ) (*abciclient.ReqRes, error) { + defer addTimeSample(app.metrics.MethodTiming.With("method", "deliver_tx", "type", "async"))() return app.appConn.DeliverTxAsync(ctx, req) } @@ -98,10 +105,12 @@ func (app *appConnConsensus) EndBlockSync( ctx context.Context, req types.RequestEndBlock, ) (*types.ResponseEndBlock, error) { + defer addTimeSample(app.metrics.MethodTiming.With("method", "deliver_tx", "type", "sync"))() return app.appConn.EndBlockSync(ctx, req) } func (app *appConnConsensus) CommitSync(ctx context.Context) (*types.ResponseCommit, error) { + defer addTimeSample(app.metrics.MethodTiming.With("method", "commit", "type", "sync"))() return app.appConn.CommitSync(ctx) } @@ -109,11 +118,13 @@ func (app *appConnConsensus) CommitSync(ctx context.Context) (*types.ResponseCom // Implements AppConnMempool (subset of abciclient.Client) type appConnMempool struct { + metrics *Metrics appConn abciclient.Client } -func NewAppConnMempool(appConn abciclient.Client) AppConnMempool { +func NewAppConnMempool(appConn abciclient.Client, metrics *Metrics) AppConnMempool { return &appConnMempool{ + metrics: metrics, appConn: appConn, } } @@ -127,18 +138,22 @@ func (app *appConnMempool) Error() error { } func (app *appConnMempool) FlushAsync(ctx context.Context) (*abciclient.ReqRes, error) { + defer addTimeSample(app.metrics.MethodTiming.With("method", "flush", "type", "async"))() return app.appConn.FlushAsync(ctx) } func (app *appConnMempool) FlushSync(ctx context.Context) error { + defer addTimeSample(app.metrics.MethodTiming.With("method", "flush", "type", "sync"))() return app.appConn.FlushSync(ctx) } func (app *appConnMempool) CheckTxAsync(ctx context.Context, req types.RequestCheckTx) (*abciclient.ReqRes, error) { + defer addTimeSample(app.metrics.MethodTiming.With("method", "check_tx", "type", "async"))() return app.appConn.CheckTxAsync(ctx, req) } func (app *appConnMempool) CheckTxSync(ctx context.Context, req types.RequestCheckTx) (*types.ResponseCheckTx, error) { + defer addTimeSample(app.metrics.MethodTiming.With("method", "check_tx", "type", "sync"))() return app.appConn.CheckTxSync(ctx, req) } @@ -146,11 +161,13 @@ func (app *appConnMempool) CheckTxSync(ctx context.Context, req types.RequestChe // Implements AppConnQuery (subset of abciclient.Client) type appConnQuery struct { + metrics *Metrics appConn abciclient.Client } -func NewAppConnQuery(appConn abciclient.Client) AppConnQuery { +func NewAppConnQuery(appConn abciclient.Client, metrics *Metrics) AppConnQuery { return &appConnQuery{ + metrics: metrics, appConn: appConn, } } @@ -160,14 +177,17 @@ func (app *appConnQuery) Error() error { } func (app *appConnQuery) EchoSync(ctx context.Context, msg string) (*types.ResponseEcho, error) { + defer addTimeSample(app.metrics.MethodTiming.With("method", "echo", "type", "sync"))() return app.appConn.EchoSync(ctx, msg) } func (app *appConnQuery) InfoSync(ctx context.Context, req types.RequestInfo) (*types.ResponseInfo, error) { + defer addTimeSample(app.metrics.MethodTiming.With("method", "info", "type", "sync"))() return app.appConn.InfoSync(ctx, req) } func (app *appConnQuery) QuerySync(ctx context.Context, reqQuery types.RequestQuery) (*types.ResponseQuery, error) { + defer addTimeSample(app.metrics.MethodTiming.With("method", "query", "type", "sync"))() return app.appConn.QuerySync(ctx, reqQuery) } @@ -175,11 +195,13 @@ func (app *appConnQuery) QuerySync(ctx context.Context, reqQuery types.RequestQu // Implements AppConnSnapshot (subset of abciclient.Client) type appConnSnapshot struct { + metrics *Metrics appConn abciclient.Client } -func NewAppConnSnapshot(appConn abciclient.Client) AppConnSnapshot { +func NewAppConnSnapshot(appConn abciclient.Client, metrics *Metrics) AppConnSnapshot { return &appConnSnapshot{ + metrics: metrics, appConn: appConn, } } @@ -192,6 +214,7 @@ func (app *appConnSnapshot) ListSnapshotsSync( ctx context.Context, req types.RequestListSnapshots, ) (*types.ResponseListSnapshots, error) { + defer addTimeSample(app.metrics.MethodTiming.With("method", "list_snapshots", "type", "sync"))() return app.appConn.ListSnapshotsSync(ctx, req) } @@ -199,17 +222,29 @@ func (app *appConnSnapshot) OfferSnapshotSync( ctx context.Context, req types.RequestOfferSnapshot, ) (*types.ResponseOfferSnapshot, error) { + defer addTimeSample(app.metrics.MethodTiming.With("method", "offer_snapshot", "type", "sync"))() return app.appConn.OfferSnapshotSync(ctx, req) } func (app *appConnSnapshot) LoadSnapshotChunkSync( ctx context.Context, req types.RequestLoadSnapshotChunk) (*types.ResponseLoadSnapshotChunk, error) { + defer addTimeSample(app.metrics.MethodTiming.With("method", "load_snapshot_chunk", "type", "sync"))() return app.appConn.LoadSnapshotChunkSync(ctx, req) } func (app *appConnSnapshot) ApplySnapshotChunkSync( ctx context.Context, req types.RequestApplySnapshotChunk) (*types.ResponseApplySnapshotChunk, error) { + defer addTimeSample(app.metrics.MethodTiming.With("method", "apply_snapshot_chunk", "type", "sync"))() return app.appConn.ApplySnapshotChunkSync(ctx, req) } + +// addTimeSample returns a function that, when called, adds an observation to m. +// The observation added to m is the number of seconds ellapsed since addTimeSample +// was initially called. addTimeSample is meant to be called in a defer to calculate +// the amount of time a function takes to complete. +func addTimeSample(m metrics.Histogram) func() { + start := time.Now() + return func() { m.Observe(time.Since(start).Seconds()) } +} diff --git a/internal/proxy/metrics.go b/internal/proxy/metrics.go new file mode 100644 index 000000000..99bd7d7b0 --- /dev/null +++ b/internal/proxy/metrics.go @@ -0,0 +1,47 @@ +package proxy + +import ( + "github.com/go-kit/kit/metrics" + "github.com/go-kit/kit/metrics/discard" + "github.com/go-kit/kit/metrics/prometheus" + stdprometheus "github.com/prometheus/client_golang/prometheus" +) + +const ( + // MetricsSubsystem is a subsystem shared by all metrics exposed by this + // package. + MetricsSubsystem = "abci_connection" +) + +// Metrics contains the prometheus metrics exposed by the proxy package. +type Metrics struct { + MethodTiming metrics.Histogram +} + +// PrometheusMetrics constructs a Metrics instance that collects metrics samples. +// The resulting metrics will be prefixed with namespace and labeled with the +// defaultLabelsAndValues. defaultLabelsAndValues must be a list of string pairs +// where the first of each pair is the label and the second is the value. +func PrometheusMetrics(namespace string, defaultLabelsAndValues ...string) *Metrics { + defaultLabels := []string{} + for i := 0; i < len(defaultLabelsAndValues); i += 2 { + defaultLabels = append(defaultLabels, defaultLabelsAndValues[i]) + } + return &Metrics{ + MethodTiming: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: MetricsSubsystem, + Name: "method_timing", + Help: "ABCI Method Timing", + Buckets: []float64{.0001, .0004, .002, .009, .02, .1, .65, 2, 6, 25}, + }, append(defaultLabels, []string{"method", "type"}...)).With(defaultLabelsAndValues...), + } +} + +// NopMetrics constructs a Metrics instance that discards all samples and is suitable +// for testing. +func NopMetrics() *Metrics { + return &Metrics{ + MethodTiming: discard.NewHistogram(), + } +} diff --git a/internal/proxy/multi_app_conn.go b/internal/proxy/multi_app_conn.go index df49df287..0bcc64af6 100644 --- a/internal/proxy/multi_app_conn.go +++ b/internal/proxy/multi_app_conn.go @@ -33,8 +33,8 @@ type AppConns interface { } // NewAppConns calls NewMultiAppConn. -func NewAppConns(clientCreator abciclient.Creator) AppConns { - return NewMultiAppConn(clientCreator) +func NewAppConns(clientCreator abciclient.Creator, metrics *Metrics) AppConns { + return NewMultiAppConn(clientCreator, metrics) } // multiAppConn implements AppConns. @@ -45,6 +45,7 @@ func NewAppConns(clientCreator abciclient.Creator) AppConns { type multiAppConn struct { service.BaseService + metrics *Metrics consensusConn AppConnConsensus mempoolConn AppConnMempool queryConn AppConnQuery @@ -59,8 +60,9 @@ type multiAppConn struct { } // NewMultiAppConn makes all necessary abci connections to the application. -func NewMultiAppConn(clientCreator abciclient.Creator) AppConns { +func NewMultiAppConn(clientCreator abciclient.Creator, metrics *Metrics) AppConns { multiAppConn := &multiAppConn{ + metrics: metrics, clientCreator: clientCreator, } multiAppConn.BaseService = *service.NewBaseService(nil, "multiAppConn", multiAppConn) @@ -89,7 +91,7 @@ func (app *multiAppConn) OnStart() error { return err } app.queryConnClient = c - app.queryConn = NewAppConnQuery(c) + app.queryConn = NewAppConnQuery(c, app.metrics) c, err = app.abciClientFor(connSnapshot) if err != nil { @@ -97,7 +99,7 @@ func (app *multiAppConn) OnStart() error { return err } app.snapshotConnClient = c - app.snapshotConn = NewAppConnSnapshot(c) + app.snapshotConn = NewAppConnSnapshot(c, app.metrics) c, err = app.abciClientFor(connMempool) if err != nil { @@ -105,7 +107,7 @@ func (app *multiAppConn) OnStart() error { return err } app.mempoolConnClient = c - app.mempoolConn = NewAppConnMempool(c) + app.mempoolConn = NewAppConnMempool(c, app.metrics) c, err = app.abciClientFor(connConsensus) if err != nil { @@ -113,7 +115,7 @@ func (app *multiAppConn) OnStart() error { return err } app.consensusConnClient = c - app.consensusConn = NewAppConnConsensus(c) + app.consensusConn = NewAppConnConsensus(c, app.metrics) // Kill Tendermint if the ABCI application crashes. go app.killTMOnClientError() diff --git a/internal/proxy/multi_app_conn_test.go b/internal/proxy/multi_app_conn_test.go index 55bcc7524..25ed692ab 100644 --- a/internal/proxy/multi_app_conn_test.go +++ b/internal/proxy/multi_app_conn_test.go @@ -31,7 +31,7 @@ func TestAppConns_Start_Stop(t *testing.T) { return clientMock, nil } - appConns := NewAppConns(creator) + appConns := NewAppConns(creator, NopMetrics()) err := appConns.Start() require.NoError(t, err) @@ -72,7 +72,7 @@ func TestAppConns_Failure(t *testing.T) { return clientMock, nil } - appConns := NewAppConns(creator) + appConns := NewAppConns(creator, NopMetrics()) err := appConns.Start() require.NoError(t, err) diff --git a/internal/state/execution_test.go b/internal/state/execution_test.go index 8eff0430d..a66b677f9 100644 --- a/internal/state/execution_test.go +++ b/internal/state/execution_test.go @@ -36,7 +36,7 @@ var ( func TestApplyBlock(t *testing.T) { app := &testApp{} cc := abciclient.NewLocalCreator(app) - proxyApp := proxy.NewAppConns(cc) + proxyApp := proxy.NewAppConns(cc, proxy.NopMetrics()) err := proxyApp.Start() require.Nil(t, err) defer proxyApp.Stop() //nolint:errcheck // ignore for tests @@ -61,7 +61,7 @@ func TestApplyBlock(t *testing.T) { func TestBeginBlockValidators(t *testing.T) { app := &testApp{} cc := abciclient.NewLocalCreator(app) - proxyApp := proxy.NewAppConns(cc) + proxyApp := proxy.NewAppConns(cc, proxy.NopMetrics()) err := proxyApp.Start() require.Nil(t, err) defer proxyApp.Stop() //nolint:errcheck // no need to check error again @@ -124,7 +124,7 @@ func TestBeginBlockValidators(t *testing.T) { func TestBeginBlockByzantineValidators(t *testing.T) { app := &testApp{} cc := abciclient.NewLocalCreator(app) - proxyApp := proxy.NewAppConns(cc) + proxyApp := proxy.NewAppConns(cc, proxy.NopMetrics()) err := proxyApp.Start() require.Nil(t, err) defer proxyApp.Stop() //nolint:errcheck // ignore for tests @@ -349,7 +349,7 @@ func TestUpdateValidators(t *testing.T) { func TestEndBlockValidatorUpdates(t *testing.T) { app := &testApp{} cc := abciclient.NewLocalCreator(app) - proxyApp := proxy.NewAppConns(cc) + proxyApp := proxy.NewAppConns(cc, proxy.NopMetrics()) err := proxyApp.Start() require.Nil(t, err) defer proxyApp.Stop() //nolint:errcheck // ignore for tests @@ -422,7 +422,7 @@ func TestEndBlockValidatorUpdates(t *testing.T) { func TestEndBlockValidatorUpdatesResultingInEmptySet(t *testing.T) { app := &testApp{} cc := abciclient.NewLocalCreator(app) - proxyApp := proxy.NewAppConns(cc) + proxyApp := proxy.NewAppConns(cc, proxy.NopMetrics()) err := proxyApp.Start() require.Nil(t, err) defer proxyApp.Stop() //nolint:errcheck // ignore for tests diff --git a/internal/state/helpers_test.go b/internal/state/helpers_test.go index f8ed77b32..0cedebb00 100644 --- a/internal/state/helpers_test.go +++ b/internal/state/helpers_test.go @@ -31,7 +31,7 @@ type paramsChangeTestCase struct { func newTestApp() proxy.AppConns { app := &testApp{} cc := abciclient.NewLocalCreator(app) - return proxy.NewAppConns(cc) + return proxy.NewAppConns(cc, proxy.NopMetrics()) } func makeAndCommitGoodBlock( diff --git a/node/node.go b/node/node.go index 6ee0d30f9..e15bde553 100644 --- a/node/node.go +++ b/node/node.go @@ -157,8 +157,10 @@ func makeNode(cfg *config.Config, } + nodeMetrics := defaultMetricsProvider(cfg.Instrumentation)(genDoc.ChainID) + // Create the proxyApp and establish connections to the ABCI app (consensus, mempool, query). - proxyApp, err := createAndStartProxyAppConns(clientCreator, logger) + proxyApp, err := createAndStartProxyAppConns(clientCreator, logger, nodeMetrics.proxy) if err != nil { return nil, combineCloseError(err, makeCloser(closers)) @@ -268,8 +270,6 @@ func makeNode(cfg *config.Config, makeCloser(closers)) } - nodeMetrics := defaultMetricsProvider(cfg.Instrumentation)(genDoc.ChainID) - router, err := createRouter(p2pLogger, nodeMetrics.p2p, nodeInfo, nodeKey.PrivKey, peerManager, transport, getRouterConfig(cfg, proxyApp)) if err != nil { @@ -988,6 +988,7 @@ type nodeMetrics struct { mempool *mempool.Metrics state *sm.Metrics statesync *statesync.Metrics + proxy *proxy.Metrics } // metricsProvider returns consensus, p2p, mempool, state, statesync Metrics. @@ -1004,6 +1005,7 @@ func defaultMetricsProvider(cfg *config.InstrumentationConfig) metricsProvider { mempool.PrometheusMetrics(cfg.Namespace, "chain_id", chainID), sm.PrometheusMetrics(cfg.Namespace, "chain_id", chainID), statesync.PrometheusMetrics(cfg.Namespace, "chain_id", chainID), + proxy.PrometheusMetrics(cfg.Namespace, "chain_id", chainID), } } return &nodeMetrics{ @@ -1012,6 +1014,7 @@ func defaultMetricsProvider(cfg *config.InstrumentationConfig) metricsProvider { mempool.NopMetrics(), sm.NopMetrics(), statesync.NopMetrics(), + proxy.NopMetrics(), } } } diff --git a/node/node_test.go b/node/node_test.go index 7039b09e1..19f27a640 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -240,7 +240,7 @@ func TestCreateProposalBlock(t *testing.T) { cfg := config.ResetTestRoot("node_create_proposal") defer os.RemoveAll(cfg.RootDir) cc := abciclient.NewLocalCreator(kvstore.NewApplication()) - proxyApp := proxy.NewAppConns(cc) + proxyApp := proxy.NewAppConns(cc, proxy.NopMetrics()) err := proxyApp.Start() require.Nil(t, err) defer proxyApp.Stop() //nolint:errcheck // ignore for tests @@ -335,7 +335,7 @@ func TestMaxTxsProposalBlockSize(t *testing.T) { cfg := config.ResetTestRoot("node_create_proposal") defer os.RemoveAll(cfg.RootDir) cc := abciclient.NewLocalCreator(kvstore.NewApplication()) - proxyApp := proxy.NewAppConns(cc) + proxyApp := proxy.NewAppConns(cc, proxy.NopMetrics()) err := proxyApp.Start() require.Nil(t, err) defer proxyApp.Stop() //nolint:errcheck // ignore for tests @@ -400,7 +400,7 @@ func TestMaxProposalBlockSize(t *testing.T) { cfg := config.ResetTestRoot("node_create_proposal") defer os.RemoveAll(cfg.RootDir) cc := abciclient.NewLocalCreator(kvstore.NewApplication()) - proxyApp := proxy.NewAppConns(cc) + proxyApp := proxy.NewAppConns(cc, proxy.NopMetrics()) err := proxyApp.Start() require.Nil(t, err) defer proxyApp.Stop() //nolint:errcheck // ignore for tests diff --git a/node/setup.go b/node/setup.go index ac36b0858..ebaa576c2 100644 --- a/node/setup.go +++ b/node/setup.go @@ -90,8 +90,9 @@ func initDBs( return blockStore, stateDB, makeCloser(closers), nil } -func createAndStartProxyAppConns(clientCreator abciclient.Creator, logger log.Logger) (proxy.AppConns, error) { - proxyApp := proxy.NewAppConns(clientCreator) +// nolint:lll +func createAndStartProxyAppConns(clientCreator abciclient.Creator, logger log.Logger, metrics *proxy.Metrics) (proxy.AppConns, error) { + proxyApp := proxy.NewAppConns(clientCreator, metrics) proxyApp.SetLogger(logger.With("module", "proxy")) if err := proxyApp.Start(); err != nil { return nil, fmt.Errorf("error starting proxy app connections: %v", err)