internal/proxy: add initial set of abci metrics backport (#7342)

* internal/proxy: add initial set of abci metrics (#7115)

This PR adds an initial set of metrics for use ABCI. The initial metrics enable the calculation of timing histograms and call counts for each of the ABCI methods. The metrics are also labeled as either 'sync' or 'async' to determine if the method call was performed using ABCI's `*Async` methods.

An example of these metrics is included here for reference:
```
tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.0001"} 0
tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.0004"} 5
tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.002"} 12
tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.009"} 13
tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.02"} 13
tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.1"} 13
tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.65"} 13
tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="2"} 13
tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="6"} 13
tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="25"} 13
tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="+Inf"} 13
tendermint_abci_connection_method_timing_sum{chain_id="ci",method="commit",type="sync"} 0.007802058000000001
tendermint_abci_connection_method_timing_count{chain_id="ci",method="commit",type="sync"} 13
```

These metrics can easily be graphed using prometheus's `histogram_quantile(...)` method to pick out a particular quantile to graph or examine. I chose buckets that were somewhat of an estimate of expected range of times for ABCI operations. They start at .0001 seconds and range to 25 seconds. The hope is that this range captures enough possible times to be useful for us and operators.

* lint++

* docs: add abci timing metrics to the metrics docs (#7311)

* cherry-pick fixup

docs/nodes/metrics.md

@@ -20,6 +20,7 @@ The following metrics are available:
 
 | **Name**                               | **Type**  | **Tags**      | **Description**                                                        |
 | -------------------------------------- | --------- | ------------- | ---------------------------------------------------------------------- |
+| abci_connection_method_timing          | Histogram | method, type  | Timings for each of the ABCI methods                                   |
 | consensus_height                       | Gauge     |               | Height of the chain                                                    |
 | consensus_validators                   | Gauge     |               | Number of validators                                                   |
 | consensus_validators_power             | Gauge     |               | Total voting power of all validators                                   |
@@ -55,6 +56,16 @@ The following metrics are available:
 
 Percentage of missing + byzantine validators:
 
-```md
-((consensus\_byzantine\_validators\_power + consensus\_missing\_validators\_power) / consensus\_validators\_power) * 100
+```prometheus
+((consensus_byzantine_validators_power + consensus_missing_validators_power) / consensus_validators_power) * 100
+```
+
+Rate at which the application is responding to each ABCI method call.
+```
+sum(rate(tendermint_abci_connection_method_timing_count[5m])) by (method)
+```
+
+The 95th percentile response time for the application to the `deliver_tx` ABCI method call.
+```
+histogram_quantile(0.95, sum by(le) (rate(tendermint_abci_connection_method_timing_bucket{method="deliver_tx"}[5m])))
 ```

internal/blocksync/v0/reactor_test.go

@@ -98,7 +98,7 @@ func (rts *reactorTestSuite) addNode(t *testing.T,
 	t.Helper()
 
 	rts.nodes = append(rts.nodes, nodeID)
-	rts.app[nodeID] = proxy.NewAppConns(abciclient.NewLocalCreator(&abci.BaseApplication{}))
+	rts.app[nodeID] = proxy.NewAppConns(abciclient.NewLocalCreator(&abci.BaseApplication{}), proxy.NopMetrics())
 	require.NoError(t, rts.app[nodeID].Start())
 
 	blockDB := dbm.NewMemDB()

internal/blocksync/v2/reactor_test.go

@@ -165,7 +165,7 @@ func newTestReactor(t *testing.T, p testReactorParams) *BlockchainReactor {
 	} else {
 		app := &testApp{}
 		cc := abciclient.NewLocalCreator(app)
-		proxyApp := proxy.NewAppConns(cc)
+		proxyApp := proxy.NewAppConns(cc, proxy.NopMetrics())
 		err := proxyApp.Start()
 		require.NoError(t, err)
 		db := dbm.NewMemDB()
@@ -486,7 +486,7 @@ func newReactorStore(
 	require.Len(t, privVals, 1)
 	app := &testApp{}
 	cc := abciclient.NewLocalCreator(app)
-	proxyApp := proxy.NewAppConns(cc)
+	proxyApp := proxy.NewAppConns(cc, proxy.NopMetrics())
 	err := proxyApp.Start()
 	if err != nil {
 		panic(fmt.Errorf("error start app: %w", err))

internal/consensus/metrics.go

@@ -61,6 +61,9 @@ type Metrics struct {
 
 	// Number of blockparts transmitted by peer.
 	BlockParts metrics.Counter
+
+	// Histogram of time taken per step annotated with reason that the step proceeded.
+	StepTime metrics.Histogram
 }
 
 // PrometheusMetrics returns Metrics build using Prometheus client library.
@@ -187,6 +190,12 @@ func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
 			Name:      "block_parts",
 			Help:      "Number of blockparts transmitted by peer.",
 		}, append(labels, "peer_id")).With(labelsAndValues...),
+		StepTime: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "step_time",
+			Help:      "Time spent per step.",
+		}, append(labels, "step", "reason")).With(labelsAndValues...),
 	}
 }
 

internal/consensus/replay_file.go

@@ -312,7 +312,7 @@ func newConsensusStateForReplay(cfg config.BaseConfig, csConfig *config.Consensu
 
 	// Create proxyAppConn connection (consensus, mempool, query)
 	clientCreator, _ := proxy.DefaultClientCreator(cfg.ProxyApp, cfg.ABCI, cfg.DBDir())
-	proxyApp := proxy.NewAppConns(clientCreator)
+	proxyApp := proxy.NewAppConns(clientCreator, proxy.NopMetrics())
 	err = proxyApp.Start()
 	if err != nil {
 		tmos.Exit(fmt.Sprintf("Error starting proxy app conns: %v", err))

internal/consensus/replay_stubs.go

@@ -64,7 +64,7 @@ func newMockProxyApp(appHash []byte, abciResponses *tmstate.ABCIResponses) proxy
 	if err != nil {
 		panic(err)
 	}
-	return proxy.NewAppConnConsensus(cli)
+	return proxy.NewAppConnConsensus(cli, proxy.NopMetrics())
 }
 
 type mockProxyApp struct {

internal/consensus/replay_test.go

@@ -748,7 +748,7 @@ func testHandshakeReplay(t *testing.T, sim *simulatorTestSuite, nBlocks int, mod
 	if nBlocks > 0 {
 		// run nBlocks against a new client to build up the app state.
 		// use a throwaway tendermint state
-		proxyApp := proxy.NewAppConns(clientCreator2)
+		proxyApp := proxy.NewAppConns(clientCreator2, proxy.NopMetrics())
 		stateDB1 := dbm.NewMemDB()
 		stateStore := sm.NewStore(stateDB1)
 		err := stateStore.Save(genesisState)
@@ -768,7 +768,7 @@ func testHandshakeReplay(t *testing.T, sim *simulatorTestSuite, nBlocks int, mod
 	// now start the app using the handshake - it should sync
 	genDoc, _ := sm.MakeGenesisDocFromFile(cfg.GenesisFile())
 	handshaker := NewHandshaker(stateStore, state, store, genDoc)
-	proxyApp := proxy.NewAppConns(clientCreator2)
+	proxyApp := proxy.NewAppConns(clientCreator2, proxy.NopMetrics())
 	if err := proxyApp.Start(); err != nil {
 		t.Fatalf("Error starting proxy app connections: %v", err)
 	}
@@ -896,7 +896,7 @@ func buildTMStateFromChain(
 	defer kvstoreApp.Close()
 	clientCreator := abciclient.NewLocalCreator(kvstoreApp)
 
-	proxyApp := proxy.NewAppConns(clientCreator)
+	proxyApp := proxy.NewAppConns(clientCreator, proxy.NopMetrics())
 	if err := proxyApp.Start(); err != nil {
 		panic(err)
 	}
@@ -964,7 +964,7 @@ func TestHandshakePanicsIfAppReturnsWrongAppHash(t *testing.T) {
 	{
 		app := &badApp{numBlocks: 3, allHashesAreWrong: true}
 		clientCreator := abciclient.NewLocalCreator(app)
-		proxyApp := proxy.NewAppConns(clientCreator)
+		proxyApp := proxy.NewAppConns(clientCreator, proxy.NopMetrics())
 		err := proxyApp.Start()
 		require.NoError(t, err)
 		t.Cleanup(func() {
@@ -988,7 +988,7 @@ func TestHandshakePanicsIfAppReturnsWrongAppHash(t *testing.T) {
 	{
 		app := &badApp{numBlocks: 3, onlyLastHashIsWrong: true}
 		clientCreator := abciclient.NewLocalCreator(app)
-		proxyApp := proxy.NewAppConns(clientCreator)
+		proxyApp := proxy.NewAppConns(clientCreator, proxy.NopMetrics())
 		err := proxyApp.Start()
 		require.NoError(t, err)
 		t.Cleanup(func() {
@@ -1248,7 +1248,7 @@ func TestHandshakeUpdatesValidators(t *testing.T) {
 	// now start the app using the handshake - it should sync
 	genDoc, _ := sm.MakeGenesisDocFromFile(cfg.GenesisFile())
 	handshaker := NewHandshaker(stateStore, state, store, genDoc)
-	proxyApp := proxy.NewAppConns(clientCreator)
+	proxyApp := proxy.NewAppConns(clientCreator, proxy.NopMetrics())
 	if err := proxyApp.Start(); err != nil {
 		t.Fatalf("Error starting proxy app connections: %v", err)
 	}

internal/consensus/wal_generator.go

@@ -65,7 +65,7 @@ func WALGenerateNBlocks(t *testing.T, wr io.Writer, numBlocks int) (err error) {
 
 	blockStore := store.NewBlockStore(blockStoreDB)
 
-	proxyApp := proxy.NewAppConns(abciclient.NewLocalCreator(app))
+	proxyApp := proxy.NewAppConns(abciclient.NewLocalCreator(app), proxy.NopMetrics())
 	proxyApp.SetLogger(logger.With("module", "proxy"))
 	if err := proxyApp.Start(); err != nil {
 		return fmt.Errorf("failed to start proxy app connections: %w", err)

internal/proxy/app_conn.go

@@ -2,7 +2,9 @@ package proxy
 
 import (
 	"context"
+	"time"
 
+	"github.com/go-kit/kit/metrics"
 	abciclient "github.com/tendermint/tendermint/abci/client"
 	"github.com/tendermint/tendermint/abci/types"
 )
@@ -56,11 +58,13 @@ type AppConnSnapshot interface {
 // Implements AppConnConsensus (subset of abciclient.Client)
 
 type appConnConsensus struct {
+	metrics *Metrics
 	appConn abciclient.Client
 }
 
-func NewAppConnConsensus(appConn abciclient.Client) AppConnConsensus {
+func NewAppConnConsensus(appConn abciclient.Client, metrics *Metrics) AppConnConsensus {
 	return &appConnConsensus{
+		metrics: metrics,
 		appConn: appConn,
 	}
 }
@@ -77,6 +81,7 @@ func (app *appConnConsensus) InitChainSync(
 	ctx context.Context,
 	req types.RequestInitChain,
 ) (*types.ResponseInitChain, error) {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "init_chain", "type", "sync"))()
 	return app.appConn.InitChainSync(ctx, req)
 }
 
@@ -84,6 +89,7 @@ func (app *appConnConsensus) BeginBlockSync(
 	ctx context.Context,
 	req types.RequestBeginBlock,
 ) (*types.ResponseBeginBlock, error) {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "begin_block", "type", "sync"))()
 	return app.appConn.BeginBlockSync(ctx, req)
 }
 
@@ -91,6 +97,7 @@ func (app *appConnConsensus) DeliverTxAsync(
 	ctx context.Context,
 	req types.RequestDeliverTx,
 ) (*abciclient.ReqRes, error) {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "deliver_tx", "type", "async"))()
 	return app.appConn.DeliverTxAsync(ctx, req)
 }
 
@@ -98,10 +105,12 @@ func (app *appConnConsensus) EndBlockSync(
 	ctx context.Context,
 	req types.RequestEndBlock,
 ) (*types.ResponseEndBlock, error) {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "deliver_tx", "type", "sync"))()
 	return app.appConn.EndBlockSync(ctx, req)
 }
 
 func (app *appConnConsensus) CommitSync(ctx context.Context) (*types.ResponseCommit, error) {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "commit", "type", "sync"))()
 	return app.appConn.CommitSync(ctx)
 }
 
@@ -109,11 +118,13 @@ func (app *appConnConsensus) CommitSync(ctx context.Context) (*types.ResponseCom
 // Implements AppConnMempool (subset of abciclient.Client)
 
 type appConnMempool struct {
+	metrics *Metrics
 	appConn abciclient.Client
 }
 
-func NewAppConnMempool(appConn abciclient.Client) AppConnMempool {
+func NewAppConnMempool(appConn abciclient.Client, metrics *Metrics) AppConnMempool {
 	return &appConnMempool{
+		metrics: metrics,
 		appConn: appConn,
 	}
 }
@@ -127,18 +138,22 @@ func (app *appConnMempool) Error() error {
 }
 
 func (app *appConnMempool) FlushAsync(ctx context.Context) (*abciclient.ReqRes, error) {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "flush", "type", "async"))()
 	return app.appConn.FlushAsync(ctx)
 }
 
 func (app *appConnMempool) FlushSync(ctx context.Context) error {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "flush", "type", "sync"))()
 	return app.appConn.FlushSync(ctx)
 }
 
 func (app *appConnMempool) CheckTxAsync(ctx context.Context, req types.RequestCheckTx) (*abciclient.ReqRes, error) {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "check_tx", "type", "async"))()
 	return app.appConn.CheckTxAsync(ctx, req)
 }
 
 func (app *appConnMempool) CheckTxSync(ctx context.Context, req types.RequestCheckTx) (*types.ResponseCheckTx, error) {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "check_tx", "type", "sync"))()
 	return app.appConn.CheckTxSync(ctx, req)
 }
 
@@ -146,11 +161,13 @@ func (app *appConnMempool) CheckTxSync(ctx context.Context, req types.RequestChe
 // Implements AppConnQuery (subset of abciclient.Client)
 
 type appConnQuery struct {
+	metrics *Metrics
 	appConn abciclient.Client
 }
 
-func NewAppConnQuery(appConn abciclient.Client) AppConnQuery {
+func NewAppConnQuery(appConn abciclient.Client, metrics *Metrics) AppConnQuery {
 	return &appConnQuery{
+		metrics: metrics,
 		appConn: appConn,
 	}
 }
@@ -160,14 +177,17 @@ func (app *appConnQuery) Error() error {
 }
 
 func (app *appConnQuery) EchoSync(ctx context.Context, msg string) (*types.ResponseEcho, error) {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "echo", "type", "sync"))()
 	return app.appConn.EchoSync(ctx, msg)
 }
 
 func (app *appConnQuery) InfoSync(ctx context.Context, req types.RequestInfo) (*types.ResponseInfo, error) {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "info", "type", "sync"))()
 	return app.appConn.InfoSync(ctx, req)
 }
 
 func (app *appConnQuery) QuerySync(ctx context.Context, reqQuery types.RequestQuery) (*types.ResponseQuery, error) {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "query", "type", "sync"))()
 	return app.appConn.QuerySync(ctx, reqQuery)
 }
 
@@ -175,11 +195,13 @@ func (app *appConnQuery) QuerySync(ctx context.Context, reqQuery types.RequestQu
 // Implements AppConnSnapshot (subset of abciclient.Client)
 
 type appConnSnapshot struct {
+	metrics *Metrics
 	appConn abciclient.Client
 }
 
-func NewAppConnSnapshot(appConn abciclient.Client) AppConnSnapshot {
+func NewAppConnSnapshot(appConn abciclient.Client, metrics *Metrics) AppConnSnapshot {
 	return &appConnSnapshot{
+		metrics: metrics,
 		appConn: appConn,
 	}
 }
@@ -192,6 +214,7 @@ func (app *appConnSnapshot) ListSnapshotsSync(
 	ctx context.Context,
 	req types.RequestListSnapshots,
 ) (*types.ResponseListSnapshots, error) {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "list_snapshots", "type", "sync"))()
 	return app.appConn.ListSnapshotsSync(ctx, req)
 }
 
@@ -199,17 +222,29 @@ func (app *appConnSnapshot) OfferSnapshotSync(
 	ctx context.Context,
 	req types.RequestOfferSnapshot,
 ) (*types.ResponseOfferSnapshot, error) {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "offer_snapshot", "type", "sync"))()
 	return app.appConn.OfferSnapshotSync(ctx, req)
 }
 
 func (app *appConnSnapshot) LoadSnapshotChunkSync(
 	ctx context.Context,
 	req types.RequestLoadSnapshotChunk) (*types.ResponseLoadSnapshotChunk, error) {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "load_snapshot_chunk", "type", "sync"))()
 	return app.appConn.LoadSnapshotChunkSync(ctx, req)
 }
 
 func (app *appConnSnapshot) ApplySnapshotChunkSync(
 	ctx context.Context,
 	req types.RequestApplySnapshotChunk) (*types.ResponseApplySnapshotChunk, error) {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "apply_snapshot_chunk", "type", "sync"))()
 	return app.appConn.ApplySnapshotChunkSync(ctx, req)
 }
+
+// addTimeSample returns a function that, when called, adds an observation to m.
+// The observation added to m is the number of seconds ellapsed since addTimeSample
+// was initially called. addTimeSample is meant to be called in a defer to calculate
+// the amount of time a function takes to complete.
+func addTimeSample(m metrics.Histogram) func() {
+	start := time.Now()
+	return func() { m.Observe(time.Since(start).Seconds()) }
+}

internal/proxy/metrics.go

@@ -0,0 +1,47 @@
+package proxy
+
+import (
+	"github.com/go-kit/kit/metrics"
+	"github.com/go-kit/kit/metrics/discard"
+	"github.com/go-kit/kit/metrics/prometheus"
+	stdprometheus "github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	// MetricsSubsystem is a subsystem shared by all metrics exposed by this
+	// package.
+	MetricsSubsystem = "abci_connection"
+)
+
+// Metrics contains the prometheus metrics exposed by the proxy package.
+type Metrics struct {
+	MethodTiming metrics.Histogram
+}
+
+// PrometheusMetrics constructs a Metrics instance that collects metrics samples.
+// The resulting metrics will be prefixed with namespace and labeled with the
+// defaultLabelsAndValues. defaultLabelsAndValues must be a list of string pairs
+// where the first of each pair is the label and the second is the value.
+func PrometheusMetrics(namespace string, defaultLabelsAndValues ...string) *Metrics {
+	defaultLabels := []string{}
+	for i := 0; i < len(defaultLabelsAndValues); i += 2 {
+		defaultLabels = append(defaultLabels, defaultLabelsAndValues[i])
+	}
+	return &Metrics{
+		MethodTiming: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "method_timing",
+			Help:      "ABCI Method Timing",
+			Buckets:   []float64{.0001, .0004, .002, .009, .02, .1, .65, 2, 6, 25},
+		}, append(defaultLabels, []string{"method", "type"}...)).With(defaultLabelsAndValues...),
+	}
+}
+
+// NopMetrics constructs a Metrics instance that discards all samples and is suitable
+// for testing.
+func NopMetrics() *Metrics {
+	return &Metrics{
+		MethodTiming: discard.NewHistogram(),
+	}
+}

internal/proxy/multi_app_conn.go

@@ -33,8 +33,8 @@ type AppConns interface {
 }
 
 // NewAppConns calls NewMultiAppConn.
-func NewAppConns(clientCreator abciclient.Creator) AppConns {
-	return NewMultiAppConn(clientCreator)
+func NewAppConns(clientCreator abciclient.Creator, metrics *Metrics) AppConns {
+	return NewMultiAppConn(clientCreator, metrics)
 }
 
 // multiAppConn implements AppConns.
@@ -45,6 +45,7 @@ func NewAppConns(clientCreator abciclient.Creator) AppConns {
 type multiAppConn struct {
 	service.BaseService
 
+	metrics       *Metrics
 	consensusConn AppConnConsensus
 	mempoolConn   AppConnMempool
 	queryConn     AppConnQuery
@@ -59,8 +60,9 @@ type multiAppConn struct {
 }
 
 // NewMultiAppConn makes all necessary abci connections to the application.
-func NewMultiAppConn(clientCreator abciclient.Creator) AppConns {
+func NewMultiAppConn(clientCreator abciclient.Creator, metrics *Metrics) AppConns {
 	multiAppConn := &multiAppConn{
+		metrics:       metrics,
 		clientCreator: clientCreator,
 	}
 	multiAppConn.BaseService = *service.NewBaseService(nil, "multiAppConn", multiAppConn)
@@ -89,7 +91,7 @@ func (app *multiAppConn) OnStart() error {
 		return err
 	}
 	app.queryConnClient = c
-	app.queryConn = NewAppConnQuery(c)
+	app.queryConn = NewAppConnQuery(c, app.metrics)
 
 	c, err = app.abciClientFor(connSnapshot)
 	if err != nil {
@@ -97,7 +99,7 @@ func (app *multiAppConn) OnStart() error {
 		return err
 	}
 	app.snapshotConnClient = c
-	app.snapshotConn = NewAppConnSnapshot(c)
+	app.snapshotConn = NewAppConnSnapshot(c, app.metrics)
 
 	c, err = app.abciClientFor(connMempool)
 	if err != nil {
@@ -105,7 +107,7 @@ func (app *multiAppConn) OnStart() error {
 		return err
 	}
 	app.mempoolConnClient = c
-	app.mempoolConn = NewAppConnMempool(c)
+	app.mempoolConn = NewAppConnMempool(c, app.metrics)
 
 	c, err = app.abciClientFor(connConsensus)
 	if err != nil {
@@ -113,7 +115,7 @@ func (app *multiAppConn) OnStart() error {
 		return err
 	}
 	app.consensusConnClient = c
-	app.consensusConn = NewAppConnConsensus(c)
+	app.consensusConn = NewAppConnConsensus(c, app.metrics)
 
 	// Kill Tendermint if the ABCI application crashes.
 	go app.killTMOnClientError()

internal/proxy/multi_app_conn_test.go

@@ -31,7 +31,7 @@ func TestAppConns_Start_Stop(t *testing.T) {
 		return clientMock, nil
 	}
 
-	appConns := NewAppConns(creator)
+	appConns := NewAppConns(creator, NopMetrics())
 
 	err := appConns.Start()
 	require.NoError(t, err)
@@ -72,7 +72,7 @@ func TestAppConns_Failure(t *testing.T) {
 		return clientMock, nil
 	}
 
-	appConns := NewAppConns(creator)
+	appConns := NewAppConns(creator, NopMetrics())
 
 	err := appConns.Start()
 	require.NoError(t, err)

internal/state/execution_test.go

@@ -36,7 +36,7 @@ var (
 func TestApplyBlock(t *testing.T) {
 	app := &testApp{}
 	cc := abciclient.NewLocalCreator(app)
-	proxyApp := proxy.NewAppConns(cc)
+	proxyApp := proxy.NewAppConns(cc, proxy.NopMetrics())
 	err := proxyApp.Start()
 	require.Nil(t, err)
 	defer proxyApp.Stop() //nolint:errcheck // ignore for tests
@@ -61,7 +61,7 @@ func TestApplyBlock(t *testing.T) {
 func TestBeginBlockValidators(t *testing.T) {
 	app := &testApp{}
 	cc := abciclient.NewLocalCreator(app)
-	proxyApp := proxy.NewAppConns(cc)
+	proxyApp := proxy.NewAppConns(cc, proxy.NopMetrics())
 	err := proxyApp.Start()
 	require.Nil(t, err)
 	defer proxyApp.Stop() //nolint:errcheck // no need to check error again
@@ -124,7 +124,7 @@ func TestBeginBlockValidators(t *testing.T) {
 func TestBeginBlockByzantineValidators(t *testing.T) {
 	app := &testApp{}
 	cc := abciclient.NewLocalCreator(app)
-	proxyApp := proxy.NewAppConns(cc)
+	proxyApp := proxy.NewAppConns(cc, proxy.NopMetrics())
 	err := proxyApp.Start()
 	require.Nil(t, err)
 	defer proxyApp.Stop() //nolint:errcheck // ignore for tests
@@ -349,7 +349,7 @@ func TestUpdateValidators(t *testing.T) {
 func TestEndBlockValidatorUpdates(t *testing.T) {
 	app := &testApp{}
 	cc := abciclient.NewLocalCreator(app)
-	proxyApp := proxy.NewAppConns(cc)
+	proxyApp := proxy.NewAppConns(cc, proxy.NopMetrics())
 	err := proxyApp.Start()
 	require.Nil(t, err)
 	defer proxyApp.Stop() //nolint:errcheck // ignore for tests
@@ -422,7 +422,7 @@ func TestEndBlockValidatorUpdates(t *testing.T) {
 func TestEndBlockValidatorUpdatesResultingInEmptySet(t *testing.T) {
 	app := &testApp{}
 	cc := abciclient.NewLocalCreator(app)
-	proxyApp := proxy.NewAppConns(cc)
+	proxyApp := proxy.NewAppConns(cc, proxy.NopMetrics())
 	err := proxyApp.Start()
 	require.Nil(t, err)
 	defer proxyApp.Stop() //nolint:errcheck // ignore for tests

internal/state/helpers_test.go

@@ -31,7 +31,7 @@ type paramsChangeTestCase struct {
 func newTestApp() proxy.AppConns {
 	app := &testApp{}
 	cc := abciclient.NewLocalCreator(app)
-	return proxy.NewAppConns(cc)
+	return proxy.NewAppConns(cc, proxy.NopMetrics())
 }
 
 func makeAndCommitGoodBlock(

node/node.go

@@ -152,7 +152,7 @@ func makeNode(cfg *config.Config,
 	nodeMetrics := defaultMetricsProvider(cfg.Instrumentation)(genDoc.ChainID)
 
 	// Create the proxyApp and establish connections to the ABCI app (consensus, mempool, query).
-	proxyApp, err := createAndStartProxyAppConns(clientCreator, logger)
+	proxyApp, err := createAndStartProxyAppConns(clientCreator, logger, nodeMetrics.proxy)
 	if err != nil {
 		return nil, err
 	}
@@ -1060,6 +1060,7 @@ type nodeMetrics struct {
 	p2p       *p2p.Metrics
 	state     *sm.Metrics
 	statesync *statesync.Metrics
+	proxy     *proxy.Metrics
 }
 
 // metricsProvider returns consensus, p2p, mempool, state, statesync Metrics.
@@ -1077,6 +1078,7 @@ func defaultMetricsProvider(cfg *config.InstrumentationConfig) metricsProvider {
 				p2p:       p2p.PrometheusMetrics(cfg.Namespace, "chain_id", chainID),
 				state:     sm.PrometheusMetrics(cfg.Namespace, "chain_id", chainID),
 				statesync: statesync.PrometheusMetrics(cfg.Namespace, "chain_id", chainID),
+				proxy:     proxy.PrometheusMetrics(cfg.Namespace, "chain_id", chainID),
 			}
 		}
 		return &nodeMetrics{
@@ -1086,6 +1088,7 @@ func defaultMetricsProvider(cfg *config.InstrumentationConfig) metricsProvider {
 			p2p:       p2p.NopMetrics(),
 			state:     sm.NopMetrics(),
 			statesync: statesync.NopMetrics(),
+			proxy:     proxy.NopMetrics(),
 		}
 	}
 }

node/node_test.go

@@ -222,7 +222,7 @@ func TestCreateProposalBlock(t *testing.T) {
 	require.NoError(t, err)
 	defer os.RemoveAll(cfg.RootDir)
 	cc := abciclient.NewLocalCreator(kvstore.NewApplication())
-	proxyApp := proxy.NewAppConns(cc)
+	proxyApp := proxy.NewAppConns(cc, proxy.NopMetrics())
 	err = proxyApp.Start()
 	require.Nil(t, err)
 	defer proxyApp.Stop() //nolint:errcheck // ignore for tests
@@ -315,7 +315,7 @@ func TestMaxTxsProposalBlockSize(t *testing.T) {
 	require.NoError(t, err)
 	defer os.RemoveAll(cfg.RootDir)
 	cc := abciclient.NewLocalCreator(kvstore.NewApplication())
-	proxyApp := proxy.NewAppConns(cc)
+	proxyApp := proxy.NewAppConns(cc, proxy.NopMetrics())
 	err = proxyApp.Start()
 	require.Nil(t, err)
 	defer proxyApp.Stop() //nolint:errcheck // ignore for tests
@@ -378,7 +378,7 @@ func TestMaxProposalBlockSize(t *testing.T) {
 	require.NoError(t, err)
 	defer os.RemoveAll(cfg.RootDir)
 	cc := abciclient.NewLocalCreator(kvstore.NewApplication())
-	proxyApp := proxy.NewAppConns(cc)
+	proxyApp := proxy.NewAppConns(cc, proxy.NopMetrics())
 	err = proxyApp.Start()
 	require.Nil(t, err)
 	defer proxyApp.Stop() //nolint:errcheck // ignore for tests

node/setup.go

@@ -52,8 +52,9 @@ func initDBs(cfg *config.Config, dbProvider config.DBProvider) (blockStore *stor
 	return
 }
 
-func createAndStartProxyAppConns(clientCreator abciclient.Creator, logger log.Logger) (proxy.AppConns, error) {
-	proxyApp := proxy.NewAppConns(clientCreator)
+// nolint:lll
+func createAndStartProxyAppConns(clientCreator abciclient.Creator, logger log.Logger, metrics *proxy.Metrics) (proxy.AppConns, error) {
+	proxyApp := proxy.NewAppConns(clientCreator, metrics)
 	proxyApp.SetLogger(logger.With("module", "proxy"))
 	if err := proxyApp.Start(); err != nil {
 		return nil, fmt.Errorf("error starting proxy app connections: %v", err)