You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

94 lines
2.0 KiB

internal/proxy: add initial set of abci metrics (#7115) This PR adds an initial set of metrics for use ABCI. The initial metrics enable the calculation of timing histograms and call counts for each of the ABCI methods. The metrics are also labeled as either 'sync' or 'async' to determine if the method call was performed using ABCI's `*Async` methods. An example of these metrics is included here for reference: ``` tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.0001"} 0 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.0004"} 5 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.002"} 12 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.009"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.02"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.1"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.65"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="2"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="6"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="25"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="+Inf"} 13 tendermint_abci_connection_method_timing_sum{chain_id="ci",method="commit",type="sync"} 0.007802058000000001 tendermint_abci_connection_method_timing_count{chain_id="ci",method="commit",type="sync"} 13 ``` These metrics can easily be graphed using prometheus's `histogram_quantile(...)` method to pick out a particular quantile to graph or examine. I chose buckets that were somewhat of an estimate of expected range of times for ABCI operations. They start at .0001 seconds and range to 25 seconds. The hope is that this range captures enough possible times to be useful for us and operators.
3 years ago
internal/proxy: add initial set of abci metrics (#7115) This PR adds an initial set of metrics for use ABCI. The initial metrics enable the calculation of timing histograms and call counts for each of the ABCI methods. The metrics are also labeled as either 'sync' or 'async' to determine if the method call was performed using ABCI's `*Async` methods. An example of these metrics is included here for reference: ``` tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.0001"} 0 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.0004"} 5 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.002"} 12 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.009"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.02"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.1"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.65"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="2"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="6"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="25"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="+Inf"} 13 tendermint_abci_connection_method_timing_sum{chain_id="ci",method="commit",type="sync"} 0.007802058000000001 tendermint_abci_connection_method_timing_count{chain_id="ci",method="commit",type="sync"} 13 ``` These metrics can easily be graphed using prometheus's `histogram_quantile(...)` method to pick out a particular quantile to graph or examine. I chose buckets that were somewhat of an estimate of expected range of times for ABCI operations. They start at .0001 seconds and range to 25 seconds. The hope is that this range captures enough possible times to be useful for us and operators.
3 years ago
  1. package proxy
  2. import (
  3. "errors"
  4. "os"
  5. "os/signal"
  6. "syscall"
  7. "testing"
  8. "time"
  9. "github.com/stretchr/testify/assert"
  10. "github.com/stretchr/testify/mock"
  11. "github.com/stretchr/testify/require"
  12. abciclient "github.com/tendermint/tendermint/abci/client"
  13. abcimocks "github.com/tendermint/tendermint/abci/client/mocks"
  14. )
  15. func TestAppConns_Start_Stop(t *testing.T) {
  16. quitCh := make(<-chan struct{})
  17. clientMock := &abcimocks.Client{}
  18. clientMock.On("SetLogger", mock.Anything).Return().Times(4)
  19. clientMock.On("Start").Return(nil).Times(4)
  20. clientMock.On("Stop").Return(nil).Times(4)
  21. clientMock.On("Quit").Return(quitCh).Times(4)
  22. creatorCallCount := 0
  23. creator := func() (abciclient.Client, error) {
  24. creatorCallCount++
  25. return clientMock, nil
  26. }
  27. appConns := NewAppConns(creator, NopMetrics())
  28. err := appConns.Start()
  29. require.NoError(t, err)
  30. time.Sleep(100 * time.Millisecond)
  31. err = appConns.Stop()
  32. require.NoError(t, err)
  33. clientMock.AssertExpectations(t)
  34. assert.Equal(t, 4, creatorCallCount)
  35. }
  36. // Upon failure, we call tmos.Kill
  37. func TestAppConns_Failure(t *testing.T) {
  38. ok := make(chan struct{})
  39. c := make(chan os.Signal, 1)
  40. signal.Notify(c, syscall.SIGTERM)
  41. go func() {
  42. for range c {
  43. close(ok)
  44. }
  45. }()
  46. quitCh := make(chan struct{})
  47. var recvQuitCh <-chan struct{} // nolint:gosimple
  48. recvQuitCh = quitCh
  49. clientMock := &abcimocks.Client{}
  50. clientMock.On("SetLogger", mock.Anything).Return()
  51. clientMock.On("Start").Return(nil)
  52. clientMock.On("Stop").Return(nil)
  53. clientMock.On("Quit").Return(recvQuitCh)
  54. clientMock.On("Error").Return(errors.New("EOF")).Once()
  55. creator := func() (abciclient.Client, error) {
  56. return clientMock, nil
  57. }
  58. appConns := NewAppConns(creator, NopMetrics())
  59. err := appConns.Start()
  60. require.NoError(t, err)
  61. t.Cleanup(func() {
  62. if err := appConns.Stop(); err != nil {
  63. t.Error(err)
  64. }
  65. })
  66. // simulate failure
  67. close(quitCh)
  68. select {
  69. case <-ok:
  70. t.Log("SIGTERM successfully received")
  71. case <-time.After(5 * time.Second):
  72. t.Fatal("expected process to receive SIGTERM signal")
  73. }
  74. }