You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

286 lines
11 KiB

6 years ago
6 years ago
internal/proxy: add initial set of abci metrics (#7115) This PR adds an initial set of metrics for use ABCI. The initial metrics enable the calculation of timing histograms and call counts for each of the ABCI methods. The metrics are also labeled as either 'sync' or 'async' to determine if the method call was performed using ABCI's `*Async` methods. An example of these metrics is included here for reference: ``` tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.0001"} 0 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.0004"} 5 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.002"} 12 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.009"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.02"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.1"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.65"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="2"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="6"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="25"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="+Inf"} 13 tendermint_abci_connection_method_timing_sum{chain_id="ci",method="commit",type="sync"} 0.007802058000000001 tendermint_abci_connection_method_timing_count{chain_id="ci",method="commit",type="sync"} 13 ``` These metrics can easily be graphed using prometheus's `histogram_quantile(...)` method to pick out a particular quantile to graph or examine. I chose buckets that were somewhat of an estimate of expected range of times for ABCI operations. They start at .0001 seconds and range to 25 seconds. The hope is that this range captures enough possible times to be useful for us and operators.
3 years ago
2 years ago
internal/proxy: add initial set of abci metrics (#7115) This PR adds an initial set of metrics for use ABCI. The initial metrics enable the calculation of timing histograms and call counts for each of the ABCI methods. The metrics are also labeled as either 'sync' or 'async' to determine if the method call was performed using ABCI's `*Async` methods. An example of these metrics is included here for reference: ``` tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.0001"} 0 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.0004"} 5 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.002"} 12 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.009"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.02"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.1"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.65"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="2"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="6"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="25"} 13 tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="+Inf"} 13 tendermint_abci_connection_method_timing_sum{chain_id="ci",method="commit",type="sync"} 0.007802058000000001 tendermint_abci_connection_method_timing_count{chain_id="ci",method="commit",type="sync"} 13 ``` These metrics can easily be graphed using prometheus's `histogram_quantile(...)` method to pick out a particular quantile to graph or examine. I chose buckets that were somewhat of an estimate of expected range of times for ABCI operations. They start at .0001 seconds and range to 25 seconds. The hope is that this range captures enough possible times to be useful for us and operators.
3 years ago
2 years ago
2 years ago
  1. package consensus
  2. import (
  3. "github.com/go-kit/kit/metrics"
  4. "github.com/go-kit/kit/metrics/discard"
  5. "github.com/tendermint/tendermint/types"
  6. prometheus "github.com/go-kit/kit/metrics/prometheus"
  7. stdprometheus "github.com/prometheus/client_golang/prometheus"
  8. )
  9. const (
  10. // MetricsSubsystem is a subsystem shared by all metrics exposed by this
  11. // package.
  12. MetricsSubsystem = "consensus"
  13. )
  14. // Metrics contains metrics exposed by this package.
  15. type Metrics struct {
  16. // Height of the chain.
  17. Height metrics.Gauge
  18. // ValidatorLastSignedHeight of a validator.
  19. ValidatorLastSignedHeight metrics.Gauge
  20. // Number of rounds.
  21. Rounds metrics.Gauge
  22. // Number of validators.
  23. Validators metrics.Gauge
  24. // Total power of all validators.
  25. ValidatorsPower metrics.Gauge
  26. // Power of a validator.
  27. ValidatorPower metrics.Gauge
  28. // Amount of blocks missed by a validator.
  29. ValidatorMissedBlocks metrics.Gauge
  30. // Number of validators who did not sign.
  31. MissingValidators metrics.Gauge
  32. // Total power of the missing validators.
  33. MissingValidatorsPower metrics.Gauge
  34. // Number of validators who tried to double sign.
  35. ByzantineValidators metrics.Gauge
  36. // Total power of the byzantine validators.
  37. ByzantineValidatorsPower metrics.Gauge
  38. // Time between this and the last block.
  39. BlockIntervalSeconds metrics.Histogram
  40. // Number of transactions.
  41. NumTxs metrics.Gauge
  42. // Size of the block.
  43. BlockSizeBytes metrics.Histogram
  44. // Total number of transactions.
  45. TotalTxs metrics.Gauge
  46. // The latest block height.
  47. CommittedHeight metrics.Gauge
  48. // Whether or not a node is block syncing. 1 if yes, 0 if no.
  49. BlockSyncing metrics.Gauge
  50. // Whether or not a node is state syncing. 1 if yes, 0 if no.
  51. StateSyncing metrics.Gauge
  52. // Number of blockparts transmitted by peer.
  53. BlockParts metrics.Counter
  54. // Histogram of time taken per step annotated with reason that the step proceeded.
  55. StepTime metrics.Histogram
  56. // QuroumPrevoteMessageDelay is the interval in seconds between the proposal
  57. // timestamp and the timestamp of the earliest prevote that achieved a quorum
  58. // during the prevote step.
  59. //
  60. // To compute it, sum the voting power over each prevote received, in increasing
  61. // order of timestamp. The timestamp of the first prevote to increase the sum to
  62. // be above 2/3 of the total voting power of the network defines the endpoint
  63. // the endpoint of the interval. Subtract the proposal timestamp from this endpoint
  64. // to obtain the quorum delay.
  65. QuorumPrevoteMessageDelay metrics.Gauge
  66. // FullPrevoteMessageDelay is the interval in seconds between the proposal
  67. // timestamp and the timestamp of the latest prevote in a round where 100%
  68. // of the voting power on the network issued prevotes.
  69. FullPrevoteMessageDelay metrics.Gauge
  70. // ProposalTimestampDifference is the difference between the timestamp in
  71. // the proposal message and the local time of the validator at the time
  72. // that the validator received the message.
  73. ProposalTimestampDifference metrics.Histogram
  74. }
  75. // PrometheusMetrics returns Metrics build using Prometheus client library.
  76. // Optionally, labels can be provided along with their values ("foo",
  77. // "fooValue").
  78. func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
  79. labels := []string{}
  80. for i := 0; i < len(labelsAndValues); i += 2 {
  81. labels = append(labels, labelsAndValues[i])
  82. }
  83. return &Metrics{
  84. Height: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
  85. Namespace: namespace,
  86. Subsystem: MetricsSubsystem,
  87. Name: "height",
  88. Help: "Height of the chain.",
  89. }, labels).With(labelsAndValues...),
  90. Rounds: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
  91. Namespace: namespace,
  92. Subsystem: MetricsSubsystem,
  93. Name: "rounds",
  94. Help: "Number of rounds.",
  95. }, labels).With(labelsAndValues...),
  96. Validators: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
  97. Namespace: namespace,
  98. Subsystem: MetricsSubsystem,
  99. Name: "validators",
  100. Help: "Number of validators.",
  101. }, labels).With(labelsAndValues...),
  102. ValidatorLastSignedHeight: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
  103. Namespace: namespace,
  104. Subsystem: MetricsSubsystem,
  105. Name: "validator_last_signed_height",
  106. Help: "Last signed height for a validator",
  107. }, append(labels, "validator_address")).With(labelsAndValues...),
  108. ValidatorMissedBlocks: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
  109. Namespace: namespace,
  110. Subsystem: MetricsSubsystem,
  111. Name: "validator_missed_blocks",
  112. Help: "Total missed blocks for a validator",
  113. }, append(labels, "validator_address")).With(labelsAndValues...),
  114. ValidatorsPower: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
  115. Namespace: namespace,
  116. Subsystem: MetricsSubsystem,
  117. Name: "validators_power",
  118. Help: "Total power of all validators.",
  119. }, labels).With(labelsAndValues...),
  120. ValidatorPower: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
  121. Namespace: namespace,
  122. Subsystem: MetricsSubsystem,
  123. Name: "validator_power",
  124. Help: "Power of a validator",
  125. }, append(labels, "validator_address")).With(labelsAndValues...),
  126. MissingValidators: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
  127. Namespace: namespace,
  128. Subsystem: MetricsSubsystem,
  129. Name: "missing_validators",
  130. Help: "Number of validators who did not sign.",
  131. }, labels).With(labelsAndValues...),
  132. MissingValidatorsPower: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
  133. Namespace: namespace,
  134. Subsystem: MetricsSubsystem,
  135. Name: "missing_validators_power",
  136. Help: "Total power of the missing validators.",
  137. }, labels).With(labelsAndValues...),
  138. ByzantineValidators: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
  139. Namespace: namespace,
  140. Subsystem: MetricsSubsystem,
  141. Name: "byzantine_validators",
  142. Help: "Number of validators who tried to double sign.",
  143. }, labels).With(labelsAndValues...),
  144. ByzantineValidatorsPower: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
  145. Namespace: namespace,
  146. Subsystem: MetricsSubsystem,
  147. Name: "byzantine_validators_power",
  148. Help: "Total power of the byzantine validators.",
  149. }, labels).With(labelsAndValues...),
  150. BlockIntervalSeconds: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
  151. Namespace: namespace,
  152. Subsystem: MetricsSubsystem,
  153. Name: "block_interval_seconds",
  154. Help: "Time between this and the last block.",
  155. }, labels).With(labelsAndValues...),
  156. NumTxs: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
  157. Namespace: namespace,
  158. Subsystem: MetricsSubsystem,
  159. Name: "num_txs",
  160. Help: "Number of transactions.",
  161. }, labels).With(labelsAndValues...),
  162. BlockSizeBytes: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
  163. Namespace: namespace,
  164. Subsystem: MetricsSubsystem,
  165. Name: "block_size_bytes",
  166. Help: "Size of the block.",
  167. }, labels).With(labelsAndValues...),
  168. TotalTxs: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
  169. Namespace: namespace,
  170. Subsystem: MetricsSubsystem,
  171. Name: "total_txs",
  172. Help: "Total number of transactions.",
  173. }, labels).With(labelsAndValues...),
  174. CommittedHeight: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
  175. Namespace: namespace,
  176. Subsystem: MetricsSubsystem,
  177. Name: "latest_block_height",
  178. Help: "The latest block height.",
  179. }, labels).With(labelsAndValues...),
  180. BlockSyncing: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
  181. Namespace: namespace,
  182. Subsystem: MetricsSubsystem,
  183. Name: "block_syncing",
  184. Help: "Whether or not a node is block syncing. 1 if yes, 0 if no.",
  185. }, labels).With(labelsAndValues...),
  186. StateSyncing: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
  187. Namespace: namespace,
  188. Subsystem: MetricsSubsystem,
  189. Name: "state_syncing",
  190. Help: "Whether or not a node is state syncing. 1 if yes, 0 if no.",
  191. }, labels).With(labelsAndValues...),
  192. BlockParts: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
  193. Namespace: namespace,
  194. Subsystem: MetricsSubsystem,
  195. Name: "block_parts",
  196. Help: "Number of blockparts transmitted by peer.",
  197. }, append(labels, "peer_id")).With(labelsAndValues...),
  198. StepTime: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
  199. Namespace: namespace,
  200. Subsystem: MetricsSubsystem,
  201. Name: "step_time",
  202. Help: "Time spent per step.",
  203. }, append(labels, "step", "reason")).With(labelsAndValues...),
  204. QuorumPrevoteMessageDelay: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
  205. Namespace: namespace,
  206. Subsystem: MetricsSubsystem,
  207. Name: "quorum_prevote_message_delay",
  208. Help: "Difference in seconds between the proposal timestamp and the timestamp " +
  209. "of the latest prevote that achieved a quorum in the prevote step.",
  210. }, append(labels, "proposer_address")).With(labelsAndValues...),
  211. FullPrevoteMessageDelay: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
  212. Namespace: namespace,
  213. Subsystem: MetricsSubsystem,
  214. Name: "full_prevote_message_delay",
  215. Help: "Difference in seconds between the proposal timestamp and the timestamp " +
  216. "of the latest prevote that achieved 100% of the voting power in the prevote step.",
  217. }, append(labels, "proposer_address")).With(labelsAndValues...),
  218. ProposalTimestampDifference: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
  219. Namespace: namespace,
  220. Subsystem: MetricsSubsystem,
  221. Name: "proposal_timestamp_difference",
  222. Help: "Difference in seconds between the timestamp in the proposal " +
  223. "message and the local time when the message was received. " +
  224. "Only calculated when a new block is proposed.",
  225. Buckets: []float64{-10, -.5, -.025, 0, .1, .5, 1, 1.5, 2, 10},
  226. }, append(labels, "is_timely")).With(labelsAndValues...),
  227. }
  228. }
  229. // NopMetrics returns no-op Metrics.
  230. func NopMetrics() *Metrics {
  231. return &Metrics{
  232. Height: discard.NewGauge(),
  233. ValidatorLastSignedHeight: discard.NewGauge(),
  234. Rounds: discard.NewGauge(),
  235. Validators: discard.NewGauge(),
  236. ValidatorsPower: discard.NewGauge(),
  237. ValidatorPower: discard.NewGauge(),
  238. ValidatorMissedBlocks: discard.NewGauge(),
  239. MissingValidators: discard.NewGauge(),
  240. MissingValidatorsPower: discard.NewGauge(),
  241. ByzantineValidators: discard.NewGauge(),
  242. ByzantineValidatorsPower: discard.NewGauge(),
  243. BlockIntervalSeconds: discard.NewHistogram(),
  244. NumTxs: discard.NewGauge(),
  245. BlockSizeBytes: discard.NewHistogram(),
  246. TotalTxs: discard.NewGauge(),
  247. CommittedHeight: discard.NewGauge(),
  248. BlockSyncing: discard.NewGauge(),
  249. StateSyncing: discard.NewGauge(),
  250. BlockParts: discard.NewCounter(),
  251. QuorumPrevoteMessageDelay: discard.NewGauge(),
  252. FullPrevoteMessageDelay: discard.NewGauge(),
  253. ProposalTimestampDifference: discard.NewHistogram(),
  254. }
  255. }
  256. // RecordConsMetrics uses for recording the block related metrics during fast-sync.
  257. func (m *Metrics) RecordConsMetrics(block *types.Block) {
  258. m.NumTxs.Set(float64(len(block.Data.Txs)))
  259. m.TotalTxs.Add(float64(len(block.Data.Txs)))
  260. m.BlockSizeBytes.Observe(float64(block.Size()))
  261. m.CommittedHeight.Set(float64(block.Height))
  262. }