From 4d03d7516bf1648c4862fb5c5921a9a54b77b7f8 Mon Sep 17 00:00:00 2001
From: Sam Kleinman <garen@tychoish.com>
Date: Wed, 14 Apr 2021 10:12:46 -0400
Subject: [PATCH] test/e2e: produce structured reporting from benchmarks
 (#6343)

---
 test/e2e/runner/benchmark.go | 42 +++++++++++++++++++++++++++++++++++-
 1 file changed, 41 insertions(+), 1 deletion(-)

diff --git a/test/e2e/runner/benchmark.go b/test/e2e/runner/benchmark.go
index 545fb773d..74d2491f5 100644
--- a/test/e2e/runner/benchmark.go
+++ b/test/e2e/runner/benchmark.go
@@ -2,8 +2,10 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"math"
+	"path/filepath"
 	"time"
 
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
@@ -26,7 +28,7 @@ func Benchmark(testnet *e2e.Testnet, benchmarkLength int64) error {
 	}
 
 	logger.Info("Beginning benchmark period...", "height", block.Height)
-
+	startAt := time.Now()
 	// wait for the length of the benchmark period in blocks to pass. We allow 5 seconds for each block
 	// which should be sufficient.
 	waitingTime := time.Duration(benchmarkLength*5) * time.Second
@@ -34,6 +36,7 @@ func Benchmark(testnet *e2e.Testnet, benchmarkLength int64) error {
 	if err != nil {
 		return err
 	}
+	dur := time.Since(startAt)
 
 	logger.Info("Ending benchmark period", "height", endHeight)
 
@@ -46,18 +49,34 @@ func Benchmark(testnet *e2e.Testnet, benchmarkLength int64) error {
 	// slice into time intervals and collate data
 	timeIntervals := splitIntoBlockIntervals(blocks)
 	testnetStats := extractTestnetStats(timeIntervals)
+
+	// populate data
+	testnetStats.populateTxns(blocks)
+	testnetStats.totalTime = dur
+	testnetStats.benchmarkLength = benchmarkLength
 	testnetStats.startHeight = blocks[0].Header.Height
 	testnetStats.endHeight = blocks[len(blocks)-1].Header.Height
 
 	// print and return
 	logger.Info(testnetStats.String())
+	logger.Info(testnetStats.getReportJSON(testnet))
 	return nil
 }
 
+func (t *testnetStats) populateTxns(blocks []*types.BlockMeta) {
+	t.numtxns = 0
+	for _, b := range blocks {
+		t.numtxns += int64(b.NumTxs)
+	}
+}
+
 type testnetStats struct {
 	startHeight int64
 	endHeight   int64
 
+	benchmarkLength int64
+	numtxns         int64
+	totalTime       time.Duration
 	// average time to produce a block
 	mean time.Duration
 	// standard deviation of block production
@@ -68,6 +87,27 @@ type testnetStats struct {
 	min time.Duration
 }
 
+func (t *testnetStats) getReportJSON(net *e2e.Testnet) string {
+	jsn, err := json.Marshal(map[string]interface{}{
+		"case":   filepath.Base(net.File),
+		"blocks": t.endHeight - t.startHeight,
+		"stddev": t.std,
+		"mean":   t.mean.Seconds(),
+		"max":    t.max.Seconds(),
+		"min":    t.min.Seconds(),
+		"size":   len(net.Nodes),
+		"txns":   t.numtxns,
+		"dur":    t.totalTime.Seconds(),
+		"length": t.benchmarkLength,
+	})
+
+	if err != nil {
+		return ""
+	}
+
+	return string(jsn)
+}
+
 func (t *testnetStats) String() string {
 	return fmt.Sprintf(`Benchmarked from height %v to %v
 	Mean Block Interval: %v