From 50ea68a42612b8f6c66fa9aa109b3937cc57b2d1 Mon Sep 17 00:00:00 2001
From: ValarDragon <dojha12@gmail.com>
Date: Tue, 10 Jul 2018 21:57:22 -0700
Subject: [PATCH 1/9] tools/tm-bench: Don't count the first block if its empty

---
 CHANGELOG.md              |  1 +
 tools/tm-bench/Gopkg.lock | 11 +++++------
 tools/tm-bench/Gopkg.toml |  5 +++++
 tools/tm-bench/main.go    | 13 ++++++++++++-
 4 files changed, 23 insertions(+), 7 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0f15507c0..1aa3c7cd3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,6 +16,7 @@ BUG FIXES
     - NOTE: this is only for URI requests. JSONRPC requests and all responses
       will use quoted integers (the proto3 JSON standard).
 - [consensus] Fix halt on shutdown
+- [tm_bench] Ignore the first block if we didn't start sending txs by then
 
 ## 0.22.1
 
diff --git a/tools/tm-bench/Gopkg.lock b/tools/tm-bench/Gopkg.lock
index 175acb3a6..aa1d819ca 100644
--- a/tools/tm-bench/Gopkg.lock
+++ b/tools/tm-bench/Gopkg.lock
@@ -119,7 +119,7 @@
     "prometheus",
     "prometheus/promhttp"
   ]
-  revision = "ae27198cdd90bf12cd134ad79d1366a6cf49f632"
+  revision = "ee1c9d7e23df7f011bdf6f12a5c9e7f0ae10a1fe"
 
 [[projects]]
   branch = "master"
@@ -237,7 +237,7 @@
     "types",
     "version"
   ]
-  revision = "9d81a74429e093f3167875e0145ad957874c77d1"
+  revision = "f5ad8ef8600c33532a16de0879ff6b9745bb394d"
 
 [[projects]]
   branch = "master"
@@ -268,7 +268,7 @@
     "netutil",
     "trace"
   ]
-  revision = "292b43bbf7cb8d35ddf40f8d5100ef3837cced3f"
+  revision = "039a4258aec0ad3c79b905677cceeab13b296a77"
 
 [[projects]]
   name = "golang.org/x/text"
@@ -292,10 +292,9 @@
   version = "v0.3.0"
 
 [[projects]]
-  branch = "master"
   name = "google.golang.org/genproto"
   packages = ["googleapis/rpc/status"]
-  revision = "e92b116572682a5b432ddd840aeaba2a559eeff1"
+  revision = "7fd901a49ba6a7f87732eb344f6e3c5b19d1b200"
 
 [[projects]]
   name = "google.golang.org/grpc"
@@ -330,6 +329,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "bc54a74ffdfc09872726fcf5c72b5df882269dc1cd949ac3fbeac9a554fc25c6"
+  inputs-digest = "5c21a60b80ac7d60f7be693de13f9fadb62226b502431bdb38fb9794a98c5b02"
   solver-name = "gps-cdcl"
   solver-version = 1
diff --git a/tools/tm-bench/Gopkg.toml b/tools/tm-bench/Gopkg.toml
index 18498cbbb..3b2dfa4ec 100644
--- a/tools/tm-bench/Gopkg.toml
+++ b/tools/tm-bench/Gopkg.toml
@@ -45,6 +45,11 @@
   name = "github.com/tendermint/tendermint"
   branch = "develop"
 
+# this got updated and broke, so locked to an old working commit ...
+[[override]]
+  name = "google.golang.org/genproto"
+  revision = "7fd901a49ba6a7f87732eb344f6e3c5b19d1b200"
+
 [prune]
   go-tests = true
   unused-packages = true
diff --git a/tools/tm-bench/main.go b/tools/tm-bench/main.go
index 0956e57ec..e93e62ace 100644
--- a/tools/tm-bench/main.go
+++ b/tools/tm-bench/main.go
@@ -189,6 +189,16 @@ func calculateStatistics(
 		offset = len(blockMetas)
 	}
 
+	// Ignore the first block created if there were no txs in it, since we likely didn't begin
+	// sending txs by then. The last index is the block that was already present when we started
+	// this process
+	if blockMetas[len(blockMetas)-2].Header.NumTxs == 0 {
+		if timeStart.Before(blockMetas[len(blockMetas)-2].Header.Time) {
+			timeStart = blockMetas[len(blockMetas)-2].Header.Time
+		}
+		blockMetas = blockMetas[:len(blockMetas)-2]
+	}
+
 	var (
 		numBlocksPerSec = make(map[int64]int64)
 		numTxsPerSec    = make(map[int64]int64)
@@ -201,7 +211,7 @@ func calculateStatistics(
 	}
 
 	// iterates from max height to min height
-	for _, blockMeta := range blockMetas {
+	for i, blockMeta := range blockMetas {
 		// check if block was created after timeStart
 		if blockMeta.Header.Time.Before(timeStart) {
 			break
@@ -218,6 +228,7 @@ func calculateStatistics(
 
 		// increase number of txs for that second
 		numTxsPerSec[sec] += blockMeta.Header.NumTxs
+		logger.Debug(fmt.Sprintf("%d txs in block %d, height %d", blockMeta.Header.NumTxs, i, blockMeta.Header.Height))
 	}
 
 	for _, n := range numBlocksPerSec {

From f04a0875465c74ad0e6ce35b92eb53eb9aeb716c Mon Sep 17 00:00:00 2001
From: ValarDragon <dojha12@gmail.com>
Date: Wed, 11 Jul 2018 21:54:46 -0700
Subject: [PATCH 2/9] Try melekes suggestion

---
 tools/tm-bench/main.go | 21 ++++++---------------
 1 file changed, 6 insertions(+), 15 deletions(-)

diff --git a/tools/tm-bench/main.go b/tools/tm-bench/main.go
index f3d15f967..a9151404a 100644
--- a/tools/tm-bench/main.go
+++ b/tools/tm-bench/main.go
@@ -42,7 +42,7 @@ func main() {
 		fmt.Println(`Tendermint blockchain benchmarking tool.
 
 Usage:
-	tm-bench [-c 1] [-T 10] [-r 1000] [endpoints] [-output-format <plain|json> [-broadcast-tx-method <async|sync|commit>]]
+	tm-bench [-c 1] [-T 10] [-r 1000] [-s 250] [endpoints] [-output-format <plain|json> [-broadcast-tx-method <async|sync|commit>]]
 
 Examples:
 	tm-bench localhost:26657`)
@@ -93,10 +93,6 @@ Examples:
 	)
 	logger.Info("Latest block height", "h", initialHeight)
 
-	// record time start
-	timeStart := time.Now()
-	logger.Info("Time started", "t", timeStart)
-
 	transacters := startTransacters(
 		endpoints,
 		connections,
@@ -104,6 +100,11 @@ Examples:
 		txSize,
 		"broadcast_tx_"+broadcastTxMethod,
 	)
+
+	// record time start
+	timeStart := time.Now()
+	logger.Info("Time last transacter started", "t", timeStart)
+
 	endTime := time.Duration(duration) * time.Second
 
 	<-time.After(endTime)
@@ -190,16 +191,6 @@ func calculateStatistics(
 		offset = len(blockMetas)
 	}
 
-	// Ignore the first block created if there were no txs in it, since we likely didn't begin
-	// sending txs by then. The last index is the block that was already present when we started
-	// this process
-	if blockMetas[len(blockMetas)-2].Header.NumTxs == 0 {
-		if timeStart.Before(blockMetas[len(blockMetas)-2].Header.Time) {
-			timeStart = blockMetas[len(blockMetas)-2].Header.Time
-		}
-		blockMetas = blockMetas[:len(blockMetas)-2]
-	}
-
 	var (
 		numBlocksPerSec = make(map[int64]int64)
 		numTxsPerSec    = make(map[int64]int64)

From e785d6851c8b24f8ed87052e5603fca320fd00d3 Mon Sep 17 00:00:00 2001
From: ValarDragon <dojha12@gmail.com>
Date: Wed, 11 Jul 2018 22:55:20 -0700
Subject: [PATCH 3/9] Fix getting the start time so the first block is no
 longer empty

---
 tools/tm-bench/README.md     | 23 +++++++++++++++++------
 tools/tm-bench/main.go       | 15 ++++++++++++++-
 tools/tm-bench/transacter.go | 11 +++++++----
 3 files changed, 38 insertions(+), 11 deletions(-)

diff --git a/tools/tm-bench/README.md b/tools/tm-bench/README.md
index 811141629..000f20f37 100644
--- a/tools/tm-bench/README.md
+++ b/tools/tm-bench/README.md
@@ -51,15 +51,26 @@ with the last command being in a seperate window.
 ## How stats are collected
 
 These stats are derived by having each connection send transactions at the
-specified rate (or as close as it can get) for the specified time. After the
-specified time, it iterates over all of the blocks that were created in that
-time. The average and stddev per second are computed based off of that, by
+specified rate (or as close as it can get) for the specified time.
+After the specified time, it iterates over all of the blocks that were created
+in that time.
+The average and stddev per second are computed based off of that, by
 grouping the data by second.
 
 To send transactions at the specified rate in each connection, we loop
-through the number of transactions. If its too slow, the loop stops at one second.
-If its too fast, we wait until the one second mark ends. The transactions per
-second stat is computed based off of what ends up in the block.
+through the number of transactions.
+If its too slow, the loop stops at one second.
+If its too fast, we wait until the one second mark ends.
+The transactions per second stat is computed based off of what ends up in the
+block.
+
+Note that there will be edge effects on the number of transactions in the first
+and last blocks.
+This is because transactions may start sending midway through when tendermint
+starts building the next block, so it only has half as much time to gather txs
+that tm-bench sends.
+Similarly the end of the duration will likely end mid-way through tendermint
+trying to build the next block.
 
 Each of the connections is handled via two separate goroutines. 
 
diff --git a/tools/tm-bench/main.go b/tools/tm-bench/main.go
index a9151404a..68b3bee16 100644
--- a/tools/tm-bench/main.go
+++ b/tools/tm-bench/main.go
@@ -101,7 +101,20 @@ Examples:
 		"broadcast_tx_"+broadcastTxMethod,
 	)
 
-	// record time start
+	// Wait until transacters have begun until we get the start time
+	for {
+		started := true
+		for _, t := range transacters {
+			for i := 0; i < t.Connections; i++ {
+				if !t.connsStarted[i] && !t.connsBroken[i] {
+					started = false
+				}
+			}
+		}
+		if started {
+			break
+		}
+	}
 	timeStart := time.Now()
 	logger.Info("Time last transacter started", "t", timeStart)
 
diff --git a/tools/tm-bench/transacter.go b/tools/tm-bench/transacter.go
index de408136d..f89c55b7b 100644
--- a/tools/tm-bench/transacter.go
+++ b/tools/tm-bench/transacter.go
@@ -34,10 +34,11 @@ type transacter struct {
 	Connections       int
 	BroadcastTxMethod string
 
-	conns       []*websocket.Conn
-	connsBroken []bool
-	wg          sync.WaitGroup
-	stopped     bool
+	conns        []*websocket.Conn
+	connsStarted []bool
+	connsBroken  []bool
+	wg           sync.WaitGroup
+	stopped      bool
 
 	logger log.Logger
 }
@@ -50,6 +51,7 @@ func newTransacter(target string, connections, rate int, size int, broadcastTxMe
 		Connections:       connections,
 		BroadcastTxMethod: broadcastTxMethod,
 		conns:             make([]*websocket.Conn, connections),
+		connsStarted:      make([]bool, connections),
 		connsBroken:       make([]bool, connections),
 		logger:            log.NewNopLogger(),
 	}
@@ -156,6 +158,7 @@ func (t *transacter) sendLoop(connIndex int) {
 			startTime := time.Now()
 			endTime := startTime.Add(time.Second)
 			numTxSent := t.Rate
+			t.connsStarted[connIndex] = true
 
 			for i := 0; i < t.Rate; i++ {
 				// each transaction embeds connection index, tx number and hash of the hostname

From 7a7f5782bc668ccb54e590eccb04e6add40829c2 Mon Sep 17 00:00:00 2001
From: ValarDragon <dojha12@gmail.com>
Date: Wed, 11 Jul 2018 22:59:13 -0700
Subject: [PATCH 4/9] Fix changelog entry

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5f057d640..5d7490aeb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,7 +16,7 @@ BUG FIXES
     - NOTE: this is only for URI requests. JSONRPC requests and all responses
       will use quoted integers (the proto3 JSON standard).
 - [consensus] Fix halt on shutdown
-- [tm_bench] Ignore the first block if we didn't start sending txs by then
+- [tm_bench] Fix method of computing start time
 
 ## 0.22.1
 

From bd050c1d0335ca71cd5ea1aeacd6247262083da8 Mon Sep 17 00:00:00 2001
From: ValarDragon <dojha12@gmail.com>
Date: Thu, 12 Jul 2018 12:42:26 -0700
Subject: [PATCH 5/9] Make code smell better

---
 tools/tm-bench/main.go       | 18 +++++-------------
 tools/tm-bench/transacter.go | 16 ++++++++++++++++
 2 files changed, 21 insertions(+), 13 deletions(-)

diff --git a/tools/tm-bench/main.go b/tools/tm-bench/main.go
index 68b3bee16..f8a979fef 100644
--- a/tools/tm-bench/main.go
+++ b/tools/tm-bench/main.go
@@ -102,19 +102,6 @@ Examples:
 	)
 
 	// Wait until transacters have begun until we get the start time
-	for {
-		started := true
-		for _, t := range transacters {
-			for i := 0; i < t.Connections; i++ {
-				if !t.connsStarted[i] && !t.connsBroken[i] {
-					started = false
-				}
-			}
-		}
-		if started {
-			break
-		}
-	}
 	timeStart := time.Now()
 	logger.Info("Time last transacter started", "t", timeStart)
 
@@ -270,6 +257,11 @@ func startTransacters(
 		transacters[i] = t
 	}
 
+	// Wait until all transacters have started firing txs
+	for _, t := range transacters {
+		t.WaitUntilAllConnectionsStartedFiringTxs()
+	}
+
 	return transacters
 }
 
diff --git a/tools/tm-bench/transacter.go b/tools/tm-bench/transacter.go
index f89c55b7b..b11c174fa 100644
--- a/tools/tm-bench/transacter.go
+++ b/tools/tm-bench/transacter.go
@@ -86,6 +86,22 @@ func (t *transacter) Start() error {
 	return nil
 }
 
+// WaitUntilAllConnectionsStartedFiringTxs waits until all of this
+// transacters connections have begun sending txs at the specified rate
+func (t *transacter) WaitUntilAllConnectionsStartedFiringTxs() {
+	for {
+		started := true
+		for i := 0; i < t.Connections; i++ {
+			if !t.connsStarted[i] && !t.connsBroken[i] {
+				started = false
+			}
+		}
+		if started {
+			break
+		}
+	}
+}
+
 // Stop closes the connections.
 func (t *transacter) Stop() {
 	t.stopped = true

From 75c9303c68864625866f580945533154b9be3dab Mon Sep 17 00:00:00 2001
From: ValarDragon <dojha12@gmail.com>
Date: Thu, 12 Jul 2018 12:59:40 -0700
Subject: [PATCH 6/9] tools/tmbench: Fix the end time being used for statistics
 calculation

Previously we were using the time at which all connections closed in statistics, not
the time after {duration} seconds.
---
 CHANGELOG.md           |  2 +-
 tools/tm-bench/main.go | 20 +++++++++++---------
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5d7490aeb..698b8226a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,7 +16,7 @@ BUG FIXES
     - NOTE: this is only for URI requests. JSONRPC requests and all responses
       will use quoted integers (the proto3 JSON standard).
 - [consensus] Fix halt on shutdown
-- [tm_bench] Fix method of computing start time
+- [tm_bench] Fix method of computing start time, and end time
 
 ## 0.22.1
 
diff --git a/tools/tm-bench/main.go b/tools/tm-bench/main.go
index f8a979fef..1bba389f5 100644
--- a/tools/tm-bench/main.go
+++ b/tools/tm-bench/main.go
@@ -25,13 +25,13 @@ type statistics struct {
 }
 
 func main() {
-	var duration, txsRate, connections, txSize int
+	var durationInt, txsRate, connections, txSize int
 	var verbose bool
 	var outputFormat, broadcastTxMethod string
 
 	flagSet := flag.NewFlagSet("tm-bench", flag.ExitOnError)
 	flagSet.IntVar(&connections, "c", 1, "Connections to keep open per endpoint")
-	flagSet.IntVar(&duration, "T", 10, "Exit after the specified amount of time in seconds")
+	flagSet.IntVar(&durationInt, "T", 10, "Exit after the specified amount of time in seconds")
 	flagSet.IntVar(&txsRate, "r", 1000, "Txs per second to send in a connection")
 	flagSet.IntVar(&txSize, "s", 250, "The size of a transaction in bytes.")
 	flagSet.StringVar(&outputFormat, "output-format", "plain", "Output format: plain or json")
@@ -73,7 +73,7 @@ Examples:
 		}
 		logger = log.NewTMLoggerWithColorFn(log.NewSyncWriter(os.Stdout), colorFn)
 
-		fmt.Printf("Running %ds test @ %s\n", duration, flagSet.Arg(0))
+		fmt.Printf("Running %ds test @ %s\n", durationInt, flagSet.Arg(0))
 	}
 
 	if broadcastTxMethod != "async" &&
@@ -105,9 +105,12 @@ Examples:
 	timeStart := time.Now()
 	logger.Info("Time last transacter started", "t", timeStart)
 
-	endTime := time.Duration(duration) * time.Second
+	duration := time.Duration(durationInt) * time.Second
 
-	<-time.After(endTime)
+	timeEnd := timeStart.Add(duration)
+	logger.Info("End time for calculation", "t", timeEnd)
+
+	<-time.After(duration)
 	for i, t := range transacters {
 		t.Stop()
 		numCrashes := countCrashes(t.connsBroken)
@@ -116,15 +119,14 @@ Examples:
 		}
 	}
 
-	timeStop := time.Now()
-	logger.Info("Time stopped", "t", timeStop)
+	logger.Debug("Time all transacters stopped", "t", time.Now())
 
 	stats, err := calculateStatistics(
 		client,
 		initialHeight,
 		timeStart,
-		timeStop,
-		duration,
+		timeEnd,
+		durationInt,
 	)
 	if err != nil {
 		fmt.Fprintln(os.Stderr, err)

From e46ae158593c558be38effad64a3e3cf3f97d4c6 Mon Sep 17 00:00:00 2001
From: ValarDragon <dojha12@gmail.com>
Date: Thu, 12 Jul 2018 15:37:46 -0700
Subject: [PATCH 7/9] Use waitgroups for starting up

---
 tools/tm-bench/main.go       | 22 ++++++++--------
 tools/tm-bench/transacter.go | 50 +++++++++++++++++-------------------
 2 files changed, 35 insertions(+), 37 deletions(-)

diff --git a/tools/tm-bench/main.go b/tools/tm-bench/main.go
index f8a979fef..bad49cba5 100644
--- a/tools/tm-bench/main.go
+++ b/tools/tm-bench/main.go
@@ -7,6 +7,7 @@ import (
 	"math"
 	"os"
 	"strings"
+	"sync"
 	"text/tabwriter"
 	"time"
 
@@ -247,20 +248,21 @@ func startTransacters(
 ) []*transacter {
 	transacters := make([]*transacter, len(endpoints))
 
+	wg := sync.WaitGroup{}
+	wg.Add(len(endpoints))
 	for i, e := range endpoints {
 		t := newTransacter(e, connections, txsRate, txSize, broadcastTxMethod)
 		t.SetLogger(logger)
-		if err := t.Start(); err != nil {
-			fmt.Fprintln(os.Stderr, err)
-			os.Exit(1)
-		}
-		transacters[i] = t
-	}
-
-	// Wait until all transacters have started firing txs
-	for _, t := range transacters {
-		t.WaitUntilAllConnectionsStartedFiringTxs()
+		go func(i int) {
+			defer wg.Done()
+			if err := t.Start(); err != nil {
+				fmt.Fprintln(os.Stderr, err)
+				os.Exit(1)
+			}
+			transacters[i] = t
+		}(i)
 	}
+	wg.Wait()
 
 	return transacters
 }
diff --git a/tools/tm-bench/transacter.go b/tools/tm-bench/transacter.go
index b11c174fa..2834727b5 100644
--- a/tools/tm-bench/transacter.go
+++ b/tools/tm-bench/transacter.go
@@ -34,11 +34,11 @@ type transacter struct {
 	Connections       int
 	BroadcastTxMethod string
 
-	conns        []*websocket.Conn
-	connsStarted []bool
-	connsBroken  []bool
-	wg           sync.WaitGroup
-	stopped      bool
+	conns       []*websocket.Conn
+	connsBroken []bool
+	startingWg  sync.WaitGroup
+	endingWg    sync.WaitGroup
+	stopped     bool
 
 	logger log.Logger
 }
@@ -51,7 +51,6 @@ func newTransacter(target string, connections, rate int, size int, broadcastTxMe
 		Connections:       connections,
 		BroadcastTxMethod: broadcastTxMethod,
 		conns:             make([]*websocket.Conn, connections),
-		connsStarted:      make([]bool, connections),
 		connsBroken:       make([]bool, connections),
 		logger:            log.NewNopLogger(),
 	}
@@ -77,35 +76,22 @@ func (t *transacter) Start() error {
 		t.conns[i] = c
 	}
 
-	t.wg.Add(2 * t.Connections)
+	t.startingWg.Add(t.Connections)
+	t.endingWg.Add(2 * t.Connections)
 	for i := 0; i < t.Connections; i++ {
 		go t.sendLoop(i)
 		go t.receiveLoop(i)
 	}
 
-	return nil
-}
+	t.startingWg.Wait()
 
-// WaitUntilAllConnectionsStartedFiringTxs waits until all of this
-// transacters connections have begun sending txs at the specified rate
-func (t *transacter) WaitUntilAllConnectionsStartedFiringTxs() {
-	for {
-		started := true
-		for i := 0; i < t.Connections; i++ {
-			if !t.connsStarted[i] && !t.connsBroken[i] {
-				started = false
-			}
-		}
-		if started {
-			break
-		}
-	}
+	return nil
 }
 
 // Stop closes the connections.
 func (t *transacter) Stop() {
 	t.stopped = true
-	t.wg.Wait()
+	t.endingWg.Wait()
 	for _, c := range t.conns {
 		c.Close()
 	}
@@ -115,7 +101,7 @@ func (t *transacter) Stop() {
 // `broadcast_tx_async`).
 func (t *transacter) receiveLoop(connIndex int) {
 	c := t.conns[connIndex]
-	defer t.wg.Done()
+	defer t.endingWg.Done()
 	for {
 		_, _, err := c.ReadMessage()
 		if err != nil {
@@ -136,6 +122,13 @@ func (t *transacter) receiveLoop(connIndex int) {
 
 // sendLoop generates transactions at a given rate.
 func (t *transacter) sendLoop(connIndex int) {
+	started := false
+	// Close the starting waitgroup, in the event that this fails to start
+	defer func() {
+		if !started {
+			t.startingWg.Done()
+		}
+	}()
 	c := t.conns[connIndex]
 
 	c.SetPingHandler(func(message string) error {
@@ -157,7 +150,7 @@ func (t *transacter) sendLoop(connIndex int) {
 	defer func() {
 		pingsTicker.Stop()
 		txsTicker.Stop()
-		t.wg.Done()
+		t.endingWg.Done()
 	}()
 
 	// hash of the host name is a part of each tx
@@ -174,7 +167,10 @@ func (t *transacter) sendLoop(connIndex int) {
 			startTime := time.Now()
 			endTime := startTime.Add(time.Second)
 			numTxSent := t.Rate
-			t.connsStarted[connIndex] = true
+			if !started {
+				t.startingWg.Done()
+				started = true
+			}
 
 			for i := 0; i < t.Rate; i++ {
 				// each transaction embeds connection index, tx number and hash of the hostname

From 1dbe7b7e687c343a59e53b51ca28f2137301f102 Mon Sep 17 00:00:00 2001
From: ValarDragon <dojha12@gmail.com>
Date: Thu, 12 Jul 2018 23:43:56 -0700
Subject: [PATCH 8/9] tools/tmbench: Improve accuracy with large tx sizes.

At larger tx sizes (e.g. > 10000) we were spending non-neglible
amounts of time in tx creation, due to making the final bytes random.
The slower the send loop, the less accurate it is at measuring the time
tendermint took. (As we can't reach the promised contract of the given rate)

There really isn't much need for that randomness, so this PR makes it
such that only the txNumber gets bumped between txs from the same
connection, thereby improving sendloop speed and accuracy.
---
 tools/tm-bench/bench_test.go      |  18 ------
 tools/tm-bench/transacter.go      |  31 +++++++--
 tools/tm-bench/transacter_test.go | 104 ++++++++++++++++++++++++++++++
 3 files changed, 128 insertions(+), 25 deletions(-)
 delete mode 100644 tools/tm-bench/bench_test.go
 create mode 100644 tools/tm-bench/transacter_test.go

diff --git a/tools/tm-bench/bench_test.go b/tools/tm-bench/bench_test.go
deleted file mode 100644
index 9eaf0f7ea..000000000
--- a/tools/tm-bench/bench_test.go
+++ /dev/null
@@ -1,18 +0,0 @@
-package main
-
-import (
-	"testing"
-	"time"
-)
-
-func BenchmarkTimingPerTx(b *testing.B) {
-	startTime := time.Now()
-	endTime := startTime.Add(time.Second)
-	for i := 0; i < b.N; i++ {
-		if i%20 == 0 {
-			if time.Now().After(endTime) {
-				continue
-			}
-		}
-	}
-}
diff --git a/tools/tm-bench/transacter.go b/tools/tm-bench/transacter.go
index 2834727b5..36cc761e5 100644
--- a/tools/tm-bench/transacter.go
+++ b/tools/tm-bench/transacter.go
@@ -160,6 +160,11 @@ func (t *transacter) sendLoop(connIndex int) {
 		hostname = "127.0.0.1"
 	}
 	hostnameHash = md5.Sum([]byte(hostname))
+	// each transaction embeds connection index, tx number and hash of the hostname
+	// we update the tx number between successive txs
+	tx := generateTx(connIndex, txNumber, t.Size, hostnameHash)
+	txHex := make([]byte, len(tx)*2)
+	hex.Encode(txHex, tx)
 
 	for {
 		select {
@@ -172,17 +177,18 @@ func (t *transacter) sendLoop(connIndex int) {
 				started = true
 			}
 
+			now := time.Now()
 			for i := 0; i < t.Rate; i++ {
-				// each transaction embeds connection index, tx number and hash of the hostname
-				tx := generateTx(connIndex, txNumber, t.Size, hostnameHash)
-				paramsJSON, err := json.Marshal(map[string]interface{}{"tx": hex.EncodeToString(tx)})
+				// update tx number of the tx, and the corresponding hex
+				updateTx(tx, txHex, txNumber)
+				paramsJSON, err := json.Marshal(map[string]interface{}{"tx": txHex})
 				if err != nil {
 					fmt.Printf("failed to encode params: %v\n", err)
 					os.Exit(1)
 				}
 				rawParamsJSON := json.RawMessage(paramsJSON)
 
-				c.SetWriteDeadline(time.Now().Add(sendTimeout))
+				c.SetWriteDeadline(now.Add(sendTimeout))
 				err = c.WriteJSON(rpctypes.RPCRequest{
 					JSONRPC: "2.0",
 					ID:      "tm-bench",
@@ -197,9 +203,10 @@ func (t *transacter) sendLoop(connIndex int) {
 					return
 				}
 
-				// Time added here is 7.13 ns/op, not significant enough to worry about
-				if i%20 == 0 {
-					if time.Now().After(endTime) {
+				// cache the time.Now() reads to save time.
+				if i%5 == 0 {
+					now = time.Now()
+					if now.After(endTime) {
 						// Plus one accounts for sending this tx
 						numTxSent = i + 1
 						break
@@ -265,3 +272,13 @@ func generateTx(connIndex int, txNumber int, txSize int, hostnameHash [md5.Size]
 
 	return tx
 }
+
+// warning, mutates input byte slice
+func updateTx(tx []byte, txHex []byte, txNumber int) {
+	binary.PutUvarint(tx[8:16], uint64(txNumber))
+	hexUpdate := make([]byte, 16)
+	hex.Encode(hexUpdate, tx[8:16])
+	for i := 16; i < 32; i++ {
+		txHex[i] = hexUpdate[i-16]
+	}
+}
diff --git a/tools/tm-bench/transacter_test.go b/tools/tm-bench/transacter_test.go
new file mode 100644
index 000000000..086a43c31
--- /dev/null
+++ b/tools/tm-bench/transacter_test.go
@@ -0,0 +1,104 @@
+package main
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/require"
+)
+
+// This test tests that the output of generate tx and update tx is consistent
+func TestGenerateTxUpdateTxConsistentency(t *testing.T) {
+	cases := []struct {
+		connIndex        int
+		startingTxNumber int
+		txSize           int
+		hostname         string
+		numTxsToTest     int
+	}{
+		{0, 0, 50, "localhost:26657", 1000},
+		{70, 300, 10000, "localhost:26657", 1000},
+		{0, 50, 100000, "localhost:26657", 1000},
+	}
+
+	for tcIndex, tc := range cases {
+		hostnameHash := md5.Sum([]byte(tc.hostname))
+		// Tx generated from update tx. This is defined outside of the loop, since we have
+		// to a have something initially to update
+		updatedTx := generateTx(tc.connIndex, tc.startingTxNumber, tc.txSize, hostnameHash)
+		updatedHex := make([]byte, len(updatedTx)*2)
+		hex.Encode(updatedHex, updatedTx)
+		for i := 0; i < tc.numTxsToTest; i++ {
+			expectedTx := generateTx(tc.connIndex, tc.startingTxNumber+i, tc.txSize, hostnameHash)
+			expectedHex := make([]byte, len(expectedTx)*2)
+			hex.Encode(expectedHex, expectedTx)
+
+			updateTx(updatedTx, updatedHex, tc.startingTxNumber+i)
+
+			// after first 32 bytes is 8 bytes of time, then purely random bytes
+			require.Equal(t, expectedTx[:32], updatedTx[:32],
+				"First 32 bytes of the txs differed. tc #%d, i #%d", tcIndex, i)
+			require.Equal(t, expectedHex[:64], updatedHex[:64],
+				"First 64 bytes of the hex differed. tc #%d, i #%d", tcIndex, i)
+			// Test the lengths of the txs are as expected
+			require.Equal(t, tc.txSize, len(expectedTx),
+				"Length of expected Tx differed. tc #%d, i #%d", tcIndex, i)
+			require.Equal(t, tc.txSize, len(updatedTx),
+				"Length of expected Tx differed. tc #%d, i #%d", tcIndex, i)
+			require.Equal(t, tc.txSize*2, len(expectedHex),
+				"Length of expected hex differed. tc #%d, i #%d", tcIndex, i)
+			require.Equal(t, tc.txSize*2, len(updatedHex),
+				"Length of updated hex differed. tc #%d, i #%d", tcIndex, i)
+		}
+	}
+}
+
+func BenchmarkIterationOfSendLoop(b *testing.B) {
+	var (
+		connIndex = 0
+		txSize    = 25000
+	)
+
+	now := time.Now()
+	// something too far away to matter
+	endTime := now.Add(time.Hour)
+	txNumber := 0
+	hostnameHash := md5.Sum([]byte{0})
+	tx := generateTx(connIndex, txNumber, txSize, hostnameHash)
+	txHex := make([]byte, len(tx)*2)
+	hex.Encode(txHex, tx)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		updateTx(tx, txHex, txNumber)
+		paramsJSON, err := json.Marshal(map[string]interface{}{"tx": txHex})
+		if err != nil {
+			fmt.Printf("failed to encode params: %v\n", err)
+			os.Exit(1)
+		}
+		_ = json.RawMessage(paramsJSON)
+		_ = now.Add(sendTimeout)
+
+		if err != nil {
+			err = errors.Wrap(err,
+				fmt.Sprintf("txs send failed on connection #%d", connIndex))
+			logger.Error(err.Error())
+			return
+		}
+
+		// Cache the now operations
+		if i%5 == 0 {
+			now = time.Now()
+			if now.After(endTime) {
+				break
+			}
+		}
+
+		txNumber++
+	}
+}

From 66c4f7aeaef3c523156e88efa123141a3e5c02c2 Mon Sep 17 00:00:00 2001
From: ValarDragon <dojha12@gmail.com>
Date: Fri, 13 Jul 2018 11:23:17 -0700
Subject: [PATCH 9/9] Remove gopkg stuff, should hopefully get these tests in
 test_cover

---
 tools/tm-bench/Gopkg.lock | 334 --------------------------------------
 tools/tm-bench/Gopkg.toml |  55 -------
 tools/tm-bench/Makefile   |  68 +-------
 3 files changed, 1 insertion(+), 456 deletions(-)
 delete mode 100644 tools/tm-bench/Gopkg.lock
 delete mode 100644 tools/tm-bench/Gopkg.toml

diff --git a/tools/tm-bench/Gopkg.lock b/tools/tm-bench/Gopkg.lock
deleted file mode 100644
index aa1d819ca..000000000
--- a/tools/tm-bench/Gopkg.lock
+++ /dev/null
@@ -1,334 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  branch = "master"
-  name = "github.com/beorn7/perks"
-  packages = ["quantile"]
-  revision = "3a771d992973f24aa725d07868b467d1ddfceafb"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/btcsuite/btcd"
-  packages = ["btcec"]
-  revision = "fdfc19097e7ac6b57035062056f5b7b4638b8898"
-
-[[projects]]
-  name = "github.com/davecgh/go-spew"
-  packages = ["spew"]
-  revision = "346938d642f2ec3594ed81d874461961cd0faa76"
-  version = "v1.1.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/ebuchman/fail-test"
-  packages = ["."]
-  revision = "95f809107225be108efcf10a3509e4ea6ceef3c4"
-
-[[projects]]
-  name = "github.com/go-kit/kit"
-  packages = [
-    "log",
-    "log/level",
-    "log/term",
-    "metrics",
-    "metrics/discard",
-    "metrics/internal/lv",
-    "metrics/prometheus"
-  ]
-  revision = "4dc7be5d2d12881735283bcab7352178e190fc71"
-  version = "v0.6.0"
-
-[[projects]]
-  name = "github.com/go-logfmt/logfmt"
-  packages = ["."]
-  revision = "390ab7935ee28ec6b286364bba9b4dd6410cb3d5"
-  version = "v0.3.0"
-
-[[projects]]
-  name = "github.com/go-stack/stack"
-  packages = ["."]
-  revision = "259ab82a6cad3992b4e21ff5cac294ccb06474bc"
-  version = "v1.7.0"
-
-[[projects]]
-  name = "github.com/gogo/protobuf"
-  packages = [
-    "gogoproto",
-    "jsonpb",
-    "proto",
-    "protoc-gen-gogo/descriptor",
-    "sortkeys",
-    "types"
-  ]
-  revision = "1adfc126b41513cc696b209667c8656ea7aac67c"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "github.com/golang/protobuf"
-  packages = [
-    "proto",
-    "ptypes",
-    "ptypes/any",
-    "ptypes/duration",
-    "ptypes/timestamp"
-  ]
-  revision = "925541529c1fa6821df4e44ce2723319eb2be768"
-  version = "v1.0.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/golang/snappy"
-  packages = ["."]
-  revision = "2e65f85255dbc3072edf28d6b5b8efc472979f5a"
-
-[[projects]]
-  name = "github.com/gorilla/websocket"
-  packages = ["."]
-  revision = "ea4d1f681babbce9545c9c5f3d5194a789c89f5b"
-  version = "v1.2.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/jmhodges/levigo"
-  packages = ["."]
-  revision = "c42d9e0ca023e2198120196f842701bb4c55d7b9"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/kr/logfmt"
-  packages = ["."]
-  revision = "b84e30acd515aadc4b783ad4ff83aff3299bdfe0"
-
-[[projects]]
-  name = "github.com/matttproud/golang_protobuf_extensions"
-  packages = ["pbutil"]
-  revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c"
-  version = "v1.0.1"
-
-[[projects]]
-  name = "github.com/pkg/errors"
-  packages = ["."]
-  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
-  version = "v0.8.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/prometheus/client_golang"
-  packages = [
-    "prometheus",
-    "prometheus/promhttp"
-  ]
-  revision = "ee1c9d7e23df7f011bdf6f12a5c9e7f0ae10a1fe"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/prometheus/client_model"
-  packages = ["go"]
-  revision = "99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/prometheus/common"
-  packages = [
-    "expfmt",
-    "internal/bitbucket.org/ww/goautoneg",
-    "model"
-  ]
-  revision = "7600349dcfe1abd18d72d3a1770870d9800a7801"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/prometheus/procfs"
-  packages = [
-    ".",
-    "internal/util",
-    "nfs",
-    "xfs"
-  ]
-  revision = "ae68e2d4c00fed4943b5f6698d504a5fe083da8a"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/rcrowley/go-metrics"
-  packages = ["."]
-  revision = "e2704e165165ec55d062f5919b4b29494e9fa790"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/syndtr/goleveldb"
-  packages = [
-    "leveldb",
-    "leveldb/cache",
-    "leveldb/comparer",
-    "leveldb/errors",
-    "leveldb/filter",
-    "leveldb/iterator",
-    "leveldb/journal",
-    "leveldb/memdb",
-    "leveldb/opt",
-    "leveldb/storage",
-    "leveldb/table",
-    "leveldb/util"
-  ]
-  revision = "c4c61651e9e37fa117f53c5a906d3b63090d8445"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/tendermint/ed25519"
-  packages = [
-    ".",
-    "edwards25519",
-    "extra25519"
-  ]
-  revision = "d8387025d2b9d158cf4efb07e7ebf814bcce2057"
-
-[[projects]]
-  name = "github.com/tendermint/go-amino"
-  packages = ["."]
-  revision = "2106ca61d91029c931fd54968c2bb02dc96b1412"
-  version = "0.10.1"
-
-[[projects]]
-  branch = "develop"
-  name = "github.com/tendermint/tendermint"
-  packages = [
-    "abci/client",
-    "abci/example/code",
-    "abci/example/kvstore",
-    "abci/types",
-    "blockchain",
-    "config",
-    "consensus",
-    "consensus/types",
-    "crypto",
-    "crypto/merkle",
-    "crypto/tmhash",
-    "evidence",
-    "libs/autofile",
-    "libs/clist",
-    "libs/common",
-    "libs/db",
-    "libs/events",
-    "libs/flowrate",
-    "libs/log",
-    "libs/pubsub",
-    "libs/pubsub/query",
-    "mempool",
-    "node",
-    "p2p",
-    "p2p/conn",
-    "p2p/pex",
-    "p2p/upnp",
-    "privval",
-    "proxy",
-    "rpc/client",
-    "rpc/core",
-    "rpc/core/types",
-    "rpc/grpc",
-    "rpc/lib",
-    "rpc/lib/client",
-    "rpc/lib/server",
-    "rpc/lib/types",
-    "state",
-    "state/txindex",
-    "state/txindex/kv",
-    "state/txindex/null",
-    "types",
-    "version"
-  ]
-  revision = "f5ad8ef8600c33532a16de0879ff6b9745bb394d"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/crypto"
-  packages = [
-    "curve25519",
-    "internal/subtle",
-    "nacl/box",
-    "nacl/secretbox",
-    "openpgp/armor",
-    "openpgp/errors",
-    "poly1305",
-    "ripemd160",
-    "salsa20/salsa"
-  ]
-  revision = "a49355c7e3f8fe157a85be2f77e6e269a0f89602"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/net"
-  packages = [
-    "context",
-    "http/httpguts",
-    "http2",
-    "http2/hpack",
-    "idna",
-    "internal/timeseries",
-    "netutil",
-    "trace"
-  ]
-  revision = "039a4258aec0ad3c79b905677cceeab13b296a77"
-
-[[projects]]
-  name = "golang.org/x/text"
-  packages = [
-    "collate",
-    "collate/build",
-    "internal/colltab",
-    "internal/gen",
-    "internal/tag",
-    "internal/triegen",
-    "internal/ucd",
-    "language",
-    "secure/bidirule",
-    "transform",
-    "unicode/bidi",
-    "unicode/cldr",
-    "unicode/norm",
-    "unicode/rangetable"
-  ]
-  revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
-  version = "v0.3.0"
-
-[[projects]]
-  name = "google.golang.org/genproto"
-  packages = ["googleapis/rpc/status"]
-  revision = "7fd901a49ba6a7f87732eb344f6e3c5b19d1b200"
-
-[[projects]]
-  name = "google.golang.org/grpc"
-  packages = [
-    ".",
-    "balancer",
-    "balancer/base",
-    "balancer/roundrobin",
-    "codes",
-    "connectivity",
-    "credentials",
-    "encoding",
-    "encoding/proto",
-    "grpclb/grpc_lb_v1/messages",
-    "grpclog",
-    "internal",
-    "keepalive",
-    "metadata",
-    "naming",
-    "peer",
-    "resolver",
-    "resolver/dns",
-    "resolver/passthrough",
-    "stats",
-    "status",
-    "tap",
-    "transport"
-  ]
-  revision = "d11072e7ca9811b1100b80ca0269ac831f06d024"
-  version = "v1.11.3"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  inputs-digest = "5c21a60b80ac7d60f7be693de13f9fadb62226b502431bdb38fb9794a98c5b02"
-  solver-name = "gps-cdcl"
-  solver-version = 1
diff --git a/tools/tm-bench/Gopkg.toml b/tools/tm-bench/Gopkg.toml
deleted file mode 100644
index 3b2dfa4ec..000000000
--- a/tools/tm-bench/Gopkg.toml
+++ /dev/null
@@ -1,55 +0,0 @@
-# Gopkg.toml example
-#
-# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html
-# for detailed Gopkg.toml documentation.
-#
-# required = ["github.com/user/thing/cmd/thing"]
-# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
-#
-# [[constraint]]
-#   name = "github.com/user/project"
-#   version = "1.0.0"
-#
-# [[constraint]]
-#   name = "github.com/user/project2"
-#   branch = "dev"
-#   source = "github.com/myfork/project2"
-#
-# [[override]]
-#   name = "github.com/x/y"
-#   version = "2.4.0"
-#
-# [prune]
-#   non-go = false
-#   go-tests = true
-#   unused-packages = true
-
-
-[[constraint]]
-  name = "github.com/go-kit/kit"
-  version = "^0.6.0"
-
-[[constraint]]
-  name = "github.com/gorilla/websocket"
-  version = "^1.2.0"
-
-[[constraint]]
-  name = "github.com/pkg/errors"
-  version = "^0.8.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/rcrowley/go-metrics"
-
-[[constraint]]
-  name = "github.com/tendermint/tendermint"
-  branch = "develop"
-
-# this got updated and broke, so locked to an old working commit ...
-[[override]]
-  name = "google.golang.org/genproto"
-  revision = "7fd901a49ba6a7f87732eb344f6e3c5b19d1b200"
-
-[prune]
-  go-tests = true
-  unused-packages = true
diff --git a/tools/tm-bench/Makefile b/tools/tm-bench/Makefile
index b95aeee93..2d427dbc1 100644
--- a/tools/tm-bench/Makefile
+++ b/tools/tm-bench/Makefile
@@ -1,35 +1,7 @@
 DIST_DIRS := find * -type d -exec
 VERSION := $(shell perl -ne '/^var version.*"([^"]+)".*$$/ && print "v$$1\n"' main.go)
-GOTOOLS = \
-					github.com/mitchellh/gox \
-					github.com/golang/dep/cmd/dep \
-					gopkg.in/alecthomas/gometalinter.v2
 
-all: check get_vendor_deps build test install metalinter
-
-check: check_tools
-
-########################################
-### Tools & dependencies
-
-check_tools:
-	@# https://stackoverflow.com/a/25668869
-	@echo "Found tools: $(foreach tool,$(GOTOOLS_CHECK),\
-		$(if $(shell which $(tool)),$(tool),$(error "No $(tool) in PATH")))"
-
-get_tools:
-	@echo "--> Installing tools"
-	go get -u -v $(GOTOOLS)
-	@gometalinter.v2 --install
-
-update_tools:
-	@echo "--> Updating tools"
-	@go get -u $(GOTOOLS)
-
-get_vendor_deps:
-	@rm -rf vendor/
-	@echo "--> Running dep ensure"
-	@dep ensure
+all: build test install
 
 ########################################
 ###  Build
@@ -72,44 +44,6 @@ clean:
 	rm -f ./tm-bench
 	rm -rf ./dist
 
-########################################
-### Formatting, linting, and vetting
-
-fmt:
-	@go fmt ./...
-
-metalinter:
-	@echo "==> Running linter"
-	gometalinter.v2 --vendor --deadline=600s --disable-all  \
-		--enable=maligned \
-		--enable=deadcode \
-		--enable=goconst \
-		--enable=goimports \
-		--enable=gosimple \
-		--enable=ineffassign \
-		--enable=megacheck \
-		--enable=misspell \
-		--enable=staticcheck \
-		--enable=safesql \
-		--enable=structcheck \
-		--enable=unconvert \
-		--enable=unused \
-		--enable=varcheck \
-		--enable=vetshadow \
-		./...
-		#--enable=gas \
-		#--enable=dupl \
-		#--enable=errcheck \
-		#--enable=gocyclo \
-		#--enable=golint \ <== comments on anything exported
-		#--enable=gotype \
-		#--enable=interfacer \
-		#--enable=unparam \
-		#--enable=vet \
-
-metalinter_all:
-	gometalinter.v2 --vendor --deadline=600s --enable-all --disable=lll ./...
-
 # To avoid unintended conflicts with file names, always add to .PHONY
 # unless there is a reason not to.
 # https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html