From 9c8379ef305091f5825a4d91244c19ec2697894c Mon Sep 17 00:00:00 2001
From: Sam Kleinman <garen@tychoish.com>
Date: Wed, 25 Aug 2021 08:24:01 -0400
Subject: [PATCH] e2e: more consistent node selection during tests (#6857)

In the transaction load generator, the e2e test harness previously distributed load randomly to hosts, which was a source of test non-determinism. This change distributes the load generation to the different nodes in the set in a round robin fashion, to produce more reliable results, but does not otherwise change the behavior of the test harness.
---
 test/e2e/pkg/testnet.go     | 10 ------
 test/e2e/runner/evidence.go | 17 +++++++++-
 test/e2e/runner/load.go     | 67 +++++++++++++++++++++++++++----------
 3 files changed, 65 insertions(+), 29 deletions(-)

diff --git a/test/e2e/pkg/testnet.go b/test/e2e/pkg/testnet.go
index cfeb54bde..cec58bd20 100644
--- a/test/e2e/pkg/testnet.go
+++ b/test/e2e/pkg/testnet.go
@@ -417,16 +417,6 @@ func (t Testnet) ArchiveNodes() []*Node {
 	return nodes
 }
 
-// RandomNode returns a random non-seed node.
-func (t Testnet) RandomNode() *Node {
-	for {
-		node := t.Nodes[rand.Intn(len(t.Nodes))]
-		if node.Mode != ModeSeed {
-			return node
-		}
-	}
-}
-
 // IPv6 returns true if the testnet is an IPv6 network.
 func (t Testnet) IPv6() bool {
 	return t.IP.IP.To4() == nil
diff --git a/test/e2e/runner/evidence.go b/test/e2e/runner/evidence.go
index 6a246dcb5..30e8d9f0a 100644
--- a/test/e2e/runner/evidence.go
+++ b/test/e2e/runner/evidence.go
@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"math/rand"
@@ -29,7 +30,21 @@ const lightClientEvidenceRatio = 4
 // DuplicateVoteEvidence.
 func InjectEvidence(testnet *e2e.Testnet, amount int) error {
 	// select a random node
-	targetNode := testnet.RandomNode()
+	var targetNode *e2e.Node
+
+	for i := 0; i < len(testnet.Nodes)-1; i++ {
+		targetNode = testnet.Nodes[rand.Intn(len(testnet.Nodes))] // nolint: gosec
+		if targetNode.Mode == e2e.ModeSeed {
+			targetNode = nil
+			continue
+		}
+
+		break
+	}
+
+	if targetNode == nil {
+		return errors.New("could not find node to inject evidence into")
+	}
 
 	logger.Info(fmt.Sprintf("Injecting evidence through %v (amount: %d)...", targetNode.Name, amount))
 
diff --git a/test/e2e/runner/load.go b/test/e2e/runner/load.go
index 518e32564..b57c96ddf 100644
--- a/test/e2e/runner/load.go
+++ b/test/e2e/runner/load.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"container/ring"
 	"context"
 	"crypto/rand"
 	"errors"
@@ -93,34 +94,64 @@ func loadGenerate(ctx context.Context, chTx chan<- types.Tx, multiplier int, siz
 
 // loadProcess processes transactions
 func loadProcess(ctx context.Context, testnet *e2e.Testnet, chTx <-chan types.Tx, chSuccess chan<- types.Tx) {
-	// Each worker gets its own client to each node, which allows for some
-	// concurrency while still bounding it.
-	clients := map[string]*rpchttp.HTTP{}
+	// Each worker gets its own client to each usable node, which
+	// allows for some concurrency while still bounding it.
+	clients := make([]*rpchttp.HTTP, 0, len(testnet.Nodes))
+
+	for idx := range testnet.Nodes {
+		// Construct a list of usable nodes for the creating
+		// load. Don't send load through seed nodes because
+		// they do not provide the RPC endpoints required to
+		// broadcast transaction.
+		if testnet.Nodes[idx].Mode == e2e.ModeSeed {
+			continue
+		}
+
+		client, err := testnet.Nodes[idx].Client()
+		if err != nil {
+			continue
+		}
+
+		clients = append(clients, client)
+	}
+
+	if len(clients) == 0 {
+		panic("no clients to process load")
+	}
+
+	// Put the clients in a ring so they can be used in a
+	// round-robin fashion.
+	clientRing := ring.New(len(clients))
+	for idx := range clients {
+		clientRing.Value = clients[idx]
+		clientRing = clientRing.Next()
+	}
 
 	var err error
-	for tx := range chTx {
-		node := testnet.RandomNode()
 
-		client, ok := clients[node.Name]
-		if !ok {
-			client, err = node.Client()
-			if err != nil {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case tx := <-chTx:
+			clientRing = clientRing.Next()
+			client := clientRing.Value.(*rpchttp.HTTP)
+
+			if _, err := client.Health(ctx); err != nil {
 				continue
 			}
 
-			// check that the node is up
-			_, err = client.Health(ctx)
-			if err != nil {
+			if _, err = client.BroadcastTxSync(ctx, tx); err != nil {
 				continue
 			}
 
-			clients[node.Name] = client
-		}
+			select {
+			case chSuccess <- tx:
+				continue
+			case <-ctx.Done():
+				return
+			}
 
-		if _, err = client.BroadcastTxSync(ctx, tx); err != nil {
-			continue
 		}
-
-		chSuccess <- tx
 	}
 }