From dc28734dadfc78dd3784c68cbaec4ec7d7017ed3 Mon Sep 17 00:00:00 2001
From: "M. J. Fromberger" <fromberger@interchain.io>
Date: Wed, 3 Nov 2021 07:39:09 -0700
Subject: [PATCH] pubsub: Remove uninformative publisher benchmarks. (#7195)

Prior to #7177, these benchmarks did not provide any useful data about the
performance of the pubsub system (in fact, prior to #7178, half of them did not
work at all).

Specifically, they create a bunch of subscribers with 1 buffer slot on a
default publisher and copy messages to them. But because the publisher is
single-threaded, and doesn't block for delivery, all this tested is how long it
takes to receive a single message from a channel and deliver it to another
channel. The resulting stat does not even vary meaningfully with batch size,
since it's testing a serial workload.

Since #7177 the benchmarks do correctly reflect delivery time (good), but still
do not tell us anything useful: The latencies that matter for pubsub are not
internal queuing, but the effects of backpressure on the publisher via the
subscribers. That's an integration problem, and simulating a fake workload does
not provide meaningful results.

On that basis, remove these benchmarks.
---
 libs/pubsub/pubsub_test.go | 108 -------------------------------------
 1 file changed, 108 deletions(-)

diff --git a/libs/pubsub/pubsub_test.go b/libs/pubsub/pubsub_test.go
index e0f1f9eb5..397a88e3d 100644
--- a/libs/pubsub/pubsub_test.go
+++ b/libs/pubsub/pubsub_test.go
@@ -451,114 +451,6 @@ func TestBufferCapacity(t *testing.T) {
 	}
 }
 
-func Benchmark10Clients(b *testing.B)   { benchmarkNClients(10, b) }
-func Benchmark100Clients(b *testing.B)  { benchmarkNClients(100, b) }
-func Benchmark1000Clients(b *testing.B) { benchmarkNClients(1000, b) }
-
-func Benchmark10ClientsOneQuery(b *testing.B)   { benchmarkNClientsOneQuery(10, b) }
-func Benchmark100ClientsOneQuery(b *testing.B)  { benchmarkNClientsOneQuery(100, b) }
-func Benchmark1000ClientsOneQuery(b *testing.B) { benchmarkNClientsOneQuery(1000, b) }
-
-func benchmarkNClients(n int, b *testing.B) {
-	s := pubsub.NewServer()
-	err := s.Start()
-	require.NoError(b, err)
-
-	b.Cleanup(func() {
-		if err := s.Stop(); err != nil {
-			b.Error(err)
-		}
-	})
-
-	ctx := context.Background()
-	for i := 0; i < n; i++ {
-		subscription, err := s.Subscribe(
-			ctx,
-			clientID,
-			query.MustParse(fmt.Sprintf("abci.Account.Owner = 'Ivan' AND abci.Invoices.Number = %d", i)),
-		)
-		if err != nil {
-			b.Fatal(err)
-		}
-		go func() {
-			for {
-				select {
-				case <-subscription.Out():
-					continue
-				case <-subscription.Canceled():
-					return
-				}
-			}
-		}()
-	}
-
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		events := []abci.Event{
-			{
-				Type:       "abci.Account",
-				Attributes: []abci.EventAttribute{{Key: "Owner", Value: "Ivan"}},
-			},
-			{
-				Type:       "abci.Invoices",
-				Attributes: []abci.EventAttribute{{Key: "Number", Value: string(rune(i))}},
-			},
-		}
-
-		require.NoError(b, s.PublishWithEvents(ctx, "Gamora", events))
-	}
-}
-
-func benchmarkNClientsOneQuery(n int, b *testing.B) {
-	s := pubsub.NewServer()
-	err := s.Start()
-	require.NoError(b, err)
-	b.Cleanup(func() {
-		if err := s.Stop(); err != nil {
-			b.Error(err)
-		}
-	})
-
-	ctx := context.Background()
-	q := query.MustParse("abci.Account.Owner = 'Ivan' AND abci.Invoices.Number = 1")
-	for i := 0; i < n; i++ {
-		id := fmt.Sprintf("clientID-%d", i+1)
-		subscription, err := s.Subscribe(ctx, id, q)
-		if err != nil {
-			b.Fatal(err)
-		}
-		go func() {
-			for {
-				select {
-				case <-subscription.Out():
-					continue
-				case <-subscription.Canceled():
-					return
-				}
-			}
-		}()
-	}
-
-	b.ReportAllocs()
-	b.ResetTimer()
-
-	for i := 0; i < b.N; i++ {
-		events := []abci.Event{
-			{
-				Type:       "abci.Account",
-				Attributes: []abci.EventAttribute{{Key: "Owner", Value: "Ivan"}},
-			},
-			{
-				Type:       "abci.Invoices",
-				Attributes: []abci.EventAttribute{{Key: "Number", Value: "1"}},
-			},
-		}
-
-		require.NoError(b, s.PublishWithEvents(ctx, "Gamora", events))
-	}
-}
-
 // HELPERS
 
 func assertReceive(t *testing.T, expected interface{}, ch <-chan pubsub.Message, msgAndArgs ...interface{}) {