From 800cce80b72660eb774175d1fbaf166ba62f55c9 Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Wed, 7 Jul 2021 12:59:27 +0200 Subject: [PATCH] e2e: allow variable tx size (#6659) --- test/e2e/generator/generate.go | 6 +++--- test/e2e/pkg/manifest.go | 7 +++---- test/e2e/pkg/testnet.go | 12 +++++++++++- test/e2e/runner/load.go | 8 ++++---- 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/test/e2e/generator/generate.go b/test/e2e/generator/generate.go index f212852f0..07a803520 100644 --- a/test/e2e/generator/generate.go +++ b/test/e2e/generator/generate.go @@ -44,6 +44,7 @@ var ( "restart": 0.1, } evidence = uniformChoice{0, 1, 10} + txSize = uniformChoice{1024, 10240} // either 1kb or 10kb ) // Generate generates random testnets using the given RNG. @@ -92,6 +93,7 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er KeyType: opt["keyType"].(string), Evidence: evidence.Choose(r).(int), QueueType: opt["queueType"].(string), + TxSize: int64(txSize.Choose(r).(int)), } var p2pNodeFactor int @@ -127,7 +129,6 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er // First we generate seed nodes, starting at the initial height. for i := 1; i <= numSeeds; i++ { node := generateNode(r, e2e.ModeSeed, 0, manifest.InitialHeight, false) - node.QueueType = manifest.QueueType if p2pNodeFactor == 0 { node.DisableLegacyP2P = manifest.DisableLegacyP2P @@ -153,7 +154,6 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er node := generateNode( r, e2e.ModeValidator, startAt, manifest.InitialHeight, i <= 2) - node.QueueType = manifest.QueueType if p2pNodeFactor == 0 { node.DisableLegacyP2P = manifest.DisableLegacyP2P } else if p2pNodeFactor%i == 0 { @@ -189,7 +189,7 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er nextStartAt += 5 } node := generateNode(r, e2e.ModeFull, startAt, manifest.InitialHeight, false) - node.QueueType = manifest.QueueType + if p2pNodeFactor == 0 { node.DisableLegacyP2P = manifest.DisableLegacyP2P } else if p2pNodeFactor%i == 0 { diff --git a/test/e2e/pkg/manifest.go b/test/e2e/pkg/manifest.go index fa861df8b..5320c4637 100644 --- a/test/e2e/pkg/manifest.go +++ b/test/e2e/pkg/manifest.go @@ -64,6 +64,9 @@ type Manifest struct { // QueueType describes the type of queue that the system uses internally QueueType string `toml:"queue_type"` + + // Number of bytes per tx. Default is 1kb (1024) + TxSize int64 } // ManifestNode represents a node in a testnet manifest. @@ -143,10 +146,6 @@ type ManifestNode struct { // UseNewP2P enables use of the new p2p layer for this node. DisableLegacyP2P bool `toml:"disable_legacy_p2p"` - - // QueueType describes the type of queue that the p2p layer - // uses internally. - QueueType string `toml:"queue_type"` } // Save saves the testnet manifest to a file. diff --git a/test/e2e/pkg/testnet.go b/test/e2e/pkg/testnet.go index 2cb626bf6..1cbe919c5 100644 --- a/test/e2e/pkg/testnet.go +++ b/test/e2e/pkg/testnet.go @@ -66,6 +66,7 @@ type Testnet struct { KeyType string Evidence int LogLevel string + TxSize int64 } // Node represents a Tendermint node in a testnet. @@ -133,10 +134,14 @@ func LoadTestnet(file string) (*Testnet, error) { Evidence: manifest.Evidence, KeyType: "ed25519", LogLevel: manifest.LogLevel, + TxSize: manifest.TxSize, } if len(manifest.KeyType) != 0 { testnet.KeyType = manifest.KeyType } + if testnet.TxSize <= 0 { + testnet.TxSize = 1024 + } if manifest.InitialHeight > 0 { testnet.InitialHeight = manifest.InitialHeight } @@ -169,8 +174,8 @@ func LoadTestnet(file string) (*Testnet, error) { RetainBlocks: nodeManifest.RetainBlocks, Perturbations: []Perturbation{}, LogLevel: manifest.LogLevel, - DisableLegacyP2P: manifest.DisableLegacyP2P, QueueType: manifest.QueueType, + DisableLegacyP2P: manifest.DisableLegacyP2P || nodeManifest.DisableLegacyP2P, } if node.StartAt == testnet.InitialHeight { @@ -320,6 +325,11 @@ func (n Node) Validate(testnet Testnet) error { default: return fmt.Errorf("invalid fast sync setting %q", n.FastSync) } + switch n.QueueType { + case "", "priority", "wdrr", "fifo": + default: + return fmt.Errorf("unsupported p2p queue type: %s", n.QueueType) + } switch n.Database { case "goleveldb", "cleveldb", "boltdb", "rocksdb", "badgerdb": default: diff --git a/test/e2e/runner/load.go b/test/e2e/runner/load.go index 573e46540..01c49efea 100644 --- a/test/e2e/runner/load.go +++ b/test/e2e/runner/load.go @@ -38,7 +38,7 @@ func Load(ctx context.Context, testnet *e2e.Testnet, multiplier int) error { logger.Info(fmt.Sprintf("Starting transaction load (%v workers)...", concurrency)) started := time.Now() - go loadGenerate(ctx, chTx, multiplier) + go loadGenerate(ctx, chTx, multiplier, testnet.TxSize) for w := 0; w < concurrency; w++ { go loadProcess(ctx, testnet, chTx, chSuccess) @@ -66,13 +66,13 @@ func Load(ctx context.Context, testnet *e2e.Testnet, multiplier int) error { } // loadGenerate generates jobs until the context is canceled -func loadGenerate(ctx context.Context, chTx chan<- types.Tx, multiplier int) { +func loadGenerate(ctx context.Context, chTx chan<- types.Tx, multiplier int, size int64) { for i := 0; i < math.MaxInt64; i++ { // We keep generating the same 1000 keys over and over, with different values. // This gives a reasonable load without putting too much data in the app. id := i % 1000 - bz := make([]byte, 1024) // 1kb hex-encoded + bz := make([]byte, size) _, err := rand.Read(bz) if err != nil { panic(fmt.Sprintf("Failed to read random bytes: %v", err)) @@ -81,7 +81,7 @@ func loadGenerate(ctx context.Context, chTx chan<- types.Tx, multiplier int) { select { case chTx <- tx: - time.Sleep(time.Second / time.Duration(multiplier)) + time.Sleep(time.Millisecond * time.Duration(int(size)/multiplier)) case <-ctx.Done(): close(chTx)