From 8023a2aeef0ee4ec5b1dd56ad25408b8d844a86b Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Mon, 27 Sep 2021 15:38:03 -0400 Subject: [PATCH] e2e: add generator tests (#7008) --- .github/workflows/coverage.yml | 1 - test/e2e/generator/generate.go | 44 +++++++++++++++--- test/e2e/generator/generate_test.go | 70 +++++++++++++++++++++++++++++ test/e2e/pkg/manifest.go | 5 +++ 4 files changed, 113 insertions(+), 7 deletions(-) create mode 100644 test/e2e/generator/generate_test.go diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 49c76ab98..3553f6356 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -4,7 +4,6 @@ on: push: paths: - "**.go" - - "!test/" branches: - master - release/** diff --git a/test/e2e/generator/generate.go b/test/e2e/generator/generate.go index 6369cf962..f314ae8c0 100644 --- a/test/e2e/generator/generate.go +++ b/test/e2e/generator/generate.go @@ -73,7 +73,15 @@ func Generate(r *rand.Rand, opts Options) ([]e2e.Manifest, error) { manifests := []e2e.Manifest{} switch opts.P2P { case NewP2PMode, LegacyP2PMode, HybridP2PMode: + defer func() { + // avoid modifying the global state. + original := make([]interface{}, len(testnetCombinations["p2p"])) + copy(original, testnetCombinations["p2p"]) + testnetCombinations["p2p"] = original + }() + testnetCombinations["p2p"] = []interface{}{opts.P2P} + default: testnetCombinations["p2p"] = []interface{}{NewP2PMode, LegacyP2PMode, HybridP2PMode} } @@ -154,13 +162,15 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er case "large": // FIXME Networks are kept small since large ones use too much CPU. numSeeds = r.Intn(2) - numLightClients = r.Intn(3) + numLightClients = r.Intn(2) numValidators = 4 + r.Intn(4) numFulls = r.Intn(4) default: return manifest, fmt.Errorf("unknown topology %q", opt["topology"]) } + const legacyP2PFactor float64 = 0.5 + // First we generate seed nodes, starting at the initial height. for i := 1; i <= numSeeds; i++ { node := generateNode(r, manifest, e2e.ModeSeed, 0, false) @@ -169,18 +179,23 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er case LegacyP2PMode: node.UseLegacyP2P = true case HybridP2PMode: - node.UseLegacyP2P = r.Intn(5) < 2 + node.UseLegacyP2P = r.Float64() < legacyP2PFactor } manifest.Nodes[fmt.Sprintf("seed%02d", i)] = node } + var ( + numSyncingNodes = 0 + hybridNumNew = 0 + hybridNumLegacy = 0 + ) + // Next, we generate validators. We make sure a BFT quorum of validators start // at the initial height, and that we have two archive nodes. We also set up // the initial validator set, and validator set updates for delayed nodes. nextStartAt := manifest.InitialHeight + 5 quorum := numValidators*2/3 + 1 - numSyncingNodes := 0 for i := 1; i <= numValidators; i++ { startAt := int64(0) if i > quorum && numSyncingNodes < 2 && r.Float64() >= 0.25 { @@ -195,7 +210,23 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er case LegacyP2PMode: node.UseLegacyP2P = true case HybridP2PMode: - node.UseLegacyP2P = r.Intn(5) < 2 + node.UseLegacyP2P = r.Float64() < legacyP2PFactor + if node.UseLegacyP2P { + hybridNumLegacy++ + if hybridNumNew == 0 { + hybridNumNew++ + hybridNumLegacy-- + node.UseLegacyP2P = false + } + } else { + hybridNumNew++ + if hybridNumLegacy == 0 { + hybridNumNew-- + hybridNumLegacy++ + node.UseLegacyP2P = true + + } + } } manifest.Nodes[name] = node @@ -222,7 +253,8 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er // Finally, we generate random full nodes. for i := 1; i <= numFulls; i++ { startAt := int64(0) - if r.Float64() >= 0.5 { + if numSyncingNodes < 2 && r.Float64() >= 0.5 { + numSyncingNodes++ startAt = nextStartAt nextStartAt += 5 } @@ -232,7 +264,7 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er case LegacyP2PMode: node.UseLegacyP2P = true case HybridP2PMode: - node.UseLegacyP2P = r.Intn(5) < 2 + node.UseLegacyP2P = r.Float64() > legacyP2PFactor } manifest.Nodes[fmt.Sprintf("full%02d", i)] = node diff --git a/test/e2e/generator/generate_test.go b/test/e2e/generator/generate_test.go new file mode 100644 index 000000000..b816ba577 --- /dev/null +++ b/test/e2e/generator/generate_test.go @@ -0,0 +1,70 @@ +package main + +import ( + "fmt" + "math/rand" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + e2e "github.com/tendermint/tendermint/test/e2e/pkg" +) + +func TestGenerator(t *testing.T) { + manifests, err := Generate(rand.New(rand.NewSource(randomSeed)), Options{P2P: MixedP2PMode}) + require.NoError(t, err) + require.True(t, len(manifests) >= 64, "insufficient combinations") + + // this just means that the numbers reported by the test + // failures map to the test cases that you'd see locally. + e2e.SortManifests(manifests, false /* ascending */) + + for idx, m := range manifests { + t.Run(fmt.Sprintf("Case%04d", idx), func(t *testing.T) { + numStateSyncs := 0 + for name, node := range m.Nodes { + if node.StateSync != e2e.StateSyncDisabled { + numStateSyncs++ + } + t.Run(name, func(t *testing.T) { + if node.StartAt > m.InitialHeight+5 && !node.Stateless() { + require.NotEqual(t, node.StateSync, e2e.StateSyncDisabled) + } + if node.StateSync != e2e.StateSyncDisabled { + require.Zero(t, node.Seeds, node.StateSync) + require.True(t, len(node.PersistentPeers) >= 2) + require.Equal(t, "v0", node.BlockSync) + } + + }) + } + require.True(t, numStateSyncs <= 2) + }) + } + + t.Run("Hybrid", func(t *testing.T) { + manifests, err := Generate(rand.New(rand.NewSource(randomSeed)), Options{P2P: HybridP2PMode}) + require.NoError(t, err) + require.True(t, len(manifests) >= 16, "insufficient combinations: %d", len(manifests)) + + for idx, m := range manifests { + t.Run(fmt.Sprintf("Case%04d", idx), func(t *testing.T) { + require.True(t, len(m.Nodes) > 1) + + var numLegacy, numNew int + for _, node := range m.Nodes { + if node.UseLegacyP2P { + numLegacy++ + } else { + numNew++ + } + } + + assert.True(t, numLegacy >= 1, "not enough legacy nodes [%d/%d]", + numLegacy, len(m.Nodes)) + assert.True(t, numNew >= 1, "not enough new nodes [%d/%d]", + numNew, len(m.Nodes)) + }) + } + }) +} diff --git a/test/e2e/pkg/manifest.go b/test/e2e/pkg/manifest.go index e3e5bb7ee..4492f4037 100644 --- a/test/e2e/pkg/manifest.go +++ b/test/e2e/pkg/manifest.go @@ -150,6 +150,11 @@ type ManifestNode struct { UseLegacyP2P bool `toml:"use_legacy_p2p"` } +// Stateless reports whether m is a node that does not own state, including light and seed nodes. +func (m ManifestNode) Stateless() bool { + return m.Mode == string(ModeLight) || m.Mode == string(ModeSeed) +} + // Save saves the testnet manifest to a file. func (m Manifest) Save(file string) error { f, err := os.Create(file)