e2e: improve manifest sorting algorithim (#6979)

test/e2e/generator/generate.go

@@ -80,26 +80,27 @@ func Generate(r *rand.Rand, opts Options) ([]e2e.Manifest, error) {
 			return nil, err
 		}
 
+		if len(manifest.Nodes) < opts.MinNetworkSize {
+			continue
+		}
+
 		if len(manifest.Nodes) == 1 {
 			if opt["p2p"] == HybridP2PMode {
 				continue
 			}
 		}
-		manifests = append(manifests, manifest)
-	}
 
-	if opts.Sorted {
-		// When the sorted flag is set (generally, as long as
-		// groups aren't set),
-		e2e.SortManifests(manifests)
+		manifests = append(manifests, manifest)
 	}
 
 	return manifests, nil
 }
 
 type Options struct {
-	P2P    P2PMode
-	Sorted bool
+	MinNetworkSize int
+	NumGroups      int
+	Directory      string
+	P2P            P2PMode
 }
 
 type P2PMode string

test/e2e/generator/main.go

@@ -3,7 +3,6 @@ package main
 
 import (
 	"fmt"
-	"math"
 	"math/rand"
 	"os"
 	"path/filepath"
@@ -11,6 +10,7 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/tendermint/tendermint/libs/log"
+	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
 )
 
 const (
@@ -26,6 +26,7 @@ func main() {
 // CLI is the Cobra-based command-line interface.
 type CLI struct {
 	root *cobra.Command
+	opts Options
 }
 
 // NewCLI sets up the CLI.
@@ -37,31 +38,36 @@ func NewCLI() *CLI {
 		SilenceUsage:  true,
 		SilenceErrors: true, // we'll output them ourselves in Run()
 		RunE: func(cmd *cobra.Command, args []string) error {
-			dir, err := cmd.Flags().GetString("dir")
+			var opts Options
+			var err error
+
+			cli.opts.Directory, err = cmd.Flags().GetString("dir")
+			if err != nil {
+				return err
+			}
+
+			cli.opts.NumGroups, err = cmd.Flags().GetInt("groups")
 			if err != nil {
 				return err
 			}
-			groups, err := cmd.Flags().GetInt("groups")
+
+			cli.opts.MinNetworkSize, err = cmd.Flags().GetInt("min-size")
 			if err != nil {
 				return err
 			}
+
 			p2pMode, err := cmd.Flags().GetString("p2p")
 			if err != nil {
 				return err
 			}
-			var opts Options
 			switch mode := P2PMode(p2pMode); mode {
 			case NewP2PMode, LegacyP2PMode, HybridP2PMode, MixedP2PMode:
-				opts = Options{P2P: mode}
+				opts.P2P = mode
 			default:
 				return fmt.Errorf("p2p mode must be either new, legacy, hybrid or mixed got %s", p2pMode)
 			}
 
-			if groups == 0 {
-				opts.Sorted = true
-			}
-
-			return cli.generate(dir, groups, opts)
+			return cli.generate()
 		},
 	}
 
@@ -70,40 +76,43 @@ func NewCLI() *CLI {
 	cli.root.PersistentFlags().IntP("groups", "g", 0, "Number of groups")
 	cli.root.PersistentFlags().StringP("p2p", "p", string(MixedP2PMode),
 		"P2P typology to be generated [\"new\", \"legacy\", \"hybrid\" or \"mixed\" ]")
+	cli.root.PersistentFlags().IntP("min-size", "m", 1, "Minimum Network Size")
 
 	return cli
 }
 
 // generate generates manifests in a directory.
-func (cli *CLI) generate(dir string, groups int, opts Options) error {
-	err := os.MkdirAll(dir, 0755)
+func (cli *CLI) generate() error {
+	err := os.MkdirAll(cli.opts.Directory, 0755)
 	if err != nil {
 		return err
 	}
 
-	manifests, err := Generate(rand.New(rand.NewSource(randomSeed)), opts)
+	manifests, err := Generate(rand.New(rand.NewSource(randomSeed)), cli.opts)
 	if err != nil {
 		return err
 	}
-	if groups <= 0 {
-		for i, manifest := range manifests {
-			err = manifest.Save(filepath.Join(dir, fmt.Sprintf("gen-%04d.toml", i)))
-			if err != nil {
-				return err
-			}
+
+	switch {
+	case cli.opts.NumGroups <= 0:
+		e2e.SortManifests(manifests)
+
+		if err := e2e.WriteManifests(filepath.Join(cli.opts.Directory, "gen"), manifests); err != nil {
+			return err
 		}
-	} else {
-		groupSize := int(math.Ceil(float64(len(manifests)) / float64(groups)))
-		for g := 0; g < groups; g++ {
-			for i := 0; i < groupSize && g*groupSize+i < len(manifests); i++ {
-				manifest := manifests[g*groupSize+i]
-				err = manifest.Save(filepath.Join(dir, fmt.Sprintf("gen-group%02d-%04d.toml", g, i)))
-				if err != nil {
-					return err
-				}
+	default:
+		groupManifests := e2e.SplitGroups(cli.opts.NumGroups, manifests)
+
+		for idx, gm := range groupManifests {
+			e2e.SortManifests(gm)
+
+			prefix := filepath.Join(cli.opts.Directory, fmt.Sprintf("gen-group%02d", idx))
+			if err := e2e.WriteManifests(prefix, gm); err != nil {
+				return err
 			}
 		}
 	}
+
 	return nil
 }
 

test/e2e/pkg/manifest.go

@@ -170,41 +170,75 @@ func LoadManifest(file string) (Manifest, error) {
 }
 
 // SortManifests orders (in-place) a list of manifests such that the
-// manifests will be ordered (vaguely) from least complex to most
-// complex.
+// manifests will be ordered in terms of complexity (or expected
+// runtime). Complexity is determined first by the number of nodes,
+// and then by the total number of perturbations in the network
 func SortManifests(manifests []Manifest) {
 	sort.SliceStable(manifests, func(i, j int) bool {
-		left, right := manifests[i], manifests[j]
-
-		if len(left.Nodes) < len(right.Nodes) {
-			return true
-		}
-
-		if left.InitialHeight < right.InitialHeight {
-			return true
-		}
-
-		if left.TxSize < right.TxSize {
-			return true
-		}
-
-		if left.Evidence < right.Evidence {
-			return true
-		}
-
+		// sort based on a point-based comparison between two
+		// manifests.
 		var (
-			leftPerturb  int
-			rightPerturb int
+			left  = manifests[i]
+			right = manifests[j]
 		)
 
+		// scores start with 100 points for each node. The
+		// number of nodes in a network is the most important
+		// factor in the complexity of the test.
+		leftScore := len(left.Nodes) * 100
+		rightScore := len(right.Nodes) * 100
+
+		// add two points for every node perturbation, and one
+		// point for every node that starts after genesis.
 		for _, n := range left.Nodes {
-			leftPerturb += len(n.Perturb)
+			leftScore += (len(n.Perturb) * 2)
+
+			if n.StartAt > 0 {
+				leftScore++
+			}
 		}
 		for _, n := range right.Nodes {
-			rightPerturb += len(n.Perturb)
+			rightScore += (len(n.Perturb) * 2)
+			if n.StartAt > 0 {
+				rightScore++
+			}
 		}
 
-		return leftPerturb < rightPerturb
+		// add one point if the network has evidence.
+		if left.Evidence > 0 {
+			leftScore++
+		}
+		if right.Evidence > 0 {
+			rightScore++
+		}
 
+		return leftScore < rightScore
 	})
 }
+
+// SplitGroups divides a list of manifests into n groups of
+// manifests.
+func SplitGroups(groups int, manifests []Manifest) [][]Manifest {
+	groupSize := (len(manifests) + groups - 1) / groups
+	splitManifests := make([][]Manifest, 0, groups)
+
+	for i := 0; i < len(manifests); i += groupSize {
+		grp := make([]Manifest, groupSize)
+		n := copy(grp, manifests[i:])
+		splitManifests = append(splitManifests, grp[:n])
+	}
+
+	return splitManifests
+}
+
+// WriteManifests writes a collection of manifests into files with the
+// specified path prefix.
+func WriteManifests(prefix string, manifests []Manifest) error {
+	for i, manifest := range manifests {
+		if err := manifest.Save(fmt.Sprintf("%s-%04d.toml", prefix, i)); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}