From 8a171b84261ed938ba3cc428169bf54026eaadb8 Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Thu, 23 Sep 2021 12:42:20 -0400 Subject: [PATCH] e2e: improve manifest sorting algorithim (#6979) --- test/e2e/generator/generate.go | 17 +++---- test/e2e/generator/main.go | 65 ++++++++++++++------------ test/e2e/pkg/manifest.go | 84 ++++++++++++++++++++++++---------- 3 files changed, 105 insertions(+), 61 deletions(-) diff --git a/test/e2e/generator/generate.go b/test/e2e/generator/generate.go index 2d6945e65..78da8ed4a 100644 --- a/test/e2e/generator/generate.go +++ b/test/e2e/generator/generate.go @@ -80,26 +80,27 @@ func Generate(r *rand.Rand, opts Options) ([]e2e.Manifest, error) { return nil, err } + if len(manifest.Nodes) < opts.MinNetworkSize { + continue + } + if len(manifest.Nodes) == 1 { if opt["p2p"] == HybridP2PMode { continue } } - manifests = append(manifests, manifest) - } - if opts.Sorted { - // When the sorted flag is set (generally, as long as - // groups aren't set), - e2e.SortManifests(manifests) + manifests = append(manifests, manifest) } return manifests, nil } type Options struct { - P2P P2PMode - Sorted bool + MinNetworkSize int + NumGroups int + Directory string + P2P P2PMode } type P2PMode string diff --git a/test/e2e/generator/main.go b/test/e2e/generator/main.go index 7dd096760..8ec373944 100644 --- a/test/e2e/generator/main.go +++ b/test/e2e/generator/main.go @@ -3,7 +3,6 @@ package main import ( "fmt" - "math" "math/rand" "os" "path/filepath" @@ -11,6 +10,7 @@ import ( "github.com/spf13/cobra" "github.com/tendermint/tendermint/libs/log" + e2e "github.com/tendermint/tendermint/test/e2e/pkg" ) const ( @@ -26,6 +26,7 @@ func main() { // CLI is the Cobra-based command-line interface. type CLI struct { root *cobra.Command + opts Options } // NewCLI sets up the CLI. @@ -37,31 +38,36 @@ func NewCLI() *CLI { SilenceUsage: true, SilenceErrors: true, // we'll output them ourselves in Run() RunE: func(cmd *cobra.Command, args []string) error { - dir, err := cmd.Flags().GetString("dir") + var opts Options + var err error + + cli.opts.Directory, err = cmd.Flags().GetString("dir") + if err != nil { + return err + } + + cli.opts.NumGroups, err = cmd.Flags().GetInt("groups") if err != nil { return err } - groups, err := cmd.Flags().GetInt("groups") + + cli.opts.MinNetworkSize, err = cmd.Flags().GetInt("min-size") if err != nil { return err } + p2pMode, err := cmd.Flags().GetString("p2p") if err != nil { return err } - var opts Options switch mode := P2PMode(p2pMode); mode { case NewP2PMode, LegacyP2PMode, HybridP2PMode, MixedP2PMode: - opts = Options{P2P: mode} + opts.P2P = mode default: return fmt.Errorf("p2p mode must be either new, legacy, hybrid or mixed got %s", p2pMode) } - if groups == 0 { - opts.Sorted = true - } - - return cli.generate(dir, groups, opts) + return cli.generate() }, } @@ -70,40 +76,43 @@ func NewCLI() *CLI { cli.root.PersistentFlags().IntP("groups", "g", 0, "Number of groups") cli.root.PersistentFlags().StringP("p2p", "p", string(MixedP2PMode), "P2P typology to be generated [\"new\", \"legacy\", \"hybrid\" or \"mixed\" ]") + cli.root.PersistentFlags().IntP("min-size", "m", 1, "Minimum Network Size") return cli } // generate generates manifests in a directory. -func (cli *CLI) generate(dir string, groups int, opts Options) error { - err := os.MkdirAll(dir, 0755) +func (cli *CLI) generate() error { + err := os.MkdirAll(cli.opts.Directory, 0755) if err != nil { return err } - manifests, err := Generate(rand.New(rand.NewSource(randomSeed)), opts) + manifests, err := Generate(rand.New(rand.NewSource(randomSeed)), cli.opts) if err != nil { return err } - if groups <= 0 { - for i, manifest := range manifests { - err = manifest.Save(filepath.Join(dir, fmt.Sprintf("gen-%04d.toml", i))) - if err != nil { - return err - } + + switch { + case cli.opts.NumGroups <= 0: + e2e.SortManifests(manifests) + + if err := e2e.WriteManifests(filepath.Join(cli.opts.Directory, "gen"), manifests); err != nil { + return err } - } else { - groupSize := int(math.Ceil(float64(len(manifests)) / float64(groups))) - for g := 0; g < groups; g++ { - for i := 0; i < groupSize && g*groupSize+i < len(manifests); i++ { - manifest := manifests[g*groupSize+i] - err = manifest.Save(filepath.Join(dir, fmt.Sprintf("gen-group%02d-%04d.toml", g, i))) - if err != nil { - return err - } + default: + groupManifests := e2e.SplitGroups(cli.opts.NumGroups, manifests) + + for idx, gm := range groupManifests { + e2e.SortManifests(gm) + + prefix := filepath.Join(cli.opts.Directory, fmt.Sprintf("gen-group%02d", idx)) + if err := e2e.WriteManifests(prefix, gm); err != nil { + return err } } } + return nil } diff --git a/test/e2e/pkg/manifest.go b/test/e2e/pkg/manifest.go index 2a8f73127..86e571ee1 100644 --- a/test/e2e/pkg/manifest.go +++ b/test/e2e/pkg/manifest.go @@ -170,41 +170,75 @@ func LoadManifest(file string) (Manifest, error) { } // SortManifests orders (in-place) a list of manifests such that the -// manifests will be ordered (vaguely) from least complex to most -// complex. +// manifests will be ordered in terms of complexity (or expected +// runtime). Complexity is determined first by the number of nodes, +// and then by the total number of perturbations in the network func SortManifests(manifests []Manifest) { sort.SliceStable(manifests, func(i, j int) bool { - left, right := manifests[i], manifests[j] - - if len(left.Nodes) < len(right.Nodes) { - return true - } - - if left.InitialHeight < right.InitialHeight { - return true - } - - if left.TxSize < right.TxSize { - return true - } - - if left.Evidence < right.Evidence { - return true - } - + // sort based on a point-based comparison between two + // manifests. var ( - leftPerturb int - rightPerturb int + left = manifests[i] + right = manifests[j] ) + // scores start with 100 points for each node. The + // number of nodes in a network is the most important + // factor in the complexity of the test. + leftScore := len(left.Nodes) * 100 + rightScore := len(right.Nodes) * 100 + + // add two points for every node perturbation, and one + // point for every node that starts after genesis. for _, n := range left.Nodes { - leftPerturb += len(n.Perturb) + leftScore += (len(n.Perturb) * 2) + + if n.StartAt > 0 { + leftScore++ + } } for _, n := range right.Nodes { - rightPerturb += len(n.Perturb) + rightScore += (len(n.Perturb) * 2) + if n.StartAt > 0 { + rightScore++ + } } - return leftPerturb < rightPerturb + // add one point if the network has evidence. + if left.Evidence > 0 { + leftScore++ + } + if right.Evidence > 0 { + rightScore++ + } + return leftScore < rightScore }) } + +// SplitGroups divides a list of manifests into n groups of +// manifests. +func SplitGroups(groups int, manifests []Manifest) [][]Manifest { + groupSize := (len(manifests) + groups - 1) / groups + splitManifests := make([][]Manifest, 0, groups) + + for i := 0; i < len(manifests); i += groupSize { + grp := make([]Manifest, groupSize) + n := copy(grp, manifests[i:]) + splitManifests = append(splitManifests, grp[:n]) + } + + return splitManifests +} + +// WriteManifests writes a collection of manifests into files with the +// specified path prefix. +func WriteManifests(prefix string, manifests []Manifest) error { + for i, manifest := range manifests { + if err := manifest.Save(fmt.Sprintf("%s-%04d.toml", prefix, i)); err != nil { + return err + } + } + + return nil +}