From cbae3613dd19198d609b282c15134e50f25c0d9f Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Mon, 29 Mar 2021 20:00:20 +0200 Subject: [PATCH] e2e: add evidence generation and testing (#6276) --- .gitignore | 1 - CONTRIBUTING.md | 9 - test/e2e/Makefile | 7 +- test/e2e/app/main.go | 41 ----- test/e2e/docker/Dockerfile | 2 - test/e2e/docker/entrypoint-maverick | 10 - test/e2e/generator/generate.go | 36 +--- test/e2e/generator/random.go | 22 --- test/e2e/networks/ci.toml | 14 +- test/e2e/pkg/manifest.go | 18 +- test/e2e/pkg/testnet.go | 57 +----- test/e2e/runner/evidence.go | 274 ++++++++++++++++++++++++++++ test/e2e/runner/main.go | 35 +++- test/e2e/runner/setup.go | 35 +--- test/e2e/tests/evidence_test.go | 51 +----- 15 files changed, 337 insertions(+), 275 deletions(-) delete mode 100755 test/e2e/docker/entrypoint-maverick create mode 100644 test/e2e/runner/evidence.go diff --git a/.gitignore b/.gitignore index 40bfa28d8..b2a29d086 100644 --- a/.gitignore +++ b/.gitignore @@ -38,7 +38,6 @@ terraform.tfstate.d test/e2e/build test/e2e/networks/*/ test/logs -test/maverick/maverick test/p2p/data/ vendor test/fuzz/**/corpus diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index acaa44867..c3cc05330 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -366,15 +366,6 @@ cd test/e2e && \ ./build/runner -f networks/ci.toml ``` -### Maverick - -**If you're changing the code in `consensus` package, please make sure to -replicate all the changes in `./test/maverick/consensus`**. Maverick is a -byzantine node used to assert that the validator gets punished for malicious -behavior. - -See [README](./test/maverick/README.md) for details. - ### Model-based tests (ADVANCED) *NOTE: if you're just submitting your first PR, you won't need to touch these diff --git a/test/e2e/Makefile b/test/e2e/Makefile index c9eb8bc19..38ce809e6 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -8,11 +8,6 @@ docker: # ABCI testing). app: go build -o build/app -tags badgerdb,boltdb,cleveldb,rocksdb ./app - -# To be used primarily by the e2e docker instance. If you want to produce this binary -# elsewhere, then run go build in the maverick directory. -maverick: - go build -o build/maverick -tags badgerdb,boltdb,cleveldb,rocksdb ../maverick generator: go build -o build/generator ./generator @@ -20,4 +15,4 @@ generator: runner: go build -o build/runner ./runner -.PHONY: all app docker generator maverick runner +.PHONY: all app docker generator runner diff --git a/test/e2e/app/main.go b/test/e2e/app/main.go index 5141750ed..cb49b8303 100644 --- a/test/e2e/app/main.go +++ b/test/e2e/app/main.go @@ -83,10 +83,6 @@ func run(configFile string) error { default: err = startNode(cfg) } - // FIXME: Temporarily remove maverick until it is redesigned - // if len(cfg.Misbehaviors) == 0 { - // err = startMaverick(cfg) - // } default: err = fmt.Errorf("invalid protocol %q", cfg.Protocol) } @@ -227,43 +223,6 @@ func startLightNode(cfg *Config) error { return nil } -// FIXME: Temporarily disconnected maverick until it is redesigned -// startMaverick starts a Maverick node that runs the application directly. It assumes the Tendermint -// configuration is in $TMHOME/config/tendermint.toml. -// func startMaverick(cfg *Config) error { -// app, err := NewApplication(cfg) -// if err != nil { -// return err -// } - -// tmcfg, logger, nodeKey, err := setupNode() -// if err != nil { -// return fmt.Errorf("failed to setup config: %w", err) -// } - -// misbehaviors := make(map[int64]mcs.Misbehavior, len(cfg.Misbehaviors)) -// for heightString, misbehaviorString := range cfg.Misbehaviors { -// height, _ := strconv.ParseInt(heightString, 10, 64) -// misbehaviors[height] = mcs.MisbehaviorList[misbehaviorString] -// } - -// n, err := maverick.NewNode(tmcfg, -// maverick.LoadOrGenFilePV(tmcfg.PrivValidatorKeyFile(), tmcfg.PrivValidatorStateFile()), -// *nodeKey, -// proxy.NewLocalClientCreator(app), -// maverick.DefaultGenesisDocProviderFunc(tmcfg), -// maverick.DefaultDBProvider, -// maverick.DefaultMetricsProvider(tmcfg.Instrumentation), -// logger, -// misbehaviors, -// ) -// if err != nil { -// return err -// } - -// return n.Start() -// } - // startSigner starts a signer server connecting to the given endpoint. func startSigner(cfg *Config) error { filePV, err := privval.LoadFilePV(cfg.PrivValKey, cfg.PrivValState) diff --git a/test/e2e/docker/Dockerfile b/test/e2e/docker/Dockerfile index d0f9cd7a2..8c76f1d5e 100644 --- a/test/e2e/docker/Dockerfile +++ b/test/e2e/docker/Dockerfile @@ -19,8 +19,6 @@ COPY . . RUN make build && cp build/tendermint /usr/bin/tendermint COPY test/e2e/docker/entrypoint* /usr/bin/ -# FIXME: Temporarily disconnect maverick node until it is redesigned -# RUN cd test/e2e && make maverick && cp build/maverick /usr/bin/maverick RUN cd test/e2e && make app && cp build/app /usr/bin/app # Set up runtime directory. We don't use a separate runtime image since we need diff --git a/test/e2e/docker/entrypoint-maverick b/test/e2e/docker/entrypoint-maverick deleted file mode 100755 index 9469e2447..000000000 --- a/test/e2e/docker/entrypoint-maverick +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -# Forcibly remove any stray UNIX sockets left behind from previous runs -rm -rf /var/run/privval.sock /var/run/app.sock - -/usr/bin/app /tendermint/config/app.toml & - -sleep 1 - -/usr/bin/maverick "$@" diff --git a/test/e2e/generator/generate.go b/test/e2e/generator/generate.go index 29614fa3c..f71e71856 100644 --- a/test/e2e/generator/generate.go +++ b/test/e2e/generator/generate.go @@ -4,7 +4,6 @@ import ( "fmt" "math/rand" "sort" - "strconv" "strings" e2e "github.com/tendermint/tendermint/test/e2e/pkg" @@ -36,20 +35,14 @@ var ( nodeStateSyncs = uniformChoice{false, true} nodePersistIntervals = uniformChoice{0, 1, 5} nodeSnapshotIntervals = uniformChoice{0, 3} - nodeRetainBlocks = uniformChoice{0, 1, 5} + nodeRetainBlocks = uniformChoice{0, int(e2e.EvidenceAgeHeight), int(e2e.EvidenceAgeHeight) + 5} nodePerturbations = probSetChoice{ "disconnect": 0.1, "pause": 0.1, "kill": 0.1, "restart": 0.1, } - nodeMisbehaviors = weightedChoice{ - // FIXME: evidence disabled due to node panicing when not - // having sufficient block history to process evidence. - // https://github.com/tendermint/tendermint/issues/5617 - // misbehaviorOption{"double-prevote"}: 1, - misbehaviorOption{}: 9, - } + evidence = uniformChoice{0, 1, 10} ) // Generate generates random testnets using the given RNG. @@ -75,6 +68,7 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er ValidatorUpdates: map[string]map[string]int64{}, Nodes: map[string]*e2e.ManifestNode{}, KeyType: opt["keyType"].(string), + Evidence: evidence.Choose(r).(int), } var numSeeds, numValidators, numFulls, numLightClients int @@ -227,17 +221,6 @@ func generateNode( node.SnapshotInterval = 3 } - if node.Mode == string(e2e.ModeValidator) { - misbehaveAt := startAt + 5 + int64(r.Intn(10)) - if startAt == 0 { - misbehaveAt += initialHeight - 1 - } - node.Misbehaviors = nodeMisbehaviors.Choose(r).(misbehaviorOption).atHeight(misbehaveAt) - if len(node.Misbehaviors) != 0 { - node.PrivvalProtocol = "file" - } - } - // If a node which does not persist state also does not retain blocks, randomly // choose to either persist state or retain all blocks. if node.PersistInterval != nil && *node.PersistInterval == 0 && node.RetainBlocks > 0 { @@ -276,16 +259,3 @@ func generateLightNode(r *rand.Rand, startAt int64, providers []string) *e2e.Man func ptrUint64(i uint64) *uint64 { return &i } - -type misbehaviorOption struct { - misbehavior string -} - -func (m misbehaviorOption) atHeight(height int64) map[string]string { - misbehaviorMap := make(map[string]string) - if m.misbehavior == "" { - return misbehaviorMap - } - misbehaviorMap[strconv.Itoa(int(height))] = m.misbehavior - return misbehaviorMap -} diff --git a/test/e2e/generator/random.go b/test/e2e/generator/random.go index ec59a01b2..f21502118 100644 --- a/test/e2e/generator/random.go +++ b/test/e2e/generator/random.go @@ -56,28 +56,6 @@ func (uc uniformChoice) Choose(r *rand.Rand) interface{} { return uc[r.Intn(len(uc))] } -// weightedChoice chooses a single random key from a map of keys and weights. -type weightedChoice map[interface{}]uint - -func (wc weightedChoice) Choose(r *rand.Rand) interface{} { - total := 0 - choices := make([]interface{}, 0, len(wc)) - for choice, weight := range wc { - total += int(weight) - choices = append(choices, choice) - } - - rem := r.Intn(total) - for _, choice := range choices { - rem -= int(wc[choice]) - if rem <= 0 { - return choice - } - } - - return nil -} - // probSetChoice picks a set of strings based on each string's probability (0-1). type probSetChoice map[string]float64 diff --git a/test/e2e/networks/ci.toml b/test/e2e/networks/ci.toml index 879d0c22f..05875dace 100644 --- a/test/e2e/networks/ci.toml +++ b/test/e2e/networks/ci.toml @@ -2,6 +2,7 @@ # functionality with a single network. initial_height = 1000 +evidence = 0 initial_state = { initial01 = "a", initial02 = "b", initial03 = "c" } [validators] @@ -37,8 +38,6 @@ seeds = ["seed01"] seeds = ["seed01"] snapshot_interval = 5 perturb = ["disconnect"] -# FIXME: maverick has been disabled until it is redesigned (https://github.com/tendermint/tendermint/issues/5575) -# misbehaviors = { 1018 = "double-prevote" } [node.validator02] seeds = ["seed02"] @@ -80,7 +79,7 @@ mode = "full" # FIXME: should be v2, disabled due to flake fast_sync = "v0" persistent_peers = ["validator01", "validator02", "validator03", "validator04", "validator05"] -retain_blocks = 1 +retain_blocks = 3 perturb = ["restart"] [node.full02] @@ -94,10 +93,5 @@ perturb = ["restart"] [node.light01] mode= "light" -start_at= 1005 -persistent_peers = ["validator01", "validator02", "validator03"] - -[node.light02] -mode= "light" -start_at= 1015 -persistent_peers = ["validator04", "full01", "validator05"] \ No newline at end of file +start_at= 1010 +persistent_peers = ["validator01", "validator02", "validator03"] \ No newline at end of file diff --git a/test/e2e/pkg/manifest.go b/test/e2e/pkg/manifest.go index 8fbaea185..84628594f 100644 --- a/test/e2e/pkg/manifest.go +++ b/test/e2e/pkg/manifest.go @@ -51,6 +51,10 @@ type Manifest struct { // Options are ed25519 & secp256k1 KeyType string `toml:"key_type"` + // Evidence indicates the amount of evidence that will be injected into the + // testnet via the RPC endpoint of a random node. Default is 0 + Evidence int `toml:"evidence"` + // LogLevel sets the log level of the entire testnet. This can be overridden // by individual nodes. LogLevel string `toml:"log_level"` @@ -113,8 +117,8 @@ type ManifestNode struct { SnapshotInterval uint64 `toml:"snapshot_interval"` // RetainBlocks specifies the number of recent blocks to retain. Defaults to - // 0, which retains all blocks. Must be greater that PersistInterval and - // SnapshotInterval. + // 0, which retains all blocks. Must be greater that PersistInterval, + // SnapshotInterval and EvidenceAgeHeight. RetainBlocks uint64 `toml:"retain_blocks"` // Perturb lists perturbations to apply to the node after it has been @@ -126,16 +130,6 @@ type ManifestNode struct { // restart: restarts the node, shutting it down with SIGTERM Perturb []string `toml:"perturb"` - // Misbehaviors sets how a validator behaves during consensus at a - // certain height. Multiple misbehaviors at different heights can be used - // - // An example of misbehaviors - // { 10 = "double-prevote", 20 = "double-prevote"} - // - // For more information, look at the readme in the maverick folder. - // A list of all behaviors can be found in ../maverick/consensus/behavior.go - Misbehaviors map[string]string `toml:"misbehaviors"` - // Log level sets the log level of the specific node i.e. "consensus:info,*:error". // This is helpful when debugging a specific problem. This overrides the network // level. diff --git a/test/e2e/pkg/testnet.go b/test/e2e/pkg/testnet.go index b773a158a..1e3eede90 100644 --- a/test/e2e/pkg/testnet.go +++ b/test/e2e/pkg/testnet.go @@ -11,6 +11,7 @@ import ( "sort" "strconv" "strings" + "time" "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/crypto/ed25519" @@ -46,6 +47,9 @@ const ( PerturbationKill Perturbation = "kill" PerturbationPause Perturbation = "pause" PerturbationRestart Perturbation = "restart" + + EvidenceAgeHeight int64 = 3 + EvidenceAgeTime time.Duration = 10 * time.Second ) // Testnet represents a single testnet. @@ -60,6 +64,7 @@ type Testnet struct { ValidatorUpdates map[int64]map[*Node]int64 Nodes []*Node KeyType string + Evidence int LogLevel string } @@ -84,7 +89,6 @@ type Node struct { Seeds []*Node PersistentPeers []*Node Perturbations []Perturbation - Misbehaviors map[int64]string LogLevel string } @@ -124,6 +128,7 @@ func LoadTestnet(file string) (*Testnet, error) { Validators: map[*Node]int64{}, ValidatorUpdates: map[int64]map[*Node]int64{}, Nodes: []*Node{}, + Evidence: manifest.Evidence, KeyType: "ed25519", LogLevel: manifest.LogLevel, } @@ -161,7 +166,6 @@ func LoadTestnet(file string) (*Testnet, error) { SnapshotInterval: nodeManifest.SnapshotInterval, RetainBlocks: nodeManifest.RetainBlocks, Perturbations: []Perturbation{}, - Misbehaviors: make(map[int64]string), LogLevel: manifest.LogLevel, } if node.StartAt == testnet.InitialHeight { @@ -185,13 +189,6 @@ func LoadTestnet(file string) (*Testnet, error) { for _, p := range nodeManifest.Perturb { node.Perturbations = append(node.Perturbations, Perturbation(p)) } - for heightString, misbehavior := range nodeManifest.Misbehaviors { - height, err := strconv.ParseInt(heightString, 10, 64) - if err != nil { - return nil, fmt.Errorf("unable to parse height %s to int64: %w", heightString, err) - } - node.Misbehaviors[height] = misbehavior - } if nodeManifest.LogLevel != "" { node.LogLevel = nodeManifest.LogLevel } @@ -344,6 +341,10 @@ func (n Node) Validate(testnet Testnet) error { if n.StateSync && n.StartAt == 0 { return errors.New("state synced nodes cannot start at the initial height") } + if n.RetainBlocks != 0 && n.RetainBlocks < uint64(EvidenceAgeHeight) { + return fmt.Errorf("retain_blocks must be greater or equal to max evidence age (%d)", + EvidenceAgeHeight) + } if n.PersistInterval == 0 && n.RetainBlocks > 0 { return errors.New("persist_interval=0 requires retain_blocks=0") } @@ -362,31 +363,6 @@ func (n Node) Validate(testnet Testnet) error { } } - if (n.PrivvalProtocol != "file" || n.Mode != "validator") && len(n.Misbehaviors) != 0 { - return errors.New("must be using \"file\" privval protocol to implement misbehaviors") - } - - for height, misbehavior := range n.Misbehaviors { - if height < n.StartAt { - return fmt.Errorf("misbehavior height %d is below node start height %d", - height, n.StartAt) - } - if height < testnet.InitialHeight { - return fmt.Errorf("misbehavior height %d is below network initial height %d", - height, testnet.InitialHeight) - } - exists := false - // FIXME: Maverick has been disabled until it is redesigned - // for possibleBehaviors := range mcs.MisbehaviorList { - // if possibleBehaviors == misbehavior { - // exists = true - // } - // } - if !exists { - return fmt.Errorf("misbehavior %s does not exist", misbehavior) - } - } - return nil } @@ -438,19 +414,6 @@ func (t Testnet) HasPerturbations() bool { return false } -// LastMisbehaviorHeight returns the height of the last misbehavior. -func (t Testnet) LastMisbehaviorHeight() int64 { - lastHeight := int64(0) - for _, node := range t.Nodes { - for height := range node.Misbehaviors { - if height > lastHeight { - lastHeight = height - } - } - } - return lastHeight -} - // Address returns a P2P endpoint address for the node. func (n Node) AddressP2P(withID bool) string { ip := n.IP.String() diff --git a/test/e2e/runner/evidence.go b/test/e2e/runner/evidence.go new file mode 100644 index 000000000..ece241cd2 --- /dev/null +++ b/test/e2e/runner/evidence.go @@ -0,0 +1,274 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "io/ioutil" + "math/rand" + "path/filepath" + "time" + + "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/crypto/tmhash" + tmjson "github.com/tendermint/tendermint/libs/json" + "github.com/tendermint/tendermint/privval" + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" + e2e "github.com/tendermint/tendermint/test/e2e/pkg" + "github.com/tendermint/tendermint/types" + "github.com/tendermint/tendermint/version" +) + +// 1 in 11 evidence is light client evidence, the rest is duplicate vote +// FIXME: Setting to 11 disables light client attack evidence since nodes +// don't follow a minimum retention height invariant. When we fix this we +// should use a ratio of 4. +const lightClientEvidenceRatio = 11 + +// InjectEvidence takes a running testnet and generates an amount of valid +// evidence and broadcasts it to a random node through the rpc endpoint `/broadcast_evidence`. +// Evidence is random and can be a mixture of LightClientAttackEvidence and +// DuplicateVoteEvidence. +func InjectEvidence(testnet *e2e.Testnet, amount int) error { + // select a random node + targetNode := testnet.RandomNode() + + logger.Info(fmt.Sprintf("Injecting evidence through %v (amount: %d)...", targetNode.Name, amount)) + + client, err := targetNode.Client() + if err != nil { + return err + } + + // request the latest block and validator set from the node + blockRes, err := client.Block(context.Background(), nil) + if err != nil { + return err + } + lightEvidenceCommonHeight := blockRes.Block.Height + waitHeight := blockRes.Block.Height + 3 + duplicateVoteHeight := waitHeight + + nValidators := 100 + valRes, err := client.Validators(context.Background(), &lightEvidenceCommonHeight, nil, &nValidators) + if err != nil { + return err + } + valSet, err := types.ValidatorSetFromExistingValidators(valRes.Validators) + if err != nil { + return err + } + + // get the private keys of all the validators in the network + privVals, err := getPrivateValidatorKeys(testnet) + if err != nil { + return err + } + + // wait for the node to reach the height above the forged height so that + // it is able to validate the evidence + status, err := waitForNode(targetNode, waitHeight, 10*time.Second) + if err != nil { + return err + } + duplicateVoteTime := status.SyncInfo.LatestBlockTime + + var ev types.Evidence + for i := 0; i < amount; i++ { + if i%lightClientEvidenceRatio == 0 { + ev, err = generateLightClientAttackEvidence( + privVals, lightEvidenceCommonHeight, valSet, testnet.Name, blockRes.Block.Time, + ) + } else { + ev, err = generateDuplicateVoteEvidence( + privVals, duplicateVoteHeight, valSet, testnet.Name, duplicateVoteTime, + ) + } + if err != nil { + return err + } + + _, err := client.BroadcastEvidence(context.Background(), ev) + if err != nil { + return err + } + } + + logger.Info(fmt.Sprintf("Finished sending evidence (height %d)", blockRes.Block.Height+2)) + + return nil +} + +func getPrivateValidatorKeys(testnet *e2e.Testnet) ([]types.MockPV, error) { + privVals := []types.MockPV{} + + for _, node := range testnet.Nodes { + if node.Mode == e2e.ModeValidator { + privKeyPath := filepath.Join(testnet.Dir, node.Name, PrivvalKeyFile) + privKey, err := readPrivKey(privKeyPath) + if err != nil { + return nil, err + } + // Create mock private validators from the validators private key. MockPV is + // stateless which means we can double vote and do other funky stuff + privVals = append(privVals, types.NewMockPVWithParams(privKey, false, false)) + } + } + + return privVals, nil +} + +// creates evidence of a lunatic attack. The height provided is the common height. +// The forged height happens 2 blocks later. +func generateLightClientAttackEvidence( + privVals []types.MockPV, + height int64, + vals *types.ValidatorSet, + chainID string, + evTime time.Time, +) (*types.LightClientAttackEvidence, error) { + // forge a random header + forgedHeight := height + 2 + forgedTime := evTime.Add(1 * time.Second) + header := makeHeaderRandom(chainID, forgedHeight) + header.Time = forgedTime + + // add a new bogus validator and remove an existing one to + // vary the validator set slightly + pv, conflictingVals, err := mutateValidatorSet(privVals, vals) + if err != nil { + return nil, err + } + + header.ValidatorsHash = conflictingVals.Hash() + + // create a commit for the forged header + blockID := makeBlockID(header.Hash(), 1000, []byte("partshash")) + voteSet := types.NewVoteSet(chainID, forgedHeight, 0, tmproto.SignedMsgType(2), conflictingVals) + commit, err := types.MakeCommit(blockID, forgedHeight, 0, voteSet, pv, forgedTime) + if err != nil { + return nil, err + } + + ev := &types.LightClientAttackEvidence{ + ConflictingBlock: &types.LightBlock{ + SignedHeader: &types.SignedHeader{ + Header: header, + Commit: commit, + }, + ValidatorSet: conflictingVals, + }, + CommonHeight: height, + TotalVotingPower: vals.TotalVotingPower(), + Timestamp: evTime, + } + ev.ByzantineValidators = ev.GetByzantineValidators(vals, &types.SignedHeader{ + Header: makeHeaderRandom(chainID, forgedHeight), + }) + return ev, nil +} + +// generateDuplicateVoteEvidence picks a random validator from the val set and +// returns duplicate vote evidence against the validator +func generateDuplicateVoteEvidence( + privVals []types.MockPV, + height int64, + vals *types.ValidatorSet, + chainID string, + time time.Time, +) (*types.DuplicateVoteEvidence, error) { + // nolint:gosec // G404: Use of weak random number generator + privVal := privVals[rand.Intn(len(privVals))] + voteA, err := types.MakeVote(height, makeRandomBlockID(), vals, privVal, chainID, time) + if err != nil { + return nil, err + } + voteB, err := types.MakeVote(height, makeRandomBlockID(), vals, privVal, chainID, time) + if err != nil { + return nil, err + } + return types.NewDuplicateVoteEvidence(voteA, voteB, time, vals), nil +} + +func readPrivKey(keyFilePath string) (crypto.PrivKey, error) { + keyJSONBytes, err := ioutil.ReadFile(keyFilePath) + if err != nil { + return nil, err + } + pvKey := privval.FilePVKey{} + err = tmjson.Unmarshal(keyJSONBytes, &pvKey) + if err != nil { + return nil, fmt.Errorf("error reading PrivValidator key from %v: %w", keyFilePath, err) + } + + return pvKey.PrivKey, nil +} + +func makeHeaderRandom(chainID string, height int64) *types.Header { + return &types.Header{ + Version: version.Consensus{Block: version.BlockProtocol, App: 1}, + ChainID: chainID, + Height: height, + Time: time.Now(), + LastBlockID: makeBlockID([]byte("headerhash"), 1000, []byte("partshash")), + LastCommitHash: crypto.CRandBytes(tmhash.Size), + DataHash: crypto.CRandBytes(tmhash.Size), + ValidatorsHash: crypto.CRandBytes(tmhash.Size), + NextValidatorsHash: crypto.CRandBytes(tmhash.Size), + ConsensusHash: crypto.CRandBytes(tmhash.Size), + AppHash: crypto.CRandBytes(tmhash.Size), + LastResultsHash: crypto.CRandBytes(tmhash.Size), + EvidenceHash: crypto.CRandBytes(tmhash.Size), + ProposerAddress: crypto.CRandBytes(crypto.AddressSize), + } +} + +func makeRandomBlockID() types.BlockID { + return makeBlockID(crypto.CRandBytes(tmhash.Size), 100, crypto.CRandBytes(tmhash.Size)) +} + +func makeBlockID(hash []byte, partSetSize uint32, partSetHash []byte) types.BlockID { + var ( + h = make([]byte, tmhash.Size) + psH = make([]byte, tmhash.Size) + ) + copy(h, hash) + copy(psH, partSetHash) + return types.BlockID{ + Hash: h, + PartSetHeader: types.PartSetHeader{ + Total: partSetSize, + Hash: psH, + }, + } +} + +func mutateValidatorSet(privVals []types.MockPV, vals *types.ValidatorSet, +) ([]types.PrivValidator, *types.ValidatorSet, error) { + newVal, newPrivVal := types.RandValidator(false, 10) + + var newVals *types.ValidatorSet + if vals.Size() > 2 { + newVals = types.NewValidatorSet(append(vals.Copy().Validators[:vals.Size()-1], newVal)) + } else { + newVals = types.NewValidatorSet(append(vals.Copy().Validators, newVal)) + } + + // we need to sort the priv validators with the same index as the validator set + pv := make([]types.PrivValidator, newVals.Size()) + for idx, val := range newVals.Validators { + found := false + for _, p := range append(privVals, newPrivVal.(types.MockPV)) { + if bytes.Equal(p.PrivKey.PubKey().Address(), val.Address) { + pv[idx] = p + found = true + break + } + } + if !found { + return nil, nil, fmt.Errorf("missing priv validator for %v", val.Address) + } + } + + return pv, newVals, nil +} diff --git a/test/e2e/runner/main.go b/test/e2e/runner/main.go index 993df98ef..35b0d952f 100644 --- a/test/e2e/runner/main.go +++ b/test/e2e/runner/main.go @@ -71,14 +71,6 @@ func NewCLI() *CLI { return err } - if lastMisbehavior := cli.testnet.LastMisbehaviorHeight(); lastMisbehavior > 0 { - // wait for misbehaviors before starting perturbations. We do a separate - // wait for another 5 blocks, since the last misbehavior height may be - // in the past depending on network startup ordering. - if err := WaitUntil(cli.testnet, lastMisbehavior); err != nil { - return err - } - } if err := Wait(cli.testnet, 5); err != nil { // allow some txs to go through return err } @@ -92,6 +84,15 @@ func NewCLI() *CLI { } } + if cli.testnet.Evidence > 0 { + if err := InjectEvidence(cli.testnet, cli.testnet.Evidence); err != nil { + return err + } + if err := Wait(cli.testnet, 1); err != nil { // ensure chain progress + return err + } + } + loadCancel() if err := <-chLoadResult; err != nil { return err @@ -188,6 +189,24 @@ func NewCLI() *CLI { }, }) + cli.root.AddCommand(&cobra.Command{ + Use: "evidence [amount]", + Args: cobra.MaximumNArgs(1), + Short: "Generates and broadcasts evidence to a random node", + RunE: func(cmd *cobra.Command, args []string) (err error) { + amount := 1 + + if len(args) == 1 { + amount, err = strconv.Atoi(args[0]) + if err != nil { + return err + } + } + + return InjectEvidence(cli.testnet, amount) + }, + }) + cli.root.AddCommand(&cobra.Command{ Use: "test", Short: "Runs test cases against a running testnet", diff --git a/test/e2e/runner/setup.go b/test/e2e/runner/setup.go index 6045afbe4..22ff58b15 100644 --- a/test/e2e/runner/setup.go +++ b/test/e2e/runner/setup.go @@ -12,7 +12,6 @@ import ( "path/filepath" "regexp" "sort" - "strconv" "strings" "text/template" "time" @@ -132,28 +131,6 @@ func Setup(testnet *e2e.Testnet) error { func MakeDockerCompose(testnet *e2e.Testnet) ([]byte, error) { // Must use version 2 Docker Compose format, to support IPv6. tmpl, err := template.New("docker-compose").Funcs(template.FuncMap{ - "startCommands": func(misbehaviors map[int64]string, logLevel string) string { - command := "start" - - // FIXME: Temporarily disable behaviors until maverick is redesigned - // misbehaviorString := "" - // for height, misbehavior := range misbehaviors { - // // after the first behavior set, a comma must be prepended - // if misbehaviorString != "" { - // misbehaviorString += "," - // } - // heightString := strconv.Itoa(int(height)) - // misbehaviorString += misbehavior + "," + heightString - // } - - // if misbehaviorString != "" { - // command += " --misbehaviors " + misbehaviorString - // } - if logLevel != "" && logLevel != config.DefaultLogLevel { - command += " --log-level " + logLevel - } - return command - }, "addUint32": func(x, y uint32) uint32 { return x + y }, @@ -181,8 +158,8 @@ services: image: tendermint/e2e-node {{- if eq .ABCIProtocol "builtin" }} entrypoint: /usr/bin/entrypoint-builtin -{{- else }} - command: {{ startCommands .Misbehaviors .LogLevel }} +{{- else if .LogLevel }} + command: start --log-level {{ .LogLevel }} {{- end }} init: true ports: @@ -223,6 +200,8 @@ func MakeGenesis(testnet *e2e.Testnet) (types.GenesisDoc, error) { default: return genesis, errors.New("unsupported KeyType") } + genesis.ConsensusParams.Evidence.MaxAgeNumBlocks = e2e.EvidenceAgeHeight + genesis.ConsensusParams.Evidence.MaxAgeDuration = e2e.EvidenceAgeTime for validator, power := range testnet.Validators { genesis.Validators = append(genesis.Validators, types.GenesisValidator{ Name: validator.Name, @@ -403,12 +382,6 @@ func MakeAppConfig(node *e2e.Node) ([]byte, error) { } } - misbehaviors := make(map[string]string) - for height, misbehavior := range node.Misbehaviors { - misbehaviors[strconv.Itoa(int(height))] = misbehavior - } - cfg["misbehaviors"] = misbehaviors - if len(node.Testnet.ValidatorUpdates) > 0 { validatorUpdates := map[string]map[string]int64{} for height, validators := range node.Testnet.ValidatorUpdates { diff --git a/test/e2e/tests/evidence_test.go b/test/e2e/tests/evidence_test.go index ea24b51e5..f7f2ede79 100644 --- a/test/e2e/tests/evidence_test.go +++ b/test/e2e/tests/evidence_test.go @@ -1,57 +1,22 @@ package e2e_test import ( - "bytes" "testing" "github.com/stretchr/testify/require" - - e2e "github.com/tendermint/tendermint/test/e2e/pkg" - "github.com/tendermint/tendermint/types" ) // assert that all nodes that have blocks at the height of a misbehavior has evidence // for that misbehavior func TestEvidence_Misbehavior(t *testing.T) { blocks := fetchBlockChain(t) - testNode(t, func(t *testing.T, node e2e.Node) { - seenEvidence := make(map[int64]struct{}) - for _, block := range blocks { - // Find any evidence blaming this node in this block - var nodeEvidence types.Evidence - for _, evidence := range block.Evidence.Evidence { - switch evidence := evidence.(type) { - case *types.DuplicateVoteEvidence: - if bytes.Equal(evidence.VoteA.ValidatorAddress, node.PrivvalKey.PubKey().Address()) { - nodeEvidence = evidence - } - default: - t.Fatalf("unexpected evidence type %T", evidence) - } - } - if nodeEvidence == nil { - continue // no evidence for the node at this height - } - - // Check that evidence was as expected - misbehavior, ok := node.Misbehaviors[nodeEvidence.Height()] - require.True(t, ok, "found unexpected evidence %v in height %v", - nodeEvidence, block.Height) - - switch misbehavior { - case "double-prevote": - require.IsType(t, &types.DuplicateVoteEvidence{}, nodeEvidence, "unexpected evidence type") - default: - t.Fatalf("unknown misbehavior %v", misbehavior) - } - - seenEvidence[nodeEvidence.Height()] = struct{}{} - } - // see if there is any evidence that we were expecting but didn't see - for height, misbehavior := range node.Misbehaviors { - _, ok := seenEvidence[height] - require.True(t, ok, "expected evidence for %v misbehavior at height %v by node but was never found", - misbehavior, height) + testnet := loadTestnet(t) + seenEvidence := 0 + for _, block := range blocks { + if len(block.Evidence.Evidence) != 0 { + seenEvidence += len(block.Evidence.Evidence) } - }) + } + require.Equal(t, testnet.Evidence, seenEvidence, + "difference between the amount of evidence produced and committed") }