Browse Source

test: add basic end-to-end test cases (#5450)

Partial fix for #5291.

This adds a basic set of test cases for core network invariants. Although small, it is sufficient to replace and extend the current set of P2P tests. Further test cases can be added later.
pull/5561/head
Erik Grinaker 4 years ago
committed by Erik Grinaker
parent
commit
64b0f5b363
4 changed files with 386 additions and 16 deletions
  1. +54
    -4
      test/e2e/tests/app_test.go
  2. +82
    -0
      test/e2e/tests/block_test.go
  3. +89
    -12
      test/e2e/tests/e2e_test.go
  4. +161
    -0
      test/e2e/tests/validator_test.go

+ 54
- 4
test/e2e/tests/app_test.go View File

@ -1,20 +1,21 @@
package e2e_test
import (
"fmt"
"math/rand"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
"github.com/tendermint/tendermint/types"
)
// Tests that any initial state given in genesis has made it into the app.
func TestApp_InitialState(t *testing.T) {
testNode(t, func(t *testing.T, node e2e.Node) {
switch {
case node.Mode == e2e.ModeSeed:
return
case len(node.Testnet.InitialState) == 0:
if len(node.Testnet.InitialState) == 0 {
return
}
@ -28,3 +29,52 @@ func TestApp_InitialState(t *testing.T) {
}
})
}
// Tests that the app hash (as reported by the app) matches the last
// block and the node sync status.
func TestApp_Hash(t *testing.T) {
testNode(t, func(t *testing.T, node e2e.Node) {
client, err := node.Client()
require.NoError(t, err)
info, err := client.ABCIInfo(ctx)
require.NoError(t, err)
require.NotEmpty(t, info.Response.LastBlockAppHash, "expected app to return app hash")
block, err := client.Block(ctx, nil)
require.NoError(t, err)
require.EqualValues(t, info.Response.LastBlockAppHash, block.Block.AppHash,
"app hash does not match last block's app hash")
status, err := client.Status(ctx)
require.NoError(t, err)
require.EqualValues(t, info.Response.LastBlockAppHash, status.SyncInfo.LatestAppHash,
"app hash does not match node status")
})
}
// Tests that we can set a value and retrieve it.
func TestApp_Tx(t *testing.T) {
testNode(t, func(t *testing.T, node e2e.Node) {
client, err := node.Client()
require.NoError(t, err)
// Generate a random value, to prevent duplicate tx errors when
// manually running the test multiple times for a testnet.
r := rand.New(rand.NewSource(time.Now().UnixNano()))
bz := make([]byte, 32)
_, err = r.Read(bz)
require.NoError(t, err)
key := fmt.Sprintf("testapp-tx-%v", node.Name)
value := fmt.Sprintf("%x", bz)
tx := types.Tx(fmt.Sprintf("%v=%v", key, value))
_, err = client.BroadcastTxCommit(ctx, tx)
require.NoError(t, err)
resp, err := client.ABCIQuery(ctx, "", []byte(key))
require.NoError(t, err)
assert.Equal(t, key, string(resp.Response.Key))
assert.Equal(t, value, string(resp.Response.Value))
})
}

+ 82
- 0
test/e2e/tests/block_test.go View File

@ -0,0 +1,82 @@
package e2e_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
)
// Tests that block headers are identical across nodes where present.
func TestBlock_Header(t *testing.T) {
blocks := fetchBlockChain(t)
testNode(t, func(t *testing.T, node e2e.Node) {
client, err := node.Client()
require.NoError(t, err)
status, err := client.Status(ctx)
require.NoError(t, err)
first := status.SyncInfo.EarliestBlockHeight
last := status.SyncInfo.LatestBlockHeight
if node.RetainBlocks > 0 {
first++ // avoid race conditions with block pruning
}
for _, block := range blocks {
if block.Header.Height < first {
continue
}
if block.Header.Height > last {
break
}
resp, err := client.Block(ctx, &block.Header.Height)
require.NoError(t, err)
require.Equal(t, block, resp.Block,
"block mismatch for height %v", block.Header.Height)
}
})
}
// Tests that the node contains the expected block range.
func TestBlock_Range(t *testing.T) {
testNode(t, func(t *testing.T, node e2e.Node) {
client, err := node.Client()
require.NoError(t, err)
status, err := client.Status(ctx)
require.NoError(t, err)
first := status.SyncInfo.EarliestBlockHeight
last := status.SyncInfo.LatestBlockHeight
switch {
case node.StateSync:
assert.Greater(t, first, node.Testnet.InitialHeight,
"state synced nodes should not contain network's initial height")
case node.RetainBlocks > 0 && int64(node.RetainBlocks) < (last-node.Testnet.InitialHeight+1):
// Delta handles race conditions in reading first/last heights.
assert.InDelta(t, node.RetainBlocks, last-first+1, 1,
"node not pruning expected blocks")
default:
assert.Equal(t, node.Testnet.InitialHeight, first,
"node's first block should be network's initial height")
}
for h := first; h <= last; h++ {
resp, err := client.Block(ctx, &(h))
if err != nil && node.RetainBlocks > 0 && h == first {
// Ignore errors in first block if node is pruning blocks due to race conditions.
continue
}
require.NoError(t, err)
assert.Equal(t, h, resp.Block.Height)
}
for h := node.Testnet.InitialHeight; h < first; h++ {
_, err := client.Block(ctx, &(h))
require.Error(t, err)
}
})
}

+ 89
- 12
test/e2e/tests/e2e_test.go View File

@ -4,21 +4,29 @@ import (
"context"
"os"
"path/filepath"
"sync"
"testing"
"github.com/stretchr/testify/require"
rpchttp "github.com/tendermint/tendermint/rpc/client/http"
rpctypes "github.com/tendermint/tendermint/rpc/core/types"
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
"github.com/tendermint/tendermint/types"
)
func init() {
// This can be used to manually specify a testnet manifest and/or node to
// run tests against. The testnet must have been started by the runner first.
//os.Setenv("E2E_MANIFEST", "networks/simple.toml")
//os.Setenv("E2E_NODE", "validator01")
// os.Setenv("E2E_MANIFEST", "networks/ci.toml")
// os.Setenv("E2E_NODE", "validator01")
}
var (
ctx = context.Background()
ctx = context.Background()
testnetCache = map[string]e2e.Testnet{}
testnetCacheMtx = sync.Mutex{}
blocksCache = map[string][]*types.Block{}
blocksCacheMtx = sync.Mutex{}
)
// testNode runs tests for testnet nodes. The callback function is given a
@ -29,16 +37,9 @@ var (
// test runs. If E2E_NODE is also set, only the specified node is tested,
// otherwise all nodes are tested.
func testNode(t *testing.T, testFunc func(*testing.T, e2e.Node)) {
manifest := os.Getenv("E2E_MANIFEST")
if manifest == "" {
t.Skip("E2E_MANIFEST not set, not an end-to-end test run")
}
if !filepath.IsAbs(manifest) {
manifest = filepath.Join("..", manifest)
}
t.Helper()
testnet, err := e2e.LoadTestnet(manifest)
require.NoError(t, err)
testnet := loadTestnet(t)
nodes := testnet.Nodes
if name := os.Getenv("E2E_NODE"); name != "" {
@ -55,3 +56,79 @@ func testNode(t *testing.T, testFunc func(*testing.T, e2e.Node)) {
})
}
}
// loadTestnet loads the testnet based on the E2E_MANIFEST envvar.
func loadTestnet(t *testing.T) e2e.Testnet {
t.Helper()
manifest := os.Getenv("E2E_MANIFEST")
if manifest == "" {
t.Skip("E2E_MANIFEST not set, not an end-to-end test run")
}
if !filepath.IsAbs(manifest) {
manifest = filepath.Join("..", manifest)
}
testnetCacheMtx.Lock()
defer testnetCacheMtx.Unlock()
if testnet, ok := testnetCache[manifest]; ok {
return testnet
}
testnet, err := e2e.LoadTestnet(manifest)
require.NoError(t, err)
testnetCache[manifest] = *testnet
return *testnet
}
// fetchBlockChain fetches a complete, up-to-date block history from
// the freshest testnet archive node.
func fetchBlockChain(t *testing.T) []*types.Block {
t.Helper()
testnet := loadTestnet(t)
// Find the freshest archive node
var (
client *rpchttp.HTTP
status *rpctypes.ResultStatus
)
for _, node := range testnet.ArchiveNodes() {
c, err := node.Client()
require.NoError(t, err)
s, err := c.Status(ctx)
require.NoError(t, err)
if status == nil || s.SyncInfo.LatestBlockHeight > status.SyncInfo.LatestBlockHeight {
client = c
status = s
}
}
require.NotNil(t, client, "couldn't find an archive node")
// Fetch blocks. Look for existing block history in the block cache, and
// extend it with any new blocks that have been produced.
blocksCacheMtx.Lock()
defer blocksCacheMtx.Unlock()
from := status.SyncInfo.EarliestBlockHeight
to := status.SyncInfo.LatestBlockHeight
blocks, ok := blocksCache[testnet.Name]
if !ok {
blocks = make([]*types.Block, 0, to-from+1)
}
if len(blocks) > 0 {
from = blocks[len(blocks)-1].Height + 1
}
for h := from; h <= to; h++ {
resp, err := client.Block(ctx, &(h))
require.NoError(t, err)
require.NotNil(t, resp.Block)
require.Equal(t, h, resp.Block.Height, "unexpected block height %v", resp.Block.Height)
blocks = append(blocks, resp.Block)
}
require.NotEmpty(t, blocks, "blockchain does not contain any blocks")
blocksCache[testnet.Name] = blocks
return blocks
}

+ 161
- 0
test/e2e/tests/validator_test.go View File

@ -0,0 +1,161 @@
package e2e_test
import (
"bytes"
"testing"
"github.com/stretchr/testify/require"
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
"github.com/tendermint/tendermint/types"
)
// Tests that validator sets are available and correct according to
// scheduled validator updates.
func TestValidator_Sets(t *testing.T) {
testNode(t, func(t *testing.T, node e2e.Node) {
client, err := node.Client()
require.NoError(t, err)
status, err := client.Status(ctx)
require.NoError(t, err)
first := status.SyncInfo.EarliestBlockHeight
last := status.SyncInfo.LatestBlockHeight
// skip first block if node is pruning blocks, to avoid race conditions
if node.RetainBlocks > 0 {
first++
}
valSchedule := newValidatorSchedule(*node.Testnet)
valSchedule.Increment(first - node.Testnet.InitialHeight)
for h := first; h <= last; h++ {
validators := []*types.Validator{}
perPage := 100
for page := 1; ; page++ {
resp, err := client.Validators(ctx, &(h), &(page), &perPage)
require.NoError(t, err)
validators = append(validators, resp.Validators...)
if len(validators) == resp.Total {
break
}
}
require.Equal(t, valSchedule.Set.Validators, validators,
"incorrect validator set at height %v", h)
valSchedule.Increment(1)
}
})
}
// Tests that a validator proposes blocks when it's supposed to. It tolerates some
// missed blocks, e.g. due to testnet perturbations.
func TestValidator_Propose(t *testing.T) {
blocks := fetchBlockChain(t)
testNode(t, func(t *testing.T, node e2e.Node) {
if node.Mode != e2e.ModeValidator {
return
}
address := node.Key.PubKey().Address()
valSchedule := newValidatorSchedule(*node.Testnet)
expectCount := 0
proposeCount := 0
for _, block := range blocks {
if bytes.Equal(valSchedule.Set.Proposer.Address, address) {
expectCount++
if bytes.Equal(block.ProposerAddress, address) {
proposeCount++
}
}
valSchedule.Increment(1)
}
require.False(t, proposeCount == 0 && expectCount > 0,
"node did not propose any blocks (expected %v)", expectCount)
require.Less(t, expectCount-proposeCount, 5,
"validator missed proposing too many blocks (proposed %v out of %v)", proposeCount, expectCount)
})
}
// Tests that a validator signs blocks when it's supposed to. It tolerates some
// missed blocks, e.g. due to testnet perturbations.
func TestValidator_Sign(t *testing.T) {
blocks := fetchBlockChain(t)
testNode(t, func(t *testing.T, node e2e.Node) {
if node.Mode != e2e.ModeValidator {
return
}
address := node.Key.PubKey().Address()
valSchedule := newValidatorSchedule(*node.Testnet)
expectCount := 0
signCount := 0
for _, block := range blocks[1:] { // Skip first block, since it has no signatures
signed := false
for _, sig := range block.LastCommit.Signatures {
if bytes.Equal(sig.ValidatorAddress, address) {
signed = true
break
}
}
if valSchedule.Set.HasAddress(address) {
expectCount++
if signed {
signCount++
}
} else {
require.False(t, signed, "unexpected signature for block %v", block.LastCommit.Height)
}
valSchedule.Increment(1)
}
require.False(t, signCount == 0 && expectCount > 0,
"node did not sign any blocks (expected %v)", expectCount)
require.Less(t, float64(expectCount-signCount)/float64(expectCount), 0.5,
"validator missed signing too many blocks (signed %v out of %v)", signCount, expectCount)
})
}
// validatorSchedule is a validator set iterator, which takes into account
// validator set updates.
type validatorSchedule struct {
Set *types.ValidatorSet
height int64
updates map[int64]map[*e2e.Node]int64
}
func newValidatorSchedule(testnet e2e.Testnet) *validatorSchedule {
valMap := testnet.Validators // genesis validators
if v, ok := testnet.ValidatorUpdates[0]; ok { // InitChain validators
valMap = v
}
return &validatorSchedule{
height: testnet.InitialHeight,
Set: types.NewValidatorSet(makeVals(valMap)),
updates: testnet.ValidatorUpdates,
}
}
func (s *validatorSchedule) Increment(heights int64) {
for i := int64(0); i < heights; i++ {
s.height++
if s.height > 2 {
// validator set updates are offset by 2, since they only take effect
// two blocks after they're returned.
if update, ok := s.updates[s.height-2]; ok {
if err := s.Set.UpdateWithChangeSet(makeVals(update)); err != nil {
panic(err)
}
}
}
s.Set.IncrementProposerPriority(1)
}
}
func makeVals(valMap map[*e2e.Node]int64) []*types.Validator {
vals := make([]*types.Validator, 0, len(valMap))
for node, power := range valMap {
vals = append(vals, types.NewValidator(node.Key.PubKey(), power))
}
return vals
}

Loading…
Cancel
Save