test: add basic end-to-end test cases (#5450)

Partial fix for #5291.

This adds a basic set of test cases for core network invariants. Although small, it is sufficient to replace and extend the current set of P2P tests. Further test cases can be added later.

test/e2e/tests/app_test.go

@@ -1,20 +1,21 @@
 package e2e_test
 
 import (
+	"fmt"
+	"math/rand"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
+	"github.com/tendermint/tendermint/types"
 )
 
 // Tests that any initial state given in genesis has made it into the app.
 func TestApp_InitialState(t *testing.T) {
 	testNode(t, func(t *testing.T, node e2e.Node) {
-		switch {
-		case node.Mode == e2e.ModeSeed:
-			return
-		case len(node.Testnet.InitialState) == 0:
+		if len(node.Testnet.InitialState) == 0 {
 			return
 		}
 
@@ -28,3 +29,52 @@ func TestApp_InitialState(t *testing.T) {
 		}
 	})
 }
+
+// Tests that the app hash (as reported by the app) matches the last
+// block and the node sync status.
+func TestApp_Hash(t *testing.T) {
+	testNode(t, func(t *testing.T, node e2e.Node) {
+		client, err := node.Client()
+		require.NoError(t, err)
+		info, err := client.ABCIInfo(ctx)
+		require.NoError(t, err)
+		require.NotEmpty(t, info.Response.LastBlockAppHash, "expected app to return app hash")
+
+		block, err := client.Block(ctx, nil)
+		require.NoError(t, err)
+		require.EqualValues(t, info.Response.LastBlockAppHash, block.Block.AppHash,
+			"app hash does not match last block's app hash")
+
+		status, err := client.Status(ctx)
+		require.NoError(t, err)
+		require.EqualValues(t, info.Response.LastBlockAppHash, status.SyncInfo.LatestAppHash,
+			"app hash does not match node status")
+	})
+}
+
+// Tests that we can set a value and retrieve it.
+func TestApp_Tx(t *testing.T) {
+	testNode(t, func(t *testing.T, node e2e.Node) {
+		client, err := node.Client()
+		require.NoError(t, err)
+
+		// Generate a random value, to prevent duplicate tx errors when
+		// manually running the test multiple times for a testnet.
+		r := rand.New(rand.NewSource(time.Now().UnixNano()))
+		bz := make([]byte, 32)
+		_, err = r.Read(bz)
+		require.NoError(t, err)
+
+		key := fmt.Sprintf("testapp-tx-%v", node.Name)
+		value := fmt.Sprintf("%x", bz)
+		tx := types.Tx(fmt.Sprintf("%v=%v", key, value))
+
+		_, err = client.BroadcastTxCommit(ctx, tx)
+		require.NoError(t, err)
+
+		resp, err := client.ABCIQuery(ctx, "", []byte(key))
+		require.NoError(t, err)
+		assert.Equal(t, key, string(resp.Response.Key))
+		assert.Equal(t, value, string(resp.Response.Value))
+	})
+}

test/e2e/tests/block_test.go

@@ -0,0 +1,82 @@
+package e2e_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
+)
+
+// Tests that block headers are identical across nodes where present.
+func TestBlock_Header(t *testing.T) {
+	blocks := fetchBlockChain(t)
+	testNode(t, func(t *testing.T, node e2e.Node) {
+		client, err := node.Client()
+		require.NoError(t, err)
+		status, err := client.Status(ctx)
+		require.NoError(t, err)
+
+		first := status.SyncInfo.EarliestBlockHeight
+		last := status.SyncInfo.LatestBlockHeight
+		if node.RetainBlocks > 0 {
+			first++ // avoid race conditions with block pruning
+		}
+
+		for _, block := range blocks {
+			if block.Header.Height < first {
+				continue
+			}
+			if block.Header.Height > last {
+				break
+			}
+			resp, err := client.Block(ctx, &block.Header.Height)
+			require.NoError(t, err)
+			require.Equal(t, block, resp.Block,
+				"block mismatch for height %v", block.Header.Height)
+		}
+	})
+}
+
+// Tests that the node contains the expected block range.
+func TestBlock_Range(t *testing.T) {
+	testNode(t, func(t *testing.T, node e2e.Node) {
+		client, err := node.Client()
+		require.NoError(t, err)
+		status, err := client.Status(ctx)
+		require.NoError(t, err)
+
+		first := status.SyncInfo.EarliestBlockHeight
+		last := status.SyncInfo.LatestBlockHeight
+
+		switch {
+		case node.StateSync:
+			assert.Greater(t, first, node.Testnet.InitialHeight,
+				"state synced nodes should not contain network's initial height")
+
+		case node.RetainBlocks > 0 && int64(node.RetainBlocks) < (last-node.Testnet.InitialHeight+1):
+			// Delta handles race conditions in reading first/last heights.
+			assert.InDelta(t, node.RetainBlocks, last-first+1, 1,
+				"node not pruning expected blocks")
+
+		default:
+			assert.Equal(t, node.Testnet.InitialHeight, first,
+				"node's first block should be network's initial height")
+		}
+
+		for h := first; h <= last; h++ {
+			resp, err := client.Block(ctx, &(h))
+			if err != nil && node.RetainBlocks > 0 && h == first {
+				// Ignore errors in first block if node is pruning blocks due to race conditions.
+				continue
+			}
+			require.NoError(t, err)
+			assert.Equal(t, h, resp.Block.Height)
+		}
+
+		for h := node.Testnet.InitialHeight; h < first; h++ {
+			_, err := client.Block(ctx, &(h))
+			require.Error(t, err)
+		}
+	})
+}

test/e2e/tests/e2e_test.go

@@ -4,21 +4,29 @@ import (
 	"context"
 	"os"
 	"path/filepath"
+	"sync"
 	"testing"
 
 	"github.com/stretchr/testify/require"
+	rpchttp "github.com/tendermint/tendermint/rpc/client/http"
+	rpctypes "github.com/tendermint/tendermint/rpc/core/types"
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
+	"github.com/tendermint/tendermint/types"
 )
 
 func init() {
 	// This can be used to manually specify a testnet manifest and/or node to
 	// run tests against. The testnet must have been started by the runner first.
-	//os.Setenv("E2E_MANIFEST", "networks/simple.toml")
-	//os.Setenv("E2E_NODE", "validator01")
+	// os.Setenv("E2E_MANIFEST", "networks/ci.toml")
+	// os.Setenv("E2E_NODE", "validator01")
 }
 
 var (
-	ctx = context.Background()
+	ctx             = context.Background()
+	testnetCache    = map[string]e2e.Testnet{}
+	testnetCacheMtx = sync.Mutex{}
+	blocksCache     = map[string][]*types.Block{}
+	blocksCacheMtx  = sync.Mutex{}
 )
 
 // testNode runs tests for testnet nodes. The callback function is given a
@@ -29,16 +37,9 @@ var (
 // test runs. If E2E_NODE is also set, only the specified node is tested,
 // otherwise all nodes are tested.
 func testNode(t *testing.T, testFunc func(*testing.T, e2e.Node)) {
-	manifest := os.Getenv("E2E_MANIFEST")
-	if manifest == "" {
-		t.Skip("E2E_MANIFEST not set, not an end-to-end test run")
-	}
-	if !filepath.IsAbs(manifest) {
-		manifest = filepath.Join("..", manifest)
-	}
+	t.Helper()
 
-	testnet, err := e2e.LoadTestnet(manifest)
-	require.NoError(t, err)
+	testnet := loadTestnet(t)
 	nodes := testnet.Nodes
 
 	if name := os.Getenv("E2E_NODE"); name != "" {
@@ -55,3 +56,79 @@ func testNode(t *testing.T, testFunc func(*testing.T, e2e.Node)) {
 		})
 	}
 }
+
+// loadTestnet loads the testnet based on the E2E_MANIFEST envvar.
+func loadTestnet(t *testing.T) e2e.Testnet {
+	t.Helper()
+
+	manifest := os.Getenv("E2E_MANIFEST")
+	if manifest == "" {
+		t.Skip("E2E_MANIFEST not set, not an end-to-end test run")
+	}
+	if !filepath.IsAbs(manifest) {
+		manifest = filepath.Join("..", manifest)
+	}
+
+	testnetCacheMtx.Lock()
+	defer testnetCacheMtx.Unlock()
+	if testnet, ok := testnetCache[manifest]; ok {
+		return testnet
+	}
+
+	testnet, err := e2e.LoadTestnet(manifest)
+	require.NoError(t, err)
+	testnetCache[manifest] = *testnet
+	return *testnet
+}
+
+// fetchBlockChain fetches a complete, up-to-date block history from
+// the freshest testnet archive node.
+func fetchBlockChain(t *testing.T) []*types.Block {
+	t.Helper()
+
+	testnet := loadTestnet(t)
+
+	// Find the freshest archive node
+	var (
+		client *rpchttp.HTTP
+		status *rpctypes.ResultStatus
+	)
+	for _, node := range testnet.ArchiveNodes() {
+		c, err := node.Client()
+		require.NoError(t, err)
+		s, err := c.Status(ctx)
+		require.NoError(t, err)
+		if status == nil || s.SyncInfo.LatestBlockHeight > status.SyncInfo.LatestBlockHeight {
+			client = c
+			status = s
+		}
+	}
+	require.NotNil(t, client, "couldn't find an archive node")
+
+	// Fetch blocks. Look for existing block history in the block cache, and
+	// extend it with any new blocks that have been produced.
+	blocksCacheMtx.Lock()
+	defer blocksCacheMtx.Unlock()
+
+	from := status.SyncInfo.EarliestBlockHeight
+	to := status.SyncInfo.LatestBlockHeight
+	blocks, ok := blocksCache[testnet.Name]
+	if !ok {
+		blocks = make([]*types.Block, 0, to-from+1)
+	}
+	if len(blocks) > 0 {
+		from = blocks[len(blocks)-1].Height + 1
+	}
+
+	for h := from; h <= to; h++ {
+		resp, err := client.Block(ctx, &(h))
+		require.NoError(t, err)
+		require.NotNil(t, resp.Block)
+		require.Equal(t, h, resp.Block.Height, "unexpected block height %v", resp.Block.Height)
+		blocks = append(blocks, resp.Block)
+	}
+	require.NotEmpty(t, blocks, "blockchain does not contain any blocks")
+	blocksCache[testnet.Name] = blocks
+
+	return blocks
+}

test/e2e/tests/validator_test.go

@@ -0,0 +1,161 @@
+package e2e_test
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
+	"github.com/tendermint/tendermint/types"
+)
+
+// Tests that validator sets are available and correct according to
+// scheduled validator updates.
+func TestValidator_Sets(t *testing.T) {
+	testNode(t, func(t *testing.T, node e2e.Node) {
+		client, err := node.Client()
+		require.NoError(t, err)
+		status, err := client.Status(ctx)
+		require.NoError(t, err)
+
+		first := status.SyncInfo.EarliestBlockHeight
+		last := status.SyncInfo.LatestBlockHeight
+
+		// skip first block if node is pruning blocks, to avoid race conditions
+		if node.RetainBlocks > 0 {
+			first++
+		}
+
+		valSchedule := newValidatorSchedule(*node.Testnet)
+		valSchedule.Increment(first - node.Testnet.InitialHeight)
+
+		for h := first; h <= last; h++ {
+			validators := []*types.Validator{}
+			perPage := 100
+			for page := 1; ; page++ {
+				resp, err := client.Validators(ctx, &(h), &(page), &perPage)
+				require.NoError(t, err)
+				validators = append(validators, resp.Validators...)
+				if len(validators) == resp.Total {
+					break
+				}
+			}
+			require.Equal(t, valSchedule.Set.Validators, validators,
+				"incorrect validator set at height %v", h)
+			valSchedule.Increment(1)
+		}
+	})
+}
+
+// Tests that a validator proposes blocks when it's supposed to. It tolerates some
+// missed blocks, e.g. due to testnet perturbations.
+func TestValidator_Propose(t *testing.T) {
+	blocks := fetchBlockChain(t)
+	testNode(t, func(t *testing.T, node e2e.Node) {
+		if node.Mode != e2e.ModeValidator {
+			return
+		}
+		address := node.Key.PubKey().Address()
+		valSchedule := newValidatorSchedule(*node.Testnet)
+
+		expectCount := 0
+		proposeCount := 0
+		for _, block := range blocks {
+			if bytes.Equal(valSchedule.Set.Proposer.Address, address) {
+				expectCount++
+				if bytes.Equal(block.ProposerAddress, address) {
+					proposeCount++
+				}
+			}
+			valSchedule.Increment(1)
+		}
+
+		require.False(t, proposeCount == 0 && expectCount > 0,
+			"node did not propose any blocks (expected %v)", expectCount)
+		require.Less(t, expectCount-proposeCount, 5,
+			"validator missed proposing too many blocks (proposed %v out of %v)", proposeCount, expectCount)
+	})
+}
+
+// Tests that a validator signs blocks when it's supposed to. It tolerates some
+// missed blocks, e.g. due to testnet perturbations.
+func TestValidator_Sign(t *testing.T) {
+	blocks := fetchBlockChain(t)
+	testNode(t, func(t *testing.T, node e2e.Node) {
+		if node.Mode != e2e.ModeValidator {
+			return
+		}
+		address := node.Key.PubKey().Address()
+		valSchedule := newValidatorSchedule(*node.Testnet)
+
+		expectCount := 0
+		signCount := 0
+		for _, block := range blocks[1:] { // Skip first block, since it has no signatures
+			signed := false
+			for _, sig := range block.LastCommit.Signatures {
+				if bytes.Equal(sig.ValidatorAddress, address) {
+					signed = true
+					break
+				}
+			}
+			if valSchedule.Set.HasAddress(address) {
+				expectCount++
+				if signed {
+					signCount++
+				}
+			} else {
+				require.False(t, signed, "unexpected signature for block %v", block.LastCommit.Height)
+			}
+			valSchedule.Increment(1)
+		}
+
+		require.False(t, signCount == 0 && expectCount > 0,
+			"node did not sign any blocks (expected %v)", expectCount)
+		require.Less(t, float64(expectCount-signCount)/float64(expectCount), 0.5,
+			"validator missed signing too many blocks (signed %v out of %v)", signCount, expectCount)
+	})
+}
+
+// validatorSchedule is a validator set iterator, which takes into account
+// validator set updates.
+type validatorSchedule struct {
+	Set     *types.ValidatorSet
+	height  int64
+	updates map[int64]map[*e2e.Node]int64
+}
+
+func newValidatorSchedule(testnet e2e.Testnet) *validatorSchedule {
+	valMap := testnet.Validators                  // genesis validators
+	if v, ok := testnet.ValidatorUpdates[0]; ok { // InitChain validators
+		valMap = v
+	}
+	return &validatorSchedule{
+		height:  testnet.InitialHeight,
+		Set:     types.NewValidatorSet(makeVals(valMap)),
+		updates: testnet.ValidatorUpdates,
+	}
+}
+
+func (s *validatorSchedule) Increment(heights int64) {
+	for i := int64(0); i < heights; i++ {
+		s.height++
+		if s.height > 2 {
+			// validator set updates are offset by 2, since they only take effect
+			// two blocks after they're returned.
+			if update, ok := s.updates[s.height-2]; ok {
+				if err := s.Set.UpdateWithChangeSet(makeVals(update)); err != nil {
+					panic(err)
+				}
+			}
+		}
+		s.Set.IncrementProposerPriority(1)
+	}
+}
+
+func makeVals(valMap map[*e2e.Node]int64) []*types.Validator {
+	vals := make([]*types.Validator, 0, len(valMap))
+	for node, power := range valMap {
+		vals = append(vals, types.NewValidator(node.Key.PubKey(), power))
+	}
+	return vals
+}