tendermint/evidence/reactor_test.go

package evidence_test

import (
	"encoding/hex"
	"fmt"
	"math/rand"
	"sync"
	"testing"
	"time"

	"github.com/stretchr/testify/mock"
	"github.com/stretchr/testify/require"

	dbm "github.com/tendermint/tm-db"

	"github.com/tendermint/tendermint/crypto"
	"github.com/tendermint/tendermint/crypto/tmhash"
	"github.com/tendermint/tendermint/evidence"
	"github.com/tendermint/tendermint/evidence/mocks"
	"github.com/tendermint/tendermint/libs/log"
	"github.com/tendermint/tendermint/p2p"
	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
	sm "github.com/tendermint/tendermint/state"
	"github.com/tendermint/tendermint/types"
)

var (
	numEvidence = 10

	rng = rand.New(rand.NewSource(time.Now().UnixNano()))
)

type reactorTestSuite struct {
	reactor *evidence.Reactor
	pool    *evidence.Pool

	peerID p2p.NodeID

	evidenceChannel   *p2p.Channel
	evidenceInCh      chan p2p.Envelope
	evidenceOutCh     chan p2p.Envelope
	evidencePeerErrCh chan p2p.PeerError

	peerUpdatesCh chan p2p.PeerUpdate
	peerUpdates   *p2p.PeerUpdates
}

func setup(t *testing.T, logger log.Logger, pool *evidence.Pool, chBuf uint) *reactorTestSuite {
	t.Helper()

	pID := make([]byte, 16)
	_, err := rng.Read(pID)
	require.NoError(t, err)

	peerUpdatesCh := make(chan p2p.PeerUpdate)

	rts := &reactorTestSuite{
		pool:              pool,
		evidenceInCh:      make(chan p2p.Envelope, chBuf),
		evidenceOutCh:     make(chan p2p.Envelope, chBuf),
		evidencePeerErrCh: make(chan p2p.PeerError, chBuf),
		peerUpdatesCh:     peerUpdatesCh,
		peerUpdates:       p2p.NewPeerUpdates(peerUpdatesCh),
		peerID:            p2p.NodeID(fmt.Sprintf("%x", pID)),
	}

	rts.evidenceChannel = p2p.NewChannel(
		evidence.EvidenceChannel,
		new(tmproto.EvidenceList),
		rts.evidenceInCh,
		rts.evidenceOutCh,
		rts.evidencePeerErrCh,
	)

	rts.reactor = evidence.NewReactor(
		logger,
		rts.evidenceChannel,
		rts.peerUpdates,
		pool,
	)

	require.NoError(t, rts.reactor.Start())
	require.True(t, rts.reactor.IsRunning())

	t.Cleanup(func() {
		require.NoError(t, rts.reactor.Stop())
		require.False(t, rts.reactor.IsRunning())
	})

	return rts
}

func createTestSuites(t *testing.T, stateStores []sm.Store, chBuf uint) []*reactorTestSuite {
	t.Helper()

	numSStores := len(stateStores)
	testSuites := make([]*reactorTestSuite, numSStores)
	evidenceTime := time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC)

	for i := 0; i < numSStores; i++ {
		logger := log.TestingLogger().With("validator", i)
		evidenceDB := dbm.NewMemDB()
		blockStore := &mocks.BlockStore{}
		blockStore.On("LoadBlockMeta", mock.AnythingOfType("int64")).Return(
			&types.BlockMeta{Header: types.Header{Time: evidenceTime}},
		)

		pool, err := evidence.NewPool(logger, evidenceDB, stateStores[i], blockStore)
		require.NoError(t, err)

		testSuites[i] = setup(t, logger, pool, chBuf)
	}

	return testSuites
}

func waitForEvidence(t *testing.T, evList types.EvidenceList, suites ...*reactorTestSuite) {
	t.Helper()

	wg := new(sync.WaitGroup)

	for _, suite := range suites {
		wg.Add(1)

		go func(s *reactorTestSuite) {
			var localEvList []types.Evidence

			currentPoolSize := 0
			for currentPoolSize != len(evList) {
				// each evidence should not be more than 500 bytes
				localEvList, _ = s.pool.PendingEvidence(int64(len(evList) * 500))
				currentPoolSize = len(localEvList)
			}

			// put the reaped evidence in a map so we can quickly check we got everything
			evMap := make(map[string]types.Evidence)
			for _, e := range localEvList {
				evMap[string(e.Hash())] = e
			}

			for i, expectedEv := range evList {
				gotEv := evMap[string(expectedEv.Hash())]
				require.Equalf(
					t,
					expectedEv,
					gotEv,
					"evidence at index %d in pool does not match; got: %v, expected: %v", i, gotEv, expectedEv,
				)
			}

			wg.Done()
		}(suite)
	}

	// wait for the evidence in all evidence pools
	wg.Wait()
}

func createEvidenceList(
	t *testing.T,
	pool *evidence.Pool,
	val types.PrivValidator,
	numEvidence int,
) types.EvidenceList {
	evList := make([]types.Evidence, numEvidence)
	for i := 0; i < numEvidence; i++ {
		ev := types.NewMockDuplicateVoteEvidenceWithValidator(
			int64(i+1),
			time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC),
			val,
			evidenceChainID,
		)

		require.NoError(t, pool.AddEvidence(ev))
		evList[i] = ev
	}

	return evList
}

// simulateRouter will increment the provided WaitGroup and execute a simulated
// router where, for each outbound p2p Envelope from the primary reactor, we
// proxy (send) the Envelope the relevant peer reactor. Done is invoked on the
// WaitGroup when numOut Envelopes are sent (i.e. read from the outbound channel).
func simulateRouter(wg *sync.WaitGroup, primary *reactorTestSuite, suites []*reactorTestSuite, numOut int) {
	wg.Add(1)

	// create a mapping for efficient suite lookup by peer ID
	suitesByPeerID := make(map[p2p.NodeID]*reactorTestSuite)
	for _, suite := range suites {
		suitesByPeerID[suite.peerID] = suite
	}

	// Simulate a router by listening for all outbound envelopes and proxying the
	// envelope to the respective peer (suite).
	go func() {
		for i := 0; i < numOut; i++ {
			envelope := <-primary.evidenceOutCh
			other := suitesByPeerID[envelope.To]

			other.evidenceInCh <- p2p.Envelope{
				From:    primary.peerID,
				To:      envelope.To,
				Message: envelope.Message,
			}
		}

		wg.Done()
	}()
}

func TestReactorMultiDisconnect(t *testing.T) {
	val := types.NewMockPV()
	height := int64(numEvidence) + 10

	stateDB1 := initializeValidatorState(t, val, height)
	stateDB2 := initializeValidatorState(t, val, height)

	testSuites := createTestSuites(t, []sm.Store{stateDB1, stateDB2}, 20)
	primary := testSuites[0]
	secondary := testSuites[1]

	_ = createEvidenceList(t, primary.pool, val, numEvidence)

	primary.peerUpdatesCh <- p2p.PeerUpdate{
		Status: p2p.PeerStatusUp,
		NodeID: secondary.peerID,
	}

	// Ensure "disconnecting" the secondary peer from the primary more than once
	// is handled gracefully.
	primary.peerUpdatesCh <- p2p.PeerUpdate{
		Status: p2p.PeerStatusDown,
		NodeID: secondary.peerID,
	}
	primary.peerUpdatesCh <- p2p.PeerUpdate{
		Status: p2p.PeerStatusDown,
		NodeID: secondary.peerID,
	}
}

// TestReactorBroadcastEvidence creates an environment of multiple peers that
// are all at the same height. One peer, designated as a primary, gossips all
// evidence to the remaining peers.
func TestReactorBroadcastEvidence(t *testing.T) {
	numPeers := 7

	// create a stateDB for all test suites (nodes)
	stateDBs := make([]sm.Store, numPeers)
	val := types.NewMockPV()

	// We need all validators saved for heights at least as high as we have
	// evidence for.
	height := int64(numEvidence) + 10
	for i := 0; i < numPeers; i++ {
		stateDBs[i] = initializeValidatorState(t, val, height)
	}

	// Create a series of test suites where each suite contains a reactor and
	// evidence pool. In addition, we mark a primary suite and the rest are
	// secondaries where each secondary is added as a peer via a PeerUpdate to the
	// primary. As a result, the primary will gossip all evidence to each secondary.
	testSuites := createTestSuites(t, stateDBs, 0)
	primary := testSuites[0]
	secondaries := testSuites[1:]

	// Simulate a router by listening for all outbound envelopes and proxying the
	// envelopes to the respective peer (suite).
	wg := new(sync.WaitGroup)
	simulateRouter(wg, primary, testSuites, numEvidence*len(secondaries))

	evList := createEvidenceList(t, primary.pool, val, numEvidence)

	// Add each secondary suite (node) as a peer to the primary suite (node). This
	// will cause the primary to gossip all evidence to the secondaries.
	for _, suite := range secondaries {
		primary.peerUpdatesCh <- p2p.PeerUpdate{
			Status: p2p.PeerStatusUp,
			NodeID: suite.peerID,
		}
	}

	// Wait till all secondary suites (reactor) received all evidence from the
	// primary suite (node).
	waitForEvidence(t, evList, secondaries...)

	for _, suite := range testSuites {
		require.Equal(t, numEvidence, int(suite.pool.Size()))
	}

	wg.Wait()

	// ensure all channels are drained
	for _, suite := range testSuites {
		require.Empty(t, suite.evidenceOutCh)
	}
}

// TestReactorSelectiveBroadcast tests a context where we have two reactors
// connected to one another but are at different heights. Reactor 1 which is
// ahead receives a list of evidence.
func TestReactorBroadcastEvidence_Lagging(t *testing.T) {
	val := types.NewMockPV()
	height1 := int64(numEvidence) + 10
	height2 := int64(numEvidence) / 2

	// stateDB1 is ahead of stateDB2, where stateDB1 has all heights (1-10) and
	// stateDB2 only has heights 1-7.
	stateDB1 := initializeValidatorState(t, val, height1)
	stateDB2 := initializeValidatorState(t, val, height2)

	testSuites := createTestSuites(t, []sm.Store{stateDB1, stateDB2}, 0)
	primary := testSuites[0]
	secondaries := testSuites[1:]

	// Simulate a router by listening for all outbound envelopes and proxying the
	// envelope to the respective peer (suite).
	wg := new(sync.WaitGroup)
	simulateRouter(wg, primary, testSuites, numEvidence*len(secondaries))

	// Send a list of valid evidence to the first reactor's, the one that is ahead,
	// evidence pool.
	evList := createEvidenceList(t, primary.pool, val, numEvidence)

	// Add each secondary suite (node) as a peer to the primary suite (node). This
	// will cause the primary to gossip all evidence to the secondaries.
	for _, suite := range secondaries {
		primary.peerUpdatesCh <- p2p.PeerUpdate{
			Status: p2p.PeerStatusUp,
			NodeID: suite.peerID,
		}
	}

	// only ones less than the peers height should make it through
	waitForEvidence(t, evList[:height2+2], secondaries...)

	require.Equal(t, numEvidence, int(primary.pool.Size()))
	require.Equal(t, int(height2+2), int(secondaries[0].pool.Size()))

	// The primary will continue to send the remaining evidence to the secondaries
	// so we wait until it has sent all the envelopes.
	wg.Wait()

	// ensure all channels are drained
	for _, suite := range testSuites {
		require.Empty(t, suite.evidenceOutCh)
	}
}

func TestReactorBroadcastEvidence_Pending(t *testing.T) {
	val := types.NewMockPV()
	height := int64(10)

	stateDB1 := initializeValidatorState(t, val, height)
	stateDB2 := initializeValidatorState(t, val, height)

	testSuites := createTestSuites(t, []sm.Store{stateDB1, stateDB2}, 0)
	primary := testSuites[0]
	secondary := testSuites[1]

	// Simulate a router by listening for all outbound envelopes and proxying the
	// envelopes to the respective peer (suite).
	wg := new(sync.WaitGroup)
	simulateRouter(wg, primary, testSuites, numEvidence)

	// add all evidence to the primary reactor
	evList := createEvidenceList(t, primary.pool, val, numEvidence)

	// Manually add half the evidence to the secondary which will mark them as
	// pending.
	for i := 0; i < numEvidence/2; i++ {
		require.NoError(t, secondary.pool.AddEvidence(evList[i]))
	}

	// the secondary should have half the evidence as pending
	require.Equal(t, uint32(numEvidence/2), secondary.pool.Size())

	// add the secondary reactor as a peer to the primary reactor
	primary.peerUpdatesCh <- p2p.PeerUpdate{
		Status: p2p.PeerStatusUp,
		NodeID: secondary.peerID,
	}

	// The secondary reactor should have received all the evidence ignoring the
	// already pending evidence.
	waitForEvidence(t, evList, secondary)

	for _, suite := range testSuites {
		require.Equal(t, numEvidence, int(suite.pool.Size()))
	}

	wg.Wait()

	// ensure all channels are drained
	for _, suite := range testSuites {
		require.Empty(t, suite.evidenceOutCh)
	}
}

func TestReactorBroadcastEvidence_Committed(t *testing.T) {
	val := types.NewMockPV()
	height := int64(10)

	stateDB1 := initializeValidatorState(t, val, height)
	stateDB2 := initializeValidatorState(t, val, height)

	testSuites := createTestSuites(t, []sm.Store{stateDB1, stateDB2}, 0)
	primary := testSuites[0]
	secondary := testSuites[1]

	// add all evidence to the primary reactor
	evList := createEvidenceList(t, primary.pool, val, numEvidence)

	// Manually add half the evidence to the secondary which will mark them as
	// pending.
	for i := 0; i < numEvidence/2; i++ {
		require.NoError(t, secondary.pool.AddEvidence(evList[i]))
	}

	// the secondary should have half the evidence as pending
	require.Equal(t, uint32(numEvidence/2), secondary.pool.Size())

	state, err := stateDB2.Load()
	require.NoError(t, err)

	// update the secondary's pool such that all pending evidence is committed
	state.LastBlockHeight++
	secondary.pool.Update(state, evList[:numEvidence/2])

	// the secondary should have half the evidence as committed
	require.Equal(t, uint32(0), secondary.pool.Size())

	// Simulate a router by listening for all outbound envelopes and proxying the
	// envelopes to the respective peer (suite).
	wg := new(sync.WaitGroup)
	simulateRouter(wg, primary, testSuites, numEvidence)

	// add the secondary reactor as a peer to the primary reactor
	primary.peerUpdatesCh <- p2p.PeerUpdate{
		Status: p2p.PeerStatusUp,
		NodeID: secondary.peerID,
	}

	// The secondary reactor should have received all the evidence ignoring the
	// already committed evidence.
	waitForEvidence(t, evList[numEvidence/2:], secondary)

	require.Equal(t, numEvidence, int(primary.pool.Size()))
	require.Equal(t, numEvidence/2, int(secondary.pool.Size()))

	wg.Wait()

	// ensure all channels are drained
	for _, suite := range testSuites {
		require.Empty(t, suite.evidenceOutCh)
	}
}

func TestReactorBroadcastEvidence_FullyConnected(t *testing.T) {
	numPeers := 7

	// create a stateDB for all test suites (nodes)
	stateDBs := make([]sm.Store, numPeers)
	val := types.NewMockPV()

	// We need all validators saved for heights at least as high as we have
	// evidence for.
	height := int64(numEvidence) + 10
	for i := 0; i < numPeers; i++ {
		stateDBs[i] = initializeValidatorState(t, val, height)
	}

	testSuites := createTestSuites(t, stateDBs, 0)

	// Simulate a router by listening for all outbound envelopes and proxying the
	// envelopes to the respective peer (suite).
	wg := new(sync.WaitGroup)
	for _, suite := range testSuites {
		simulateRouter(wg, suite, testSuites, numEvidence*(len(testSuites)-1))
	}

	evList := createEvidenceList(t, testSuites[0].pool, val, numEvidence)

	// every suite (reactor) connects to every other suite (reactor)
	for _, suiteI := range testSuites {
		for _, suiteJ := range testSuites {
			if suiteI.peerID != suiteJ.peerID {
				suiteI.peerUpdatesCh <- p2p.PeerUpdate{
					Status: p2p.PeerStatusUp,
					NodeID: suiteJ.peerID,
				}
			}
		}
	}

	// wait till all suites (reactors) received all evidence from other suites (reactors)
	waitForEvidence(t, evList, testSuites...)

	for _, suite := range testSuites {
		require.Equal(t, numEvidence, int(suite.pool.Size()))

		// commit state so we do not continue to repeat gossiping the same evidence
		state := suite.pool.State()
		state.LastBlockHeight++
		suite.pool.Update(state, evList)
	}

	wg.Wait()
}

func TestReactorBroadcastEvidence_RemovePeer(t *testing.T) {
	val := types.NewMockPV()
	height := int64(10)

	stateDB1 := initializeValidatorState(t, val, height)
	stateDB2 := initializeValidatorState(t, val, height)

	testSuites := createTestSuites(t, []sm.Store{stateDB1, stateDB2}, uint(numEvidence))
	primary := testSuites[0]
	secondary := testSuites[1]

	// Simulate a router by listening for all outbound envelopes and proxying the
	// envelopes to the respective peer (suite).
	wg := new(sync.WaitGroup)
	simulateRouter(wg, primary, testSuites, numEvidence/2)

	// add all evidence to the primary reactor
	evList := createEvidenceList(t, primary.pool, val, numEvidence)

	// add the secondary reactor as a peer to the primary reactor
	primary.peerUpdatesCh <- p2p.PeerUpdate{
		Status: p2p.PeerStatusUp,
		NodeID: secondary.peerID,
	}

	// have the secondary reactor receive only half the evidence
	waitForEvidence(t, evList[:numEvidence/2], secondary)

	// disconnect the peer
	primary.peerUpdatesCh <- p2p.PeerUpdate{
		Status: p2p.PeerStatusDown,
		NodeID: secondary.peerID,
	}

	// Ensure the secondary only received half of the evidence before being
	// disconnected.
	require.Equal(t, numEvidence/2, int(secondary.pool.Size()))

	wg.Wait()

	// The primary reactor should still be attempting to send the remaining half.
	//
	// NOTE: The channel is buffered (size numEvidence) as to ensure the primary
	// reactor will send all envelopes at once before receiving the signal to stop
	// gossiping.
	for i := 0; i < numEvidence/2; i++ {
		<-primary.evidenceOutCh
	}

	// ensure all channels are drained
	for _, suite := range testSuites {
		require.Empty(t, suite.evidenceOutCh)
	}
}

// nolint:lll
func TestEvidenceListSerialization(t *testing.T) {
	exampleVote := func(msgType byte) *types.Vote {
		var stamp, err = time.Parse(types.TimeFormat, "2017-12-25T03:00:01.234Z")
		require.NoError(t, err)

		return &types.Vote{
			Type:      tmproto.SignedMsgType(msgType),
			Height:    3,
			Round:     2,
			Timestamp: stamp,
			BlockID: types.BlockID{
				Hash: tmhash.Sum([]byte("blockID_hash")),
				PartSetHeader: types.PartSetHeader{
					Total: 1000000,
					Hash:  tmhash.Sum([]byte("blockID_part_set_header_hash")),
				},
			},
			ValidatorAddress: crypto.AddressHash([]byte("validator_address")),
			ValidatorIndex:   56789,
		}
	}

	val := &types.Validator{
		Address:     crypto.AddressHash([]byte("validator_address")),
		VotingPower: 10,
	}

	valSet := types.NewValidatorSet([]*types.Validator{val})

	dupl := types.NewDuplicateVoteEvidence(
		exampleVote(1),
		exampleVote(2),
		defaultEvidenceTime,
		valSet,
	)

	testCases := map[string]struct {
		evidenceList []types.Evidence
		expBytes     string
	}{
		"DuplicateVoteEvidence": {
			[]types.Evidence{dupl},
			"0a85020a82020a79080210031802224a0a208b01023386c371778ecb6368573e539afc3cc860ec3a2f614e54fe5652f4fc80122608c0843d122072db3d959635dff1bb567bedaa70573392c5159666a3f8caf11e413aac52207a2a0b08b1d381d20510809dca6f32146af1f4111082efb388211bc72c55bcd61e9ac3d538d5bb031279080110031802224a0a208b01023386c371778ecb6368573e539afc3cc860ec3a2f614e54fe5652f4fc80122608c0843d122072db3d959635dff1bb567bedaa70573392c5159666a3f8caf11e413aac52207a2a0b08b1d381d20510809dca6f32146af1f4111082efb388211bc72c55bcd61e9ac3d538d5bb03180a200a2a060880dbaae105",
		},
	}

	for name, tc := range testCases {
		tc := tc

		t.Run(name, func(t *testing.T) {
			protoEv := make([]tmproto.Evidence, len(tc.evidenceList))
			for i := 0; i < len(tc.evidenceList); i++ {
				ev, err := types.EvidenceToProto(tc.evidenceList[i])
				require.NoError(t, err)
				protoEv[i] = *ev
			}

			epl := tmproto.EvidenceList{
				Evidence: protoEv,
			}

			bz, err := epl.Marshal()
			require.NoError(t, err)

			require.Equal(t, tc.expBytes, hex.EncodeToString(bz))
		})
	}
}