tendermint/light/client_test.go

package light_test

import (
	"sync"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	dbm "github.com/tendermint/tm-db"

	"github.com/tendermint/tendermint/libs/log"
	"github.com/tendermint/tendermint/light"
	"github.com/tendermint/tendermint/light/provider"
	mockp "github.com/tendermint/tendermint/light/provider/mock"
	dbs "github.com/tendermint/tendermint/light/store/db"
	"github.com/tendermint/tendermint/types"
)

const (
	chainID = "test"
)

var (
	keys     = genPrivKeys(4)
	vals     = keys.ToValidators(20, 10)
	bTime, _ = time.Parse(time.RFC3339, "2006-01-02T15:04:05Z")
	h1       = keys.GenSignedHeader(chainID, 1, bTime, nil, vals, vals,
		hash("app_hash"), hash("cons_hash"), hash("results_hash"), 0, len(keys))
	// 3/3 signed
	h2 = keys.GenSignedHeaderLastBlockID(chainID, 2, bTime.Add(30*time.Minute), nil, vals, vals,
		hash("app_hash"), hash("cons_hash"), hash("results_hash"), 0, len(keys), types.BlockID{Hash: h1.Hash()})
	// 3/3 signed
	h3 = keys.GenSignedHeaderLastBlockID(chainID, 3, bTime.Add(1*time.Hour), nil, vals, vals,
		hash("app_hash"), hash("cons_hash"), hash("results_hash"), 0, len(keys), types.BlockID{Hash: h2.Hash()})
	trustPeriod  = 4 * time.Hour
	trustOptions = light.TrustOptions{
		Period: 4 * time.Hour,
		Height: 1,
		Hash:   h1.Hash(),
	}
	valSet = map[int64]*types.ValidatorSet{
		1: vals,
		2: vals,
		3: vals,
		4: vals,
	}
	headerSet = map[int64]*types.SignedHeader{
		1: h1,
		// interim header (3/3 signed)
		2: h2,
		// last header (3/3 signed)
		3: h3,
	}
	fullNode = mockp.New(
		chainID,
		headerSet,
		valSet,
	)
	deadNode      = mockp.NewDeadMock(chainID)
	largeFullNode = mockp.New(GenMockNode(chainID, 10, 3, 0, bTime))
)

func TestValidateTrustOptions(t *testing.T) {
	testCases := []struct {
		err bool
		to  light.TrustOptions
	}{
		{
			false,
			trustOptions,
		},
		{
			true,
			light.TrustOptions{
				Period: -1 * time.Hour,
				Height: 1,
				Hash:   h1.Hash(),
			},
		},
		{
			true,
			light.TrustOptions{
				Period: 1 * time.Hour,
				Height: 0,
				Hash:   h1.Hash(),
			},
		},
		{
			true,
			light.TrustOptions{
				Period: 1 * time.Hour,
				Height: 1,
				Hash:   []byte("incorrect hash"),
			},
		},
	}

	for _, tc := range testCases {
		err := tc.to.ValidateBasic()
		if tc.err {
			assert.Error(t, err)
		} else {
			assert.NoError(t, err)
		}
	}

}

func TestClient_SequentialVerification(t *testing.T) {
	newKeys := genPrivKeys(4)
	newVals := newKeys.ToValidators(10, 1)

	testCases := []struct {
		name         string
		otherHeaders map[int64]*types.SignedHeader // all except ^
		vals         map[int64]*types.ValidatorSet
		initErr      bool
		verifyErr    bool
	}{
		{
			"good",
			headerSet,
			valSet,
			false,
			false,
		},
		{
			"bad: different first header",
			map[int64]*types.SignedHeader{
				// different header
				1: keys.GenSignedHeader(chainID, 1, bTime.Add(1*time.Hour), nil, vals, vals,
					hash("app_hash"), hash("cons_hash"), hash("results_hash"), 0, len(keys)),
			},
			map[int64]*types.ValidatorSet{
				1: vals,
			},
			true,
			false,
		},
		{
			"bad: 1/3 signed interim header",
			map[int64]*types.SignedHeader{
				// trusted header
				1: h1,
				// interim header (1/3 signed)
				2: keys.GenSignedHeader(chainID, 2, bTime.Add(1*time.Hour), nil, vals, vals,
					hash("app_hash"), hash("cons_hash"), hash("results_hash"), len(keys)-1, len(keys)),
				// last header (3/3 signed)
				3: keys.GenSignedHeader(chainID, 3, bTime.Add(2*time.Hour), nil, vals, vals,
					hash("app_hash"), hash("cons_hash"), hash("results_hash"), 0, len(keys)),
			},
			valSet,
			false,
			true,
		},
		{
			"bad: 1/3 signed last header",
			map[int64]*types.SignedHeader{
				// trusted header
				1: h1,
				// interim header (3/3 signed)
				2: keys.GenSignedHeader(chainID, 2, bTime.Add(1*time.Hour), nil, vals, vals,
					hash("app_hash"), hash("cons_hash"), hash("results_hash"), 0, len(keys)),
				// last header (1/3 signed)
				3: keys.GenSignedHeader(chainID, 3, bTime.Add(2*time.Hour), nil, vals, vals,
					hash("app_hash"), hash("cons_hash"), hash("results_hash"), len(keys)-1, len(keys)),
			},
			valSet,
			false,
			true,
		},
		{
			"bad: different validator set at height 3",
			headerSet,
			map[int64]*types.ValidatorSet{
				1: vals,
				2: vals,
				3: newVals,
			},
			false,
			true,
		},
	}

	for _, tc := range testCases {
		tc := tc
		t.Run(tc.name, func(t *testing.T) {
			c, err := light.NewClient(
				chainID,
				trustOptions,
				mockp.New(
					chainID,
					tc.otherHeaders,
					tc.vals,
				),
				[]provider.Provider{mockp.New(
					chainID,
					tc.otherHeaders,
					tc.vals,
				)},
				dbs.New(dbm.NewMemDB(), chainID),
				light.SequentialVerification(),
			)

			if tc.initErr {
				require.Error(t, err)
				return
			}

			require.NoError(t, err)

			_, err = c.VerifyHeaderAtHeight(3, bTime.Add(3*time.Hour))
			if tc.verifyErr {
				assert.Error(t, err)
			} else {
				assert.NoError(t, err)
			}
		})
	}
}

func TestClient_SkippingVerification(t *testing.T) {
	// required for 2nd test case
	newKeys := genPrivKeys(4)
	newVals := newKeys.ToValidators(10, 1)

	// 1/3+ of vals, 2/3- of newVals
	transitKeys := keys.Extend(3)
	transitVals := transitKeys.ToValidators(10, 1)

	testCases := []struct {
		name         string
		otherHeaders map[int64]*types.SignedHeader // all except ^
		vals         map[int64]*types.ValidatorSet
		initErr      bool
		verifyErr    bool
	}{
		{
			"good",
			map[int64]*types.SignedHeader{
				// trusted header
				1: h1,
				// last header (3/3 signed)
				3: h3,
			},
			valSet,
			false,
			false,
		},
		{
			"good, but val set changes by 2/3 (1/3 of vals is still present)",
			map[int64]*types.SignedHeader{
				// trusted header
				1: h1,
				3: transitKeys.GenSignedHeader(chainID, 3, bTime.Add(2*time.Hour), nil, transitVals, transitVals,
					hash("app_hash"), hash("cons_hash"), hash("results_hash"), 0, len(transitKeys)),
			},
			map[int64]*types.ValidatorSet{
				1: vals,
				2: vals,
				3: transitVals,
			},
			false,
			false,
		},
		{
			"good, but val set changes 100% at height 2",
			map[int64]*types.SignedHeader{
				// trusted header
				1: h1,
				// interim header (3/3 signed)
				2: keys.GenSignedHeader(chainID, 2, bTime.Add(1*time.Hour), nil, vals, newVals,
					hash("app_hash"), hash("cons_hash"), hash("results_hash"), 0, len(keys)),
				// last header (0/4 of the original val set signed)
				3: newKeys.GenSignedHeader(chainID, 3, bTime.Add(2*time.Hour), nil, newVals, newVals,
					hash("app_hash"), hash("cons_hash"), hash("results_hash"), 0, len(newKeys)),
			},
			map[int64]*types.ValidatorSet{
				1: vals,
				2: vals,
				3: newVals,
			},
			false,
			false,
		},
		{
			"bad: last header signed by newVals, interim header has no signers",
			map[int64]*types.SignedHeader{
				// trusted header
				1: h1,
				// last header (0/4 of the original val set signed)
				2: keys.GenSignedHeader(chainID, 2, bTime.Add(1*time.Hour), nil, vals, newVals,
					hash("app_hash"), hash("cons_hash"), hash("results_hash"), 0, 0),
				// last header (0/4 of the original val set signed)
				3: newKeys.GenSignedHeader(chainID, 3, bTime.Add(2*time.Hour), nil, newVals, newVals,
					hash("app_hash"), hash("cons_hash"), hash("results_hash"), 0, len(newKeys)),
			},
			map[int64]*types.ValidatorSet{
				1: vals,
				2: vals,
				3: newVals,
			},
			false,
			true,
		},
	}

	for _, tc := range testCases {
		tc := tc
		t.Run(tc.name, func(t *testing.T) {
			c, err := light.NewClient(
				chainID,
				trustOptions,
				mockp.New(
					chainID,
					tc.otherHeaders,
					tc.vals,
				),
				[]provider.Provider{mockp.New(
					chainID,
					tc.otherHeaders,
					tc.vals,
				)},
				dbs.New(dbm.NewMemDB(), chainID),
				light.SkippingVerification(light.DefaultTrustLevel),
			)
			if tc.initErr {
				require.Error(t, err)
				return
			}

			require.NoError(t, err)

			_, err = c.VerifyHeaderAtHeight(3, bTime.Add(3*time.Hour))
			if tc.verifyErr {
				assert.Error(t, err)
			} else {
				assert.NoError(t, err)
			}
		})
	}

}

// start from a large header to make sure that the pivot height doesn't select a height outside
// the appropriate range
func TestClientLargeBisectionVerification(t *testing.T) {
	veryLargeFullNode := mockp.New(GenMockNode(chainID, 100, 3, 1, bTime))
	h1, err := veryLargeFullNode.SignedHeader(90)
	require.NoError(t, err)
	c, err := light.NewClient(
		chainID,
		light.TrustOptions{
			Period: 4 * time.Hour,
			Height: 90,
			Hash:   h1.Hash(),
		},
		veryLargeFullNode,
		[]provider.Provider{veryLargeFullNode},
		dbs.New(dbm.NewMemDB(), chainID),
		light.SkippingVerification(light.DefaultTrustLevel),
	)
	require.NoError(t, err)
	h, err := c.Update(bTime.Add(100 * time.Minute))
	assert.NoError(t, err)
	h2, err := veryLargeFullNode.SignedHeader(100)
	require.NoError(t, err)
	assert.Equal(t, h, h2)
}

func TestClientBisectionBetweenTrustedHeaders(t *testing.T) {
	c, err := light.NewClient(
		chainID,
		light.TrustOptions{
			Period: 4 * time.Hour,
			Height: 1,
			Hash:   h1.Hash(),
		},
		fullNode,
		[]provider.Provider{fullNode},
		dbs.New(dbm.NewMemDB(), chainID),
		light.SkippingVerification(light.DefaultTrustLevel),
	)
	require.NoError(t, err)

	_, err = c.VerifyHeaderAtHeight(3, bTime.Add(2*time.Hour))
	require.NoError(t, err)

	// confirm that the client already doesn't have the header
	_, err = c.TrustedHeader(2)
	require.Error(t, err)

	// verify using bisection the header between the two trusted headers
	_, err = c.VerifyHeaderAtHeight(2, bTime.Add(1*time.Hour))
	assert.NoError(t, err)

}

func TestClient_Cleanup(t *testing.T) {
	c, err := light.NewClient(
		chainID,
		trustOptions,
		fullNode,
		[]provider.Provider{fullNode},
		dbs.New(dbm.NewMemDB(), chainID),
		light.Logger(log.TestingLogger()),
	)
	require.NoError(t, err)
	_, err = c.TrustedHeader(1)
	require.NoError(t, err)

	err = c.Cleanup()
	require.NoError(t, err)

	// Check no headers/valsets exist after Cleanup.
	h, err := c.TrustedHeader(1)
	assert.Error(t, err)
	assert.Nil(t, h)

	valSet, _, err := c.TrustedValidatorSet(1)
	assert.Error(t, err)
	assert.Nil(t, valSet)
}

// trustedHeader.Height == options.Height
func TestClientRestoresTrustedHeaderAfterStartup1(t *testing.T) {
	// 1. options.Hash == trustedHeader.Hash
	{
		trustedStore := dbs.New(dbm.NewMemDB(), chainID)
		err := trustedStore.SaveSignedHeaderAndValidatorSet(h1, vals)
		require.NoError(t, err)

		c, err := light.NewClient(
			chainID,
			trustOptions,
			fullNode,
			[]provider.Provider{fullNode},
			trustedStore,
			light.Logger(log.TestingLogger()),
		)
		require.NoError(t, err)

		h, err := c.TrustedHeader(1)
		assert.NoError(t, err)
		assert.NotNil(t, h)
		assert.Equal(t, h.Hash(), h1.Hash())

		valSet, _, err := c.TrustedValidatorSet(1)
		assert.NoError(t, err)
		assert.NotNil(t, valSet)
		if assert.NotNil(t, valSet) {
			assert.Equal(t, h.ValidatorsHash.Bytes(), valSet.Hash())
		}
	}

	// 2. options.Hash != trustedHeader.Hash
	{
		trustedStore := dbs.New(dbm.NewMemDB(), chainID)
		err := trustedStore.SaveSignedHeaderAndValidatorSet(h1, vals)
		require.NoError(t, err)

		// header1 != header
		header1 := keys.GenSignedHeader(chainID, 1, bTime.Add(1*time.Hour), nil, vals, vals,
			hash("app_hash"), hash("cons_hash"), hash("results_hash"), 0, len(keys))

		primary := mockp.New(
			chainID,
			map[int64]*types.SignedHeader{
				// trusted header
				1: header1,
			},
			valSet,
		)

		c, err := light.NewClient(
			chainID,
			light.TrustOptions{
				Period: 4 * time.Hour,
				Height: 1,
				Hash:   header1.Hash(),
			},
			primary,
			[]provider.Provider{primary},
			trustedStore,
			light.Logger(log.TestingLogger()),
		)
		require.NoError(t, err)

		h, err := c.TrustedHeader(1)
		assert.NoError(t, err)
		if assert.NotNil(t, h) {
			assert.Equal(t, h.Hash(), header1.Hash())
		}

		valSet, _, err := c.TrustedValidatorSet(1)
		assert.NoError(t, err)
		assert.NotNil(t, valSet)
		if assert.NotNil(t, valSet) {
			assert.Equal(t, h.ValidatorsHash.Bytes(), valSet.Hash())
		}
	}
}

// trustedHeader.Height < options.Height
func TestClientRestoresTrustedHeaderAfterStartup2(t *testing.T) {
	// 1. options.Hash == trustedHeader.Hash
	{
		trustedStore := dbs.New(dbm.NewMemDB(), chainID)
		err := trustedStore.SaveSignedHeaderAndValidatorSet(h1, vals)
		require.NoError(t, err)

		c, err := light.NewClient(
			chainID,
			light.TrustOptions{
				Period: 4 * time.Hour,
				Height: 2,
				Hash:   h2.Hash(),
			},
			fullNode,
			[]provider.Provider{fullNode},
			trustedStore,
			light.Logger(log.TestingLogger()),
		)
		require.NoError(t, err)

		// Check we still have the 1st header (+header+).
		h, err := c.TrustedHeader(1)
		assert.NoError(t, err)
		assert.NotNil(t, h)
		assert.Equal(t, h.Hash(), h1.Hash())

		valSet, _, err := c.TrustedValidatorSet(1)
		assert.NoError(t, err)
		assert.NotNil(t, valSet)
		if assert.NotNil(t, valSet) {
			assert.Equal(t, h.ValidatorsHash.Bytes(), valSet.Hash())
		}
	}

	// 2. options.Hash != trustedHeader.Hash
	// This could happen if previous provider was lying to us.
	{
		trustedStore := dbs.New(dbm.NewMemDB(), chainID)
		err := trustedStore.SaveSignedHeaderAndValidatorSet(h1, vals)
		require.NoError(t, err)

		// header1 != header
		diffHeader1 := keys.GenSignedHeader(chainID, 1, bTime.Add(1*time.Hour), nil, vals, vals,
			hash("app_hash"), hash("cons_hash"), hash("results_hash"), 0, len(keys))

		diffHeader2 := keys.GenSignedHeader(chainID, 2, bTime.Add(2*time.Hour), nil, vals, vals,
			hash("app_hash"), hash("cons_hash"), hash("results_hash"), 0, len(keys))

		primary := mockp.New(
			chainID,
			map[int64]*types.SignedHeader{
				1: diffHeader1,
				2: diffHeader2,
			},
			valSet,
		)

		c, err := light.NewClient(
			chainID,
			light.TrustOptions{
				Period: 4 * time.Hour,
				Height: 2,
				Hash:   diffHeader2.Hash(),
			},
			primary,
			[]provider.Provider{primary},
			trustedStore,
			light.Logger(log.TestingLogger()),
		)
		require.NoError(t, err)

		// Check we no longer have the invalid 1st header (+header+).
		h, err := c.TrustedHeader(1)
		assert.Error(t, err)
		assert.Nil(t, h)

		valSet, _, err := c.TrustedValidatorSet(1)
		assert.Error(t, err)
		assert.Nil(t, valSet)
	}
}

// trustedHeader.Height > options.Height
func TestClientRestoresTrustedHeaderAfterStartup3(t *testing.T) {
	// 1. options.Hash == trustedHeader.Hash
	{
		// load the first three headers into the trusted store
		trustedStore := dbs.New(dbm.NewMemDB(), chainID)
		err := trustedStore.SaveSignedHeaderAndValidatorSet(h1, vals)
		require.NoError(t, err)

		err = trustedStore.SaveSignedHeaderAndValidatorSet(h2, vals)
		require.NoError(t, err)

		c, err := light.NewClient(
			chainID,
			trustOptions,
			fullNode,
			[]provider.Provider{fullNode},
			trustedStore,
			light.Logger(log.TestingLogger()),
		)
		require.NoError(t, err)

		// Check we still have the 1st header (+header+).
		h, err := c.TrustedHeader(1)
		assert.NoError(t, err)
		assert.NotNil(t, h)
		assert.Equal(t, h.Hash(), h1.Hash())

		valSet, _, err := c.TrustedValidatorSet(1)
		assert.NoError(t, err)
		assert.NotNil(t, valSet)
		if assert.NotNil(t, valSet) {
			assert.Equal(t, h.ValidatorsHash.Bytes(), valSet.Hash())
		}

		// Check we no longer have 2nd header (+header2+).
		h, err = c.TrustedHeader(2)
		assert.Error(t, err)
		assert.Nil(t, h)

		valSet, _, err = c.TrustedValidatorSet(2)
		assert.Error(t, err)
		assert.Nil(t, valSet)

		h, err = c.TrustedHeader(3)
		assert.Error(t, err)
		assert.Nil(t, h)
	}

	// 2. options.Hash != trustedHeader.Hash
	// This could happen if previous provider was lying to us.
	{
		trustedStore := dbs.New(dbm.NewMemDB(), chainID)
		err := trustedStore.SaveSignedHeaderAndValidatorSet(h1, vals)
		require.NoError(t, err)

		// header1 != header
		header1 := keys.GenSignedHeader(chainID, 1, bTime.Add(1*time.Hour), nil, vals, vals,
			hash("app_hash"), hash("cons_hash"), hash("results_hash"), 0, len(keys))

		header2 := keys.GenSignedHeader(chainID, 2, bTime.Add(2*time.Hour), nil, vals, vals,
			hash("app_hash"), hash("cons_hash"), hash("results_hash"), 0, len(keys))
		err = trustedStore.SaveSignedHeaderAndValidatorSet(header2, vals)
		require.NoError(t, err)

		primary := mockp.New(
			chainID,
			map[int64]*types.SignedHeader{
				1: header1,
			},
			valSet,
		)

		c, err := light.NewClient(
			chainID,
			light.TrustOptions{
				Period: 4 * time.Hour,
				Height: 1,
				Hash:   header1.Hash(),
			},
			primary,
			[]provider.Provider{primary},
			trustedStore,
			light.Logger(log.TestingLogger()),
		)
		require.NoError(t, err)

		// Check we have swapped invalid 1st header (+header+) with correct one (+header1+).
		h, err := c.TrustedHeader(1)
		assert.NoError(t, err)
		assert.NotNil(t, h)
		assert.Equal(t, h.Hash(), header1.Hash())

		valSet, _, err := c.TrustedValidatorSet(1)
		assert.NoError(t, err)
		assert.NotNil(t, valSet)
		if assert.NotNil(t, valSet) {
			assert.Equal(t, h.ValidatorsHash.Bytes(), valSet.Hash())
		}

		// Check we no longer have invalid 2nd header (+header2+).
		h, err = c.TrustedHeader(2)
		assert.Error(t, err)
		assert.Nil(t, h)

		valSet, _, err = c.TrustedValidatorSet(2)
		assert.Error(t, err)
		assert.Nil(t, valSet)
	}
}

func TestClient_Update(t *testing.T) {
	c, err := light.NewClient(
		chainID,
		trustOptions,
		fullNode,
		[]provider.Provider{fullNode},
		dbs.New(dbm.NewMemDB(), chainID),
		light.Logger(log.TestingLogger()),
	)
	require.NoError(t, err)

	// should result in downloading & verifying header #3
	h, err := c.Update(bTime.Add(2 * time.Hour))
	assert.NoError(t, err)
	if assert.NotNil(t, h) {
		assert.EqualValues(t, 3, h.Height)
	}

	valSet, _, err := c.TrustedValidatorSet(3)
	assert.NoError(t, err)
	if assert.NotNil(t, valSet) {
		assert.Equal(t, h.ValidatorsHash.Bytes(), valSet.Hash())
	}
}

func TestClient_Concurrency(t *testing.T) {
	c, err := light.NewClient(
		chainID,
		trustOptions,
		fullNode,
		[]provider.Provider{fullNode},
		dbs.New(dbm.NewMemDB(), chainID),
		light.Logger(log.TestingLogger()),
	)
	require.NoError(t, err)

	_, err = c.VerifyHeaderAtHeight(2, bTime.Add(2*time.Hour))
	require.NoError(t, err)

	var wg sync.WaitGroup
	for i := 0; i < 100; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()

			// NOTE: Cleanup, Stop, VerifyHeaderAtHeight and Verify are not supposed
			// to be concurrenly safe.

			assert.Equal(t, chainID, c.ChainID())

			_, err := c.LastTrustedHeight()
			assert.NoError(t, err)

			_, err = c.FirstTrustedHeight()
			assert.NoError(t, err)

			h, err := c.TrustedHeader(1)
			assert.NoError(t, err)
			assert.NotNil(t, h)

			vals, _, err := c.TrustedValidatorSet(2)
			assert.NoError(t, err)
			assert.NotNil(t, vals)
		}()
	}

	wg.Wait()
}

func TestClientReplacesPrimaryWithWitnessIfPrimaryIsUnavailable(t *testing.T) {
	c, err := light.NewClient(
		chainID,
		trustOptions,
		deadNode,
		[]provider.Provider{fullNode, fullNode},
		dbs.New(dbm.NewMemDB(), chainID),
		light.Logger(log.TestingLogger()),
		light.MaxRetryAttempts(1),
	)

	require.NoError(t, err)
	_, err = c.Update(bTime.Add(2 * time.Hour))
	require.NoError(t, err)

	assert.NotEqual(t, c.Primary(), deadNode)
	assert.Equal(t, 1, len(c.Witnesses()))
}

func TestClient_BackwardsVerification(t *testing.T) {
	{
		trustHeader, _ := largeFullNode.SignedHeader(6)
		c, err := light.NewClient(
			chainID,
			light.TrustOptions{
				Period: 4 * time.Minute,
				Height: trustHeader.Height,
				Hash:   trustHeader.Hash(),
			},
			largeFullNode,
			[]provider.Provider{largeFullNode},
			dbs.New(dbm.NewMemDB(), chainID),
			light.Logger(log.TestingLogger()),
		)
		require.NoError(t, err)

		// 1) verify before the trusted header using backwards => expect no error
		h, err := c.VerifyHeaderAtHeight(5, bTime.Add(6*time.Minute))
		require.NoError(t, err)
		if assert.NotNil(t, h) {
			assert.EqualValues(t, 5, h.Height)
		}

		// 2) untrusted header is expired but trusted header is not => expect no error
		h, err = c.VerifyHeaderAtHeight(3, bTime.Add(8*time.Minute))
		assert.NoError(t, err)
		assert.NotNil(t, h)

		// 3) already stored headers should return the header without error
		h, err = c.VerifyHeaderAtHeight(5, bTime.Add(6*time.Minute))
		assert.NoError(t, err)
		assert.NotNil(t, h)

		// 4a) First verify latest header
		_, err = c.VerifyHeaderAtHeight(9, bTime.Add(9*time.Minute))
		require.NoError(t, err)

		// 4b) Verify backwards using bisection => expect no error
		_, err = c.VerifyHeaderAtHeight(7, bTime.Add(10*time.Minute))
		assert.NoError(t, err)
		// shouldn't have verified this header in the process
		_, err = c.TrustedHeader(8)
		assert.Error(t, err)

		// 5) trusted header has expired => expect error
		_, err = c.VerifyHeaderAtHeight(1, bTime.Add(20*time.Minute))
		assert.Error(t, err)

		// 6) Try bisection method, but closest header (at 7) has expired
		// so change to backwards => expect no error
		_, err = c.VerifyHeaderAtHeight(8, bTime.Add(12*time.Minute))
		assert.NoError(t, err)

	}
	{
		testCases := []struct {
			provider provider.Provider
		}{
			{
				// 7) provides incorrect height
				mockp.New(
					chainID,
					map[int64]*types.SignedHeader{
						1: h1,
						2: keys.GenSignedHeader(chainID, 1, bTime.Add(1*time.Hour), nil, vals, vals,
							hash("app_hash"), hash("cons_hash"), hash("results_hash"), 0, len(keys)),
						3: h3,
					},
					valSet,
				),
			},
			{
				// 8) provides incorrect hash
				mockp.New(
					chainID,
					map[int64]*types.SignedHeader{
						1: h1,
						2: keys.GenSignedHeader(chainID, 2, bTime.Add(30*time.Minute), nil, vals, vals,
							hash("app_hash"), hash("cons_hash"), hash("results_hash"), 0, len(keys)),
						3: h3,
					},
					valSet,
				),
			},
		}

		for _, tc := range testCases {
			c, err := light.NewClient(
				chainID,
				light.TrustOptions{
					Period: 1 * time.Hour,
					Height: 3,
					Hash:   h3.Hash(),
				},
				tc.provider,
				[]provider.Provider{tc.provider},
				dbs.New(dbm.NewMemDB(), chainID),
				light.Logger(log.TestingLogger()),
			)
			require.NoError(t, err)

			_, err = c.VerifyHeaderAtHeight(2, bTime.Add(1*time.Hour).Add(1*time.Second))
			assert.Error(t, err)
		}
	}
}

func TestClient_NewClientFromTrustedStore(t *testing.T) {
	// 1) Initiate DB and fill with a "trusted" header
	db := dbs.New(dbm.NewMemDB(), chainID)
	err := db.SaveSignedHeaderAndValidatorSet(h1, vals)
	require.NoError(t, err)

	c, err := light.NewClientFromTrustedStore(
		chainID,
		trustPeriod,
		deadNode,
		[]provider.Provider{deadNode},
		db,
	)
	require.NoError(t, err)

	// 2) Check header exists (deadNode is being used to ensure we're not getting
	// it from primary)
	h, err := c.TrustedHeader(1)
	assert.NoError(t, err)
	assert.EqualValues(t, 1, h.Height)

	valSet, _, err := c.TrustedValidatorSet(1)
	assert.NoError(t, err)
	assert.NotNil(t, valSet)
	if assert.NotNil(t, valSet) {
		assert.Equal(t, h.ValidatorsHash.Bytes(), valSet.Hash())
	}
}

func TestNewClientErrorsIfAllWitnessesUnavailable(t *testing.T) {
	_, err := light.NewClient(
		chainID,
		trustOptions,
		fullNode,
		[]provider.Provider{deadNode, deadNode},
		dbs.New(dbm.NewMemDB(), chainID),
		light.Logger(log.TestingLogger()),
		light.MaxRetryAttempts(1),
	)
	if assert.Error(t, err) {
		assert.Contains(t, err.Error(), "awaiting response from all witnesses exceeded dropout time")
	}
}

func TestClientRemovesWitnessIfItSendsUsIncorrectHeader(t *testing.T) {
	// different headers hash then primary plus less than 1/3 signed (no fork)
	badProvider1 := mockp.New(
		chainID,
		map[int64]*types.SignedHeader{
			1: h1,
			2: keys.GenSignedHeaderLastBlockID(chainID, 2, bTime.Add(30*time.Minute), nil, vals, vals,
				hash("app_hash2"), hash("cons_hash"), hash("results_hash"),
				len(keys), len(keys), types.BlockID{Hash: h1.Hash()}),
		},
		map[int64]*types.ValidatorSet{
			1: vals,
			2: vals,
		},
	)
	// header is empty
	badProvider2 := mockp.New(
		chainID,
		map[int64]*types.SignedHeader{
			1: h1,
			2: h2,
			3: {Header: nil, Commit: nil},
		},
		map[int64]*types.ValidatorSet{
			1: vals,
			2: vals,
		},
	)

	c, err := light.NewClient(
		chainID,
		trustOptions,
		fullNode,
		[]provider.Provider{badProvider1, badProvider2},
		dbs.New(dbm.NewMemDB(), chainID),
		light.Logger(log.TestingLogger()),
		light.MaxRetryAttempts(1),
	)
	// witness should have behaved properly -> no error
	require.NoError(t, err)
	assert.EqualValues(t, 2, len(c.Witnesses()))

	// witness behaves incorrectly -> removed from list, no error
	h, err := c.VerifyHeaderAtHeight(2, bTime.Add(2*time.Hour))
	assert.NoError(t, err)
	assert.EqualValues(t, 1, len(c.Witnesses()))
	// header should still be verified
	assert.EqualValues(t, 2, h.Height)

	// remaining withness doesn't have header -> error
	_, err = c.VerifyHeaderAtHeight(3, bTime.Add(2*time.Hour))
	if assert.Error(t, err) {
		assert.Equal(t, "awaiting response from all witnesses exceeded dropout time", err.Error())
	}
	assert.EqualValues(t, 0, len(c.Witnesses()))

	// no witnesses left, will not be allowed to verify a header
	_, err = c.VerifyHeaderAtHeight(3, bTime.Add(2*time.Hour))
	if assert.Error(t, err) {
		assert.Equal(t, "no witnesses connected. please reset light client", err.Error())
	}
}

func TestClientTrustedValidatorSet(t *testing.T) {
	noValSetNode := mockp.New(
		chainID,
		headerSet,
		map[int64]*types.ValidatorSet{
			1: nil,
			2: nil,
			3: nil,
		},
	)

	differentVals, _ := types.RandValidatorSet(10, 100)

	badValSetNode := mockp.New(
		chainID,
		headerSet,
		map[int64]*types.ValidatorSet{
			1: vals,
			2: differentVals,
			3: differentVals,
		},
	)

	c, err := light.NewClient(
		chainID,
		trustOptions,
		noValSetNode,
		[]provider.Provider{badValSetNode, fullNode, fullNode},
		dbs.New(dbm.NewMemDB(), chainID),
		light.Logger(log.TestingLogger()),
	)
	require.NoError(t, err)
	assert.Equal(t, 2, len(c.Witnesses()))

	_, err = c.VerifyHeaderAtHeight(2, bTime.Add(2*time.Hour).Add(1*time.Second))
	assert.Error(t, err)
	assert.Equal(t, 1, len(c.Witnesses()))

	_, err = c.VerifyHeaderAtHeight(2, bTime.Add(2*time.Hour).Add(1*time.Second))
	assert.NoError(t, err)

	valSet, height, err := c.TrustedValidatorSet(0)
	assert.NoError(t, err)
	assert.NotNil(t, valSet)
	assert.EqualValues(t, 2, height)
}

func TestClientReportsConflictingHeadersEvidence(t *testing.T) {
	// fullNode2 sends us different header
	altH2 := keys.GenSignedHeaderLastBlockID(chainID, 2, bTime.Add(30*time.Minute), nil, vals, vals,
		hash("app_hash2"), hash("cons_hash"), hash("results_hash"),
		0, len(keys), types.BlockID{Hash: h1.Hash()})
	fullNode2 := mockp.New(
		chainID,
		map[int64]*types.SignedHeader{
			1: h1,
			2: altH2,
		},
		map[int64]*types.ValidatorSet{
			1: vals,
			2: vals,
		},
	)

	c, err := light.NewClient(
		chainID,
		trustOptions,
		fullNode,
		[]provider.Provider{fullNode2},
		dbs.New(dbm.NewMemDB(), chainID),
		light.Logger(log.TestingLogger()),
		light.MaxRetryAttempts(1),
	)
	require.NoError(t, err)

	// Check verification returns an error.
	_, err = c.VerifyHeaderAtHeight(2, bTime.Add(2*time.Hour))
	if assert.Error(t, err) {
		assert.Contains(t, err.Error(), "does not match one")
	}

	// Check evidence was sent to both full nodes.
	ev := &types.ConflictingHeadersEvidence{H1: h2, H2: altH2}
	assert.True(t, fullNode2.HasEvidence(ev))
	assert.True(t, fullNode.HasEvidence(ev))
}

func TestClientPrunesHeadersAndValidatorSets(t *testing.T) {
	c, err := light.NewClient(
		chainID,
		trustOptions,
		fullNode,
		[]provider.Provider{fullNode},
		dbs.New(dbm.NewMemDB(), chainID),
		light.Logger(log.TestingLogger()),
		light.PruningSize(1),
	)
	require.NoError(t, err)
	_, err = c.TrustedHeader(1)
	require.NoError(t, err)

	h, err := c.Update(bTime.Add(2 * time.Hour))
	require.NoError(t, err)
	require.Equal(t, int64(3), h.Height)

	_, err = c.TrustedHeader(1)
	assert.Error(t, err)
}

func TestClientEnsureValidHeadersAndValSets(t *testing.T) {
	emptyValSet := &types.ValidatorSet{
		Validators: nil,
		Proposer:   nil,
	}

	testCases := []struct {
		headers map[int64]*types.SignedHeader
		vals    map[int64]*types.ValidatorSet
		err     bool
	}{
		{
			headerSet,
			valSet,
			false,
		},
		{
			headerSet,
			map[int64]*types.ValidatorSet{
				1: vals,
				2: vals,
				3: nil,
			},
			true,
		},
		{
			map[int64]*types.SignedHeader{
				1: h1,
				2: h2,
				3: nil,
			},
			valSet,
			true,
		},
		{
			headerSet,
			map[int64]*types.ValidatorSet{
				1: vals,
				2: vals,
				3: emptyValSet,
			},
			true,
		},
	}

	for _, tc := range testCases {
		badNode := mockp.New(
			chainID,
			tc.headers,
			tc.vals,
		)
		c, err := light.NewClient(
			chainID,
			trustOptions,
			badNode,
			[]provider.Provider{badNode, badNode},
			dbs.New(dbm.NewMemDB(), chainID),
			light.MaxRetryAttempts(1),
		)
		require.NoError(t, err)

		_, err = c.VerifyHeaderAtHeight(3, bTime.Add(2*time.Hour))
		if tc.err {
			assert.Error(t, err)
		} else {
			assert.NoError(t, err)
		}
	}

}