lite2: fetch missing headers (#4362)

Closes #4328

When TrustedHeader(height) is called, if the height is less than the trusted height but the header is not in the trusted store then a function finds the previous lowest height with a trusted header and performs a forwards sequential verification to the header of the height that was given. If no error is found it updates the trusted store with the header and validator set for that height and can then return them to the user.

Commits:

* drafted trusted header

* created function to find previous trusted height

* updates missing headers less than the trusted height

* minor cosmetic tweaks

* incorporated suggestions

* lite2: implement Backwards verification

and add SignedHeaderAfter func to Store interface

Refs https://github.com/tendermint/tendermint/issues/4328#issuecomment-581878549

* remove unused method

* write tests

* start with next height in SignedHeaderAfter func

* fix linter errors

* address Callum's comments

Co-authored-by: Anton Kaliaev <anton.kalyaev@gmail.com>

lite2/client.go

@@ -398,11 +398,13 @@ func (c *Client) Stop() {
 	c.routinesWaitGroup.Wait()
 }
 
-// TrustedHeader returns a trusted header at the given height (0 - the latest)
-// or nil if no such header exist.
+// TrustedHeader returns a trusted header at the given height (0 - the latest).
+// If a header is missing in trustedStore (e.g. it was skipped during
+// bisection), it will be downloaded from primary.
 //
-// Headers, which can't be trusted anymore, are removed once a day (can be
-// changed with RemoveNoLongerTrustedHeadersPeriod option).
+// Headers along with validator sets, which can't be trusted anymore, are
+// removed once a day (can be changed with RemoveNoLongerTrustedHeadersPeriod
+// option).
 // .
 // height must be >= 0.
 //
@@ -411,7 +413,7 @@ func (c *Client) Stop() {
 //  - there are some issues with the trusted store, although that should not
 //  happen normally;
 //  - negative height is passed;
-//  - header is not found.
+//  - header has not been verified yet
 //
 // Safe for concurrent use by multiple goroutines.
 func (c *Client) TrustedHeader(height int64, now time.Time) (*types.SignedHeader, error) {
@@ -419,20 +421,35 @@ func (c *Client) TrustedHeader(height int64, now time.Time) (*types.SignedHeader
 		return nil, errors.New("negative height")
 	}
 
+	// 1) Get latest height.
+	latestHeight, err := c.LastTrustedHeight()
+	if err != nil {
+		return nil, err
+	}
+	if latestHeight == -1 {
+		return nil, errors.New("no headers exist")
+	}
+	if height > latestHeight {
+		return nil, errors.Errorf("unverified header requested (latest: %d)", latestHeight)
+	}
 	if height == 0 {
-		var err error
-		height, err = c.LastTrustedHeight()
-		if err != nil {
-			return nil, err
-		}
+		height = latestHeight
 	}
 
+	// 2) Get header from store.
 	h, err := c.trustedStore.SignedHeader(height)
-	if err != nil {
+	switch {
+	case errors.Is(err, store.ErrSignedHeaderNotFound):
+		// 2.1) If not found, try to fetch header from primary.
+		h, err = c.fetchMissingTrustedHeader(height, now)
+		if err != nil {
+			return nil, err
+		}
+	case err != nil:
 		return nil, err
 	}
 
-	// Ensure header can still be trusted.
+	// 3) Ensure header can still be trusted.
 	if HeaderExpired(h, c.trustingPeriod, now) {
 		return nil, ErrOldHeaderExpired{h.Time.Add(c.trustingPeriod), now}
 	}
@@ -440,23 +457,31 @@ func (c *Client) TrustedHeader(height int64, now time.Time) (*types.SignedHeader
 	return h, nil
 }
 
-// TrustedValidatorSet returns a trusted validator set at the given height (0 -
-// the latest) or nil if no such validator set exist. The second return
-// parameter is height validator set corresponds to (useful when you pass 0).
+// TrustedValidatorSet returns a trusted validator set at the given height. If
+// a validator set is missing in trustedStore (e.g. the associated header was
+// skipped during bisection), it will be downloaded from primary. The second
+// return parameter is height validator set corresponds to (useful when you
+// pass 0).
 //
 // height must be >= 0.
 //
+// Headers along with validator sets, which can't be trusted anymore, are
+// removed once a day (can be changed with RemoveNoLongerTrustedHeadersPeriod
+// option).
+//
 // It returns an error if:
 //	- header signed by that validator set expired (ErrOldHeaderExpired)
 //  - there are some issues with the trusted store, although that should not
 //  happen normally;
 //  - negative height is passed;
-//  - validator set is not found.
+//  - header signed by that validator set has not been verified yet
 //
 // Safe for concurrent use by multiple goroutines.
 func (c *Client) TrustedValidatorSet(height int64, now time.Time) (*types.ValidatorSet, error) {
-	// Checks height is positive and header is not expired.
-	_, err := c.TrustedHeader(height, now)
+	// Checks height is positive and header (note: height - 1) is not expired.
+	// Additionally, it fetches validator set from primary if it's missing in
+	// store.
+	_, err := c.TrustedHeader(height-1, now)
 	if err != nil {
 		return nil, err
 	}
@@ -757,6 +782,79 @@ func (c *Client) fetchHeaderAndValsAtHeight(height int64) (*types.SignedHeader,
 	return h, vals, nil
 }
 
+// fetchMissingTrustedHeader finds the closest height after the
+// requested height and does backwards verification.
+func (c *Client) fetchMissingTrustedHeader(height int64, now time.Time) (*types.SignedHeader, error) {
+	c.logger.Info("Fetching missing header", "height", height)
+
+	closestHeader, err := c.trustedStore.SignedHeaderAfter(height)
+	if err != nil {
+		return nil, errors.Wrapf(err, "can't get signed header after %d", height)
+	}
+
+	// Perform backwards verification from closestHeader to header at the given
+	// height.
+	h, err := c.backwards(height, closestHeader, now)
+	if err != nil {
+		return nil, err
+	}
+
+	// Fetch next validator set from primary and persist it.
+	nextVals, err := c.validatorSetFromPrimary(height + 1)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to obtain the vals #%d", height)
+	}
+	if !bytes.Equal(h.NextValidatorsHash, nextVals.Hash()) {
+		return nil, errors.Errorf("expected next validator's hash %X, but got %X",
+			h.NextValidatorsHash, nextVals.Hash())
+	}
+	if err := c.trustedStore.SaveSignedHeaderAndNextValidatorSet(h, nextVals); err != nil {
+		return nil, errors.Wrap(err, "failed to save trusted header")
+	}
+
+	return h, nil
+}
+
+// Backwards verification (see VerifyHeaderBackwards func in the spec)
+func (c *Client) backwards(toHeight int64, fromHeader *types.SignedHeader, now time.Time) (*types.SignedHeader, error) {
+	var (
+		trustedHeader   = fromHeader
+		untrustedHeader *types.SignedHeader
+		err             error
+	)
+
+	for i := trustedHeader.Height - 1; i >= toHeight; i-- {
+		untrustedHeader, err = c.signedHeaderFromPrimary(i)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to obtain the header #%d", i)
+		}
+
+		if err := untrustedHeader.ValidateBasic(c.chainID); err != nil {
+			return nil, errors.Wrap(err, "untrustedHeader.ValidateBasic failed")
+		}
+
+		if !untrustedHeader.Time.Before(trustedHeader.Time) {
+			return nil, errors.Errorf("expected older header time %v to be before newer header time %v",
+				untrustedHeader.Time,
+				trustedHeader.Time)
+		}
+
+		if HeaderExpired(untrustedHeader, c.trustingPeriod, now) {
+			return nil, ErrOldHeaderExpired{untrustedHeader.Time.Add(c.trustingPeriod), now}
+		}
+
+		if !bytes.Equal(untrustedHeader.Hash(), trustedHeader.LastBlockID.Hash) {
+			return nil, errors.Errorf("older header hash %X does not match trusted header's last block %X",
+				untrustedHeader.Hash(),
+				trustedHeader.LastBlockID.Hash)
+		}
+
+		trustedHeader = untrustedHeader
+	}
+
+	return trustedHeader, nil
+}
+
 // compare header with one from a random witness.
 func (c *Client) compareNewHeaderWithRandomWitness(h *types.SignedHeader) error {
 	c.providerMutex.Lock()
@@ -916,8 +1014,15 @@ func (c *Client) signedHeaderFromPrimary(height int64) (*types.SignedHeader, err
 		c.providerMutex.Lock()
 		h, err := c.primary.SignedHeader(height)
 		c.providerMutex.Unlock()
-		if err == nil || err == provider.ErrSignedHeaderNotFound {
-			return h, err
+		if err == nil {
+			// sanity check
+			if height > 0 && h.Height != height {
+				return nil, errors.Errorf("expected %d height, got %d", height, h.Height)
+			}
+			return h, nil
+		}
+		if err == provider.ErrSignedHeaderNotFound {
+			return nil, err
 		}
 		time.Sleep(backoffTimeout(attempt))
 	}
@@ -936,10 +1041,10 @@ func (c *Client) signedHeaderFromPrimary(height int64) (*types.SignedHeader, err
 func (c *Client) validatorSetFromPrimary(height int64) (*types.ValidatorSet, error) {
 	for attempt := 1; attempt <= maxRetryAttempts; attempt++ {
 		c.providerMutex.Lock()
-		h, err := c.primary.ValidatorSet(height)
+		vals, err := c.primary.ValidatorSet(height)
 		c.providerMutex.Unlock()
 		if err == nil || err == provider.ErrValidatorSetNotFound {
-			return h, err
+			return vals, err
 		}
 		time.Sleep(backoffTimeout(attempt))
 	}

lite2/client_test.go

@@ -919,7 +919,70 @@ func TestProvider_Replacement(t *testing.T) {
 	err = c.Start()
 	require.NoError(t, err)
 	defer c.Stop()
+
 	assert.NotEqual(t, c.Primary(), primary)
 	assert.Equal(t, 0, len(c.Witnesses()))
+}
+
+func TestProvider_TrustedHeaderFetchesMissingHeader(t *testing.T) {
+	const (
+		chainID = "TestProvider_TrustedHeaderFetchesMissingHeader"
+	)
 
+	var (
+		keys = genPrivKeys(4)
+		// 20, 30, 40, 50 - the first 3 don't have 2/3, the last 3 do!
+		vals     = keys.ToValidators(20, 10)
+		bTime, _ = time.Parse(time.RFC3339, "2006-01-02T15:04:05Z")
+		h1       = keys.GenSignedHeader(chainID, 1, bTime, nil, vals, vals,
+			[]byte("app_hash"), []byte("cons_hash"), []byte("results_hash"), 0, len(keys))
+		h2 = keys.GenSignedHeaderLastBlockID(chainID, 2, bTime.Add(30*time.Minute), nil, vals, vals,
+			[]byte("app_hash"), []byte("cons_hash"), []byte("results_hash"), 0, len(keys), types.BlockID{Hash: h1.Hash()})
+		h3 = keys.GenSignedHeaderLastBlockID(chainID, 3, bTime.Add(1*time.Hour), nil, vals, vals,
+			[]byte("app_hash"), []byte("cons_hash"), []byte("results_hash"), 0, len(keys), types.BlockID{Hash: h2.Hash()})
+		primary = mockp.New(
+			chainID,
+			map[int64]*types.SignedHeader{
+				1: h1,
+				2: h2,
+				3: h3,
+			},
+			map[int64]*types.ValidatorSet{
+				1: vals,
+				2: vals,
+				3: vals,
+				4: vals,
+			},
+		)
+	)
+
+	c, err := NewClient(
+		chainID,
+		TrustOptions{
+			Period: 1 * time.Hour,
+			Height: 3,
+			Hash:   h3.Hash(),
+		},
+		primary,
+		[]provider.Provider{primary},
+		dbs.New(dbm.NewMemDB(), chainID),
+		UpdatePeriod(0),
+		Logger(log.TestingLogger()),
+	)
+	require.NoError(t, err)
+	err = c.Start()
+	require.NoError(t, err)
+	defer c.Stop()
+
+	// 1) header is missing => expect no error
+	h, err := c.TrustedHeader(2, bTime.Add(1*time.Hour).Add(1*time.Second))
+	require.NoError(t, err)
+	if assert.NotNil(t, h) {
+		assert.EqualValues(t, 2, h.Height)
+	}
+
+	// 2) header is missing, but it's expired => expect error
+	h, err = c.TrustedHeader(1, bTime.Add(1*time.Hour).Add(1*time.Second))
+	assert.Error(t, err)
+	assert.Nil(t, h)
 }

lite2/store/db/db.go

@@ -81,7 +81,7 @@ func (s *dbs) SignedHeader(height int64) (*types.SignedHeader, error) {
 		panic(err)
 	}
 	if len(bz) == 0 {
-		return nil, errors.New("signed header not found")
+		return nil, store.ErrSignedHeaderNotFound
 	}
 
 	var signedHeader *types.SignedHeader
@@ -100,7 +100,7 @@ func (s *dbs) ValidatorSet(height int64) (*types.ValidatorSet, error) {
 		panic(err)
 	}
 	if len(bz) == 0 {
-		return nil, errors.New("validator set not found")
+		return nil, store.ErrValidatorSetNotFound
 	}
 
 	var valSet *types.ValidatorSet
@@ -154,6 +154,32 @@ func (s *dbs) FirstSignedHeaderHeight() (int64, error) {
 	return -1, nil
 }
 
+func (s *dbs) SignedHeaderAfter(height int64) (*types.SignedHeader, error) {
+	if height <= 0 {
+		panic("negative or zero height")
+	}
+
+	itr, err := s.db.ReverseIterator(
+		s.shKey(height+1),
+		append(s.shKey(1<<63-1), byte(0x00)),
+	)
+	if err != nil {
+		panic(err)
+	}
+	defer itr.Close()
+
+	for itr.Valid() {
+		key := itr.Key()
+		_, existingHeight, ok := parseShKey(key)
+		if ok {
+			return s.SignedHeader(existingHeight)
+		}
+		itr.Next()
+	}
+
+	panic(fmt.Sprintf("no header after height %d. make sure height is not greater than latest existing height", height))
+}
+
 func (s *dbs) shKey(height int64) []byte {
 	return []byte(fmt.Sprintf("sh/%s/%020d", s.prefix, height))
 }

lite2/store/db/db_test.go

@@ -74,3 +74,22 @@ func Test_SaveSignedHeaderAndNextValidatorSet(t *testing.T) {
 	require.Error(t, err)
 	assert.Nil(t, valSet)
 }
+
+func Test_SignedHeaderAfter(t *testing.T) {
+	dbStore := New(dbm.NewMemDB(), "Test_SignedHeaderAfter")
+
+	assert.Panics(t, func() {
+		dbStore.SignedHeaderAfter(0)
+		dbStore.SignedHeaderAfter(100)
+	})
+
+	err := dbStore.SaveSignedHeaderAndNextValidatorSet(
+		&types.SignedHeader{Header: &types.Header{Height: 2}}, &types.ValidatorSet{})
+	require.NoError(t, err)
+
+	h, err := dbStore.SignedHeaderAfter(1)
+	require.NoError(t, err)
+	if assert.NotNil(t, h) {
+		assert.EqualValues(t, 2, h.Height)
+	}
+}

lite2/store/errors.go

@@ -0,0 +1,12 @@
+package store
+
+import "errors"
+
+var (
+	// ErrSignedHeaderNotFound is returned when a store does not have the
+	// requested header.
+	ErrSignedHeaderNotFound = errors.New("signed header not found")
+	// ErrValidatorSetNotFound is returned when a store does not have the
+	// requested validator set.
+	ErrValidatorSetNotFound = errors.New("validator set not found")
+)

lite2/store/store.go

@@ -21,14 +21,14 @@ type Store interface {
 	//
 	// height must be > 0.
 	//
-	// If SignedHeader is not found, an error is returned.
+	// If SignedHeader is not found, ErrSignedHeaderNotFound is returned.
 	SignedHeader(height int64) (*types.SignedHeader, error)
 
 	// ValidatorSet returns the ValidatorSet that corresponds to height.
 	//
 	// height must be > 0.
 	//
-	// If ValidatorSet is not found, an error is returned.
+	// If ValidatorSet is not found, ErrValidatorSetNotFound is returned.
 	ValidatorSet(height int64) (*types.ValidatorSet, error)
 
 	// LastSignedHeaderHeight returns the last (newest) SignedHeader height.
@@ -40,4 +40,9 @@ type Store interface {
 	//
 	// If the store is empty, -1 and nil error are returned.
 	FirstSignedHeaderHeight() (int64, error)
+
+	// SignedHeaderAfter returns the SignedHeader after the certain height.
+	//
+	// height must be > 0 && <= LastSignedHeaderHeight.
+	SignedHeaderAfter(height int64) (*types.SignedHeader, error)
 }

lite2/test_helpers.go

@@ -147,3 +147,16 @@ func (pkz privKeys) GenSignedHeader(chainID string, height int64, bTime time.Tim
 		Commit: pkz.signHeader(header, first, last),
 	}
 }
+
+// GenSignedHeaderLastBlockID calls genHeader and signHeader and combines them into a SignedHeader.
+func (pkz privKeys) GenSignedHeaderLastBlockID(chainID string, height int64, bTime time.Time, txs types.Txs,
+	valset, nextValset *types.ValidatorSet, appHash, consHash, resHash []byte, first, last int,
+	lastBlockID types.BlockID) *types.SignedHeader {
+
+	header := genHeader(chainID, height, bTime, txs, valset, nextValset, appHash, consHash, resHash)
+	header.LastBlockID = lastBlockID
+	return &types.SignedHeader{
+		Header: header,
+		Commit: pkz.signHeader(header, first, last),
+	}
+}