lite2: cache headers in bisection (#4562)

Closes: #4546

The algorithm uses an array to store the headers and validators and populates it at every bisection (which is an unsuccessful verification). When a successful verification finally occurs it updates the new trusted header, trims that header from the cache (the array) and sets the depth pointer back to 0. Instead of retrieving new headers it will use the cached headers, incrementing in depth until it reaches the end of the cache which by then it will start to retrieve new headers from the provider. 

Mathematically, this method doesn't properly bisect after the first round but it will always choose a pivot header that is within 1/8th of the upper header's height. I.e. if we are trying to jump 128 headers, the maximum offset from bisection height (64) is 64 + 16(128/8) = 80, therefore a better heuristic would be to obtain the new pivot header height as the middle of these two numbers which would therefore mean to multiply it by 9/16ths instead of 1/2  (sorry this might be a bit more complicated in writing but I can try better explain if someone is interested). Therefore I would also, upon consensus, propose that we change the pivot height to 9/16th's of the previous height

CHANGELOG_PENDING.md

@@ -21,6 +21,7 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi
 - [p2p] [\#4548](https://github.com/tendermint/tendermint/pull/4548) Add ban list to address book (@cmwaters)
 - [privval] \#4534 Add `error` as a return value on`GetPubKey()`
 - [Docker] \#4569 Default configuration added to docker image (you can still mount your own config the same way) (@greg-szabo)
+- [lite2] [\#4562](https://github.com/tendermint/tendermint/pull/4562) Cache headers when using bisection (@cmwaters)
 
 ### BUG FIXES:
 

lite2/client.go

@@ -24,6 +24,11 @@ const (
 
 	defaultPruningSize      = 1000
 	defaultMaxRetryAttempts = 10
+	// For bisection, when using the cache of headers from the previous batch,
+	// they will always be at a height greater than 1/2 (normal bisection) so to
+	// find something in between the range, 9/16 is used.
+	bisectionNumerator   = 9
+	bisectionDenominator = 16
 )
 
 // Option sets a parameter for the light client.
@@ -449,29 +454,6 @@ func (c *Client) compareWithLatestHeight(height int64) (int64, error) {
 	return height, nil
 }
 
-// LastTrustedHeight returns a last trusted height. -1 and nil are returned if
-// there are no trusted headers.
-//
-// Safe for concurrent use by multiple goroutines.
-func (c *Client) LastTrustedHeight() (int64, error) {
-	return c.trustedStore.LastSignedHeaderHeight()
-}
-
-// FirstTrustedHeight returns a first trusted height. -1 and nil are returned if
-// there are no trusted headers.
-//
-// Safe for concurrent use by multiple goroutines.
-func (c *Client) FirstTrustedHeight() (int64, error) {
-	return c.trustedStore.FirstSignedHeaderHeight()
-}
-
-// ChainID returns the chain ID the light client was configured with.
-//
-// Safe for concurrent use by multiple goroutines.
-func (c *Client) ChainID() string {
-	return c.chainID
-}
-
 // VerifyHeaderAtHeight fetches header and validators at the given height
 // and calls VerifyHeader. It returns header immediately if such exists in
 // trustedStore (no verification is needed).
@@ -504,16 +486,17 @@ func (c *Client) VerifyHeaderAtHeight(height int64, now time.Time) (*types.Signe
 
 // VerifyHeader verifies new header against the trusted state. It returns
 // immediately if newHeader exists in trustedStore (no verification is
-// needed).
+// needed). Else it performs one of the two types of verification:
 //
 // SequentialVerification: verifies that 2/3 of the trusted validator set has
 // signed the new header. If the headers are not adjacent, **all** intermediate
-// headers will be requested.
+// headers will be requested. Intermediate headers are not saved to database.
 //
 // SkippingVerification(trustLevel): verifies that {trustLevel} of the trusted
 // validator set has signed the new header. If it's not the case and the
 // headers are not adjacent, bisection is performed and necessary (not all)
 // intermediate headers will be requested. See the specification for details.
+// Intermediate headers are not saved to database.
 // https://github.com/tendermint/spec/blob/master/spec/consensus/light-client.md
 //
 // It returns ErrOldHeaderExpired if the latest trusted header expired.
@@ -584,65 +567,6 @@ func (c *Client) verifyHeader(newHeader *types.SignedHeader, newVals *types.Vali
 	return c.updateTrustedHeaderAndVals(newHeader, newVals)
 }
 
-// Primary returns the primary provider.
-//
-// NOTE: provider may be not safe for concurrent access.
-func (c *Client) Primary() provider.Provider {
-	c.providerMutex.Lock()
-	defer c.providerMutex.Unlock()
-	return c.primary
-}
-
-// Witnesses returns the witness providers.
-//
-// NOTE: providers may be not safe for concurrent access.
-func (c *Client) Witnesses() []provider.Provider {
-	c.providerMutex.Lock()
-	defer c.providerMutex.Unlock()
-	return c.witnesses
-}
-
-// Cleanup removes all the data (headers and validator sets) stored. Note: the
-// client must be stopped at this point.
-func (c *Client) Cleanup() error {
-	c.logger.Info("Removing all the data")
-	c.latestTrustedHeader = nil
-	c.latestTrustedVals = nil
-	return c.trustedStore.Prune(0)
-}
-
-// cleanupAfter deletes all headers & validator sets after +height+. It also
-// resets latestTrustedHeader to the latest header.
-func (c *Client) cleanupAfter(height int64) error {
-	nextHeight := height
-
-	for {
-		h, err := c.trustedStore.SignedHeaderAfter(nextHeight)
-		if err == store.ErrSignedHeaderNotFound {
-			break
-		} else if err != nil {
-			return errors.Wrapf(err, "failed to get header after %d", nextHeight)
-		}
-
-		err = c.trustedStore.DeleteSignedHeaderAndValidatorSet(h.Height)
-		if err != nil {
-			c.logger.Error("can't remove a trusted header & validator set", "err", err,
-				"height", h.Height)
-		}
-
-		nextHeight = h.Height
-	}
-
-	c.latestTrustedHeader = nil
-	c.latestTrustedVals = nil
-	err := c.restoreTrustedHeaderAndVals()
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 // see VerifyHeader
 func (c *Client) sequence(
 	initiallyTrustedHeader *types.SignedHeader,
@@ -707,6 +631,10 @@ func (c *Client) sequence(
 }
 
 // see VerifyHeader
+// Bisection finds the middle header between a trusted and new header, reiterating the action until it
+// verifies a header. A cache of headers requested by the primary is kept such that when a
+// verification is made, and the light client tries again to verify the new header in the middle,
+// the light client does not need to ask for all the same headers again.
 func (c *Client) bisection(
 	initiallyTrustedHeader *types.SignedHeader,
 	initiallyTrustedVals *types.ValidatorSet,
@@ -714,40 +642,53 @@ func (c *Client) bisection(
 	newVals *types.ValidatorSet,
 	now time.Time) error {
 
+	type headerSet struct {
+		sh     *types.SignedHeader
+		valSet *types.ValidatorSet
+	}
+
 	var (
+		headerCache = []headerSet{{newHeader, newVals}}
+		depth       = 0
+
 		trustedHeader = initiallyTrustedHeader
 		trustedVals   = initiallyTrustedVals
-
-		interimHeader = newHeader
-		interimVals   = newVals
 	)
 
 	for {
 		c.logger.Debug("Verify newHeader against trustedHeader",
 			"trustedHeight", trustedHeader.Height,
 			"trustedHash", hash2str(trustedHeader.Hash()),
-			"newHeight", interimHeader.Height,
-			"newHash", hash2str(interimHeader.Hash()))
+			"newHeight", headerCache[depth].sh.Height,
+			"newHash", hash2str(headerCache[depth].sh.Hash()))
 
-		err := Verify(c.chainID, trustedHeader, trustedVals, interimHeader, interimVals, c.trustingPeriod, now,
-			c.trustLevel)
+		err := Verify(c.chainID, trustedHeader, trustedVals, headerCache[depth].sh, headerCache[depth].valSet,
+			c.trustingPeriod, now, c.trustLevel)
 		switch err.(type) {
 		case nil:
-			if interimHeader.Height == newHeader.Height {
+			// Have we verified the last header
+			if depth == 0 {
 				return nil
 			}
-
-			// Update the lower bound to the previous upper bound
-			trustedHeader, trustedVals = interimHeader, interimVals
-			// Update the upper bound to the untrustedHeader
-			interimHeader, interimVals = newHeader, newVals
+			// If not, update the lower bound to the previous upper bound
+			trustedHeader, trustedVals = headerCache[depth].sh, headerCache[depth].valSet
+			// Remove the untrusted header at the lower bound in the header cache - it's no longer useful
+			headerCache = headerCache[:depth]
+			// Reset the cache depth so that we start from the upper bound again
+			depth = 0
 
 		case ErrNewValSetCantBeTrusted:
-			pivotHeight := (interimHeader.Height + trustedHeader.Height) / 2
-			interimHeader, interimVals, err = c.fetchHeaderAndValsAtHeight(pivotHeight)
-			if err != nil {
-				return err
+			// do add another header to the end of the cache
+			if depth == len(headerCache)-1 {
+				pivotHeight := (headerCache[depth].sh.Height + trustedHeader.
+					Height) * bisectionNumerator / bisectionDenominator
+				interimHeader, interimVals, err := c.fetchHeaderAndValsAtHeight(pivotHeight)
+				if err != nil {
+					return err
+				}
+				headerCache = append(headerCache, headerSet{interimHeader, interimVals})
 			}
+			depth++
 
 		case ErrInvalidHeader:
 			c.logger.Error("primary sent invalid header -> replacing", "err", err)
@@ -756,16 +697,98 @@ func (c *Client) bisection(
 				c.logger.Error("Can't replace primary", "err", replaceErr)
 				// return original error
 				return errors.Wrapf(err, "verify from #%d to #%d failed",
-					trustedHeader.Height, interimHeader.Height)
+					trustedHeader.Height, headerCache[depth].sh.Height)
 			}
 			// attempt to verify the header again
 			continue
 
 		default:
 			return errors.Wrapf(err, "verify from #%d to #%d failed",
-				trustedHeader.Height, interimHeader.Height)
+				trustedHeader.Height, headerCache[depth].sh.Height)
+		}
+	}
+}
+
+// LastTrustedHeight returns a last trusted height. -1 and nil are returned if
+// there are no trusted headers.
+//
+// Safe for concurrent use by multiple goroutines.
+func (c *Client) LastTrustedHeight() (int64, error) {
+	return c.trustedStore.LastSignedHeaderHeight()
+}
+
+// FirstTrustedHeight returns a first trusted height. -1 and nil are returned if
+// there are no trusted headers.
+//
+// Safe for concurrent use by multiple goroutines.
+func (c *Client) FirstTrustedHeight() (int64, error) {
+	return c.trustedStore.FirstSignedHeaderHeight()
+}
+
+// ChainID returns the chain ID the light client was configured with.
+//
+// Safe for concurrent use by multiple goroutines.
+func (c *Client) ChainID() string {
+	return c.chainID
+}
+
+// Primary returns the primary provider.
+//
+// NOTE: provider may be not safe for concurrent access.
+func (c *Client) Primary() provider.Provider {
+	c.providerMutex.Lock()
+	defer c.providerMutex.Unlock()
+	return c.primary
+}
+
+// Witnesses returns the witness providers.
+//
+// NOTE: providers may be not safe for concurrent access.
+func (c *Client) Witnesses() []provider.Provider {
+	c.providerMutex.Lock()
+	defer c.providerMutex.Unlock()
+	return c.witnesses
+}
+
+// Cleanup removes all the data (headers and validator sets) stored. Note: the
+// client must be stopped at this point.
+func (c *Client) Cleanup() error {
+	c.logger.Info("Removing all the data")
+	c.latestTrustedHeader = nil
+	c.latestTrustedVals = nil
+	return c.trustedStore.Prune(0)
+}
+
+// cleanupAfter deletes all headers & validator sets after +height+. It also
+// resets latestTrustedHeader to the latest header.
+func (c *Client) cleanupAfter(height int64) error {
+	nextHeight := height
+
+	for {
+		h, err := c.trustedStore.SignedHeaderAfter(nextHeight)
+		if err == store.ErrSignedHeaderNotFound {
+			break
+		} else if err != nil {
+			return errors.Wrapf(err, "failed to get header after %d", nextHeight)
+		}
+
+		err = c.trustedStore.DeleteSignedHeaderAndValidatorSet(h.Height)
+		if err != nil {
+			c.logger.Error("can't remove a trusted header & validator set", "err", err,
+				"height", h.Height)
 		}
+
+		nextHeight = h.Height
 	}
+
+	c.latestTrustedHeader = nil
+	c.latestTrustedVals = nil
+	err := c.restoreTrustedHeaderAndVals()
+	if err != nil {
+		return err
+	}
+
+	return nil
 }
 
 func (c *Client) updateTrustedHeaderAndVals(h *types.SignedHeader, vals *types.ValidatorSet) error {

lite2/doc.go

@@ -97,6 +97,18 @@ Verify function verifies a new header against some trusted header. See
 https://github.com/tendermint/spec/blob/master/spec/consensus/light-client/verification.md
 for details.
 
+There are two methods of verification: sequential and bisection
+
+Sequential uses the headers hashes and the validator sets to verify each adjacent header until
+it reaches the target header.
+
+Bisection finds the middle header between a trusted and new header, reiterating the action until it
+verifies a header. A cache of headers requested by the primary is kept such that when a
+verification is made, and the light client tries again to verify the new header in the middle,
+the light client does not need to ask for all the same headers again.
+
+refer to docs/imgs/light_client_bisection_alg.png
+
 ## 3. Secure RPC proxy
 
 Tendermint RPC exposes a lot of info, but a malicious node could return any
@@ -108,5 +120,8 @@ some other node.
 See
 https://docs.tendermint.com/master/tendermint-core/light-client-protocol.html
 for usage example.
+Or see
+https://github.com/tendermint/spec/tree/master/spec/consensus/light-client
+for the full spec
 */
 package lite