diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 7d63fba03..1b4c41870 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -29,3 +29,4 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi - [rpc/jsonrpc/server] \#6191 Correctly unmarshal `RPCRequest` when data is `null` (@melekes) - [p2p] \#6289 Fix "unknown channels" bug on CustomReactors (@gchaincl) +- [light/evidence] Adds logic to handle forward lunatic attacks (@cmwaters) diff --git a/docs/architecture/adr-047-handling-evidence-from-light-client.md b/docs/architecture/adr-047-handling-evidence-from-light-client.md index 54624dd5c..5dcbc6ae4 100644 --- a/docs/architecture/adr-047-handling-evidence-from-light-client.md +++ b/docs/architecture/adr-047-handling-evidence-from-light-client.md @@ -8,6 +8,7 @@ * 14-08-2020: Introduce light traces (listed now as an alternative approach) * 20-08-2020: Light client produces evidence when detected instead of passing to full node * 16-09-2020: Post-implementation revision +* 15-03-2020: Ammends for the case of a forward lunatic attack ### Glossary of Terms @@ -106,8 +107,10 @@ This is done with: ```golang func (c *Client) examineConflictingHeaderAgainstTrace( trace []*types.LightBlock, - divergentHeader *types.SignedHeader, - source provider.Provider, now time.Time) ([]*types.LightBlock, *types.LightBlock, error) + targetBlock *types.LightBlock, + source provider.Provider, + now time.Time, + ) ([]*types.LightBlock, *types.LightBlock, error) ``` which performs the following @@ -117,16 +120,21 @@ because witnesses cannot be added and removed after the client is initialized. B as a sanity check. If this fails we have to drop the witness. 2. Querying and verifying the witness's headers using bisection at the same heights of all the -intermediary headers of the primary (In the above example this is A, B, C, D, F, H). If bisection fails or the witness stops responding then -we can call the witness faulty and drop it. +intermediary headers of the primary (In the above example this is A, B, C, D, F, H). If bisection fails +or the witness stops responding then we can call the witness faulty and drop it. -3. We eventually reach a verified header by the witness which is not the same as the intermediary header (In the above example this is E). -This is the point of bifurcation (This could also be the last header). +3. We eventually reach a verified header by the witness which is not the same as the intermediary header +(In the above example this is E). This is the point of bifurcation (This could also be the last header). + +4. There is a unique case where the trace that is being examined against has blocks that have a greater +height than the targetBlock. This can occur as part of a forward lunatic attack where the primary has +provided a light block that has a height greater than the head of the chain (see Appendix B). In this +case, the light client will verify the sources blocks up to the targetBlock and return the block in the +trace that is directly after the targetBlock in height as the `ConflictingBlock` This function then returns the trace of blocks from the witness node between the common header and the -divergent header of the primary as it -is likely as seen in the example to the right below that multiple headers where required in order to -verify the divergent one. This trace will +divergent header of the primary as it is likely, as seen in the example to the right, that multiple +headers where required in order to verify the divergent one. This trace will be used later (as is also described later in this document). ![](../imgs/bifurcation-point.png) @@ -225,3 +233,22 @@ would be validators that currently still have something staked. Not only this but there was a large degree of extra computation required in storing all the currently staked validators that could possibly fall into the group of being a phantom validator. Given this, it was removed. + +## Appendix B + +A unique flavor of lunatic attack is a forward lunatic attack. This is where a malicious +node provides a header with a height greater than the height of the blockchain. Thus there +are no witnesses capable of rebutting the malicious header. Such an attack will also +require an accomplice, i.e. at least one other witness to also return the same forged header. +Although such attacks can be any arbitrary height ahead, they must still remain within the +clock drift of the light clients real time. Therefore, to detect such an attack, a light +client will wait for a time + +``` +2 * MAX_CLOCK_DRIFT + LAG +``` + +for a witness to provide the latest block it has. Given the time constraints, if the witness +is operating at the head of the blockchain, it will have a header with an earlier height but +a later timestamp. This can be used to prove that the primary has submitted a lunatic header +which violates monotonically increasing time. diff --git a/docs/architecture/adr-059-evidence-composition-and-lifecycle.md b/docs/architecture/adr-059-evidence-composition-and-lifecycle.md index 707a18bfb..5b86164b2 100644 --- a/docs/architecture/adr-059-evidence-composition-and-lifecycle.md +++ b/docs/architecture/adr-059-evidence-composition-and-lifecycle.md @@ -4,6 +4,7 @@ - 04/09/2020: Initial Draft (Unabridged) - 07/09/2020: First Version +- 13.03.21: Ammendment to accomodate forward lunatic attack ## Scope @@ -159,7 +160,7 @@ For `LightClientAttack` - Fetch the common signed header and val set from the common height and use skipping verification to verify the conflicting header -- Fetch the trusted signed header at the same height as the conflicting header and compare with the conflicting header to work out which type of attack it is and in doing so return the malicious validators. +- Fetch the trusted signed header at the same height as the conflicting header and compare with the conflicting header to work out which type of attack it is and in doing so return the malicious validators. NOTE: If the node doesn't have the signed header at the height of the conflicting header, it instead fetches the latest header it has and checks to see if it can prove the evidence based on a violation of header time. This is known as forward lunatic attack. - If equivocation, return the validators that signed for the commits of both the trusted and signed header @@ -167,7 +168,11 @@ For `LightClientAttack` - If amnesia, return no validators (since we can't know which validators are malicious). This also means that we don't currently send amnesia evidence to the application, although we will introduce more robust amnesia evidence handling in future Tendermint Core releases -- For each validator, check the look up table to make sure there already isn't evidence against this validator +- Check that the hashes of the conflicting header and the trusted header are different + +- In the case of a forward lunatic attack, where the trusted header height is less than the conflicting header height, the node checks that the time of the trusted header is later than the time of conflicting header. This proves that the conflicting header breaks monotonically increasing time. If the node doesn't have a trusted header with a later time then it is unable to validate the evidence for now. + +- Lastly, for each validator, check the look up table to make sure there already isn't evidence against this validator After verification we persist the evidence with the key `height/hash` to the pending evidence database in the evidence pool with the following format: diff --git a/evidence/mocks/block_store.go b/evidence/mocks/block_store.go index e24cac4e0..e6205939a 100644 --- a/evidence/mocks/block_store.go +++ b/evidence/mocks/block_store.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.1.0. DO NOT EDIT. +// Code generated by mockery v0.0.0-dev. DO NOT EDIT. package mocks @@ -12,6 +12,20 @@ type BlockStore struct { mock.Mock } +// Height provides a mock function with given fields: +func (_m *BlockStore) Height() int64 { + ret := _m.Called() + + var r0 int64 + if rf, ok := ret.Get(0).(func() int64); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(int64) + } + + return r0 +} + // LoadBlockCommit provides a mock function with given fields: height func (_m *BlockStore) LoadBlockCommit(height int64) *types.Commit { ret := _m.Called(height) diff --git a/evidence/services.go b/evidence/services.go index 5e15b38b4..274433cbe 100644 --- a/evidence/services.go +++ b/evidence/services.go @@ -9,4 +9,5 @@ import ( type BlockStore interface { LoadBlockMeta(height int64) *types.BlockMeta LoadBlockCommit(height int64) *types.Commit + Height() int64 } diff --git a/evidence/verify.go b/evidence/verify.go index 0721ade9a..cf0df8cfb 100644 --- a/evidence/verify.go +++ b/evidence/verify.go @@ -71,7 +71,21 @@ func (evpool *Pool) verify(evidence types.Evidence) error { if evidence.Height() != ev.ConflictingBlock.Height { trustedHeader, err = getSignedHeader(evpool.blockStore, ev.ConflictingBlock.Height) if err != nil { - return err + // FIXME: This multi step process is a bit unergonomic. We may want to consider a more efficient process + // that doesn't require as much io and is atomic. + + // If the node doesn't have a block at the height of the conflicting block, then this could be + // a forward lunatic attack. Thus the node must get the latest height it has + latestHeight := evpool.blockStore.Height() + trustedHeader, err = getSignedHeader(evpool.blockStore, latestHeight) + if err != nil { + return err + } + if trustedHeader.Time.Before(ev.ConflictingBlock.Time) { + return fmt.Errorf("latest block time (%v) is before conflicting block time (%v)", + trustedHeader.Time, ev.ConflictingBlock.Time, + ) + } } } @@ -119,36 +133,47 @@ func (evpool *Pool) verify(evidence types.Evidence) error { // the following checks: // - the common header from the full node has at least 1/3 voting power which is also present in // the conflicting header's commit +// - 2/3+ of the conflicting validator set correctly signed the conflicting block // - the nodes trusted header at the same height as the conflicting header has a different hash +// +// CONTRACT: must run ValidateBasic() on the evidence before verifying +// must check that the evidence has not expired (i.e. is outside the maximum age threshold) func VerifyLightClientAttack(e *types.LightClientAttackEvidence, commonHeader, trustedHeader *types.SignedHeader, commonVals *types.ValidatorSet, now time.Time, trustPeriod time.Duration) error { - // In the case of lunatic attack we need to perform a single verification jump between the - // common header and the conflicting one - if commonHeader.Height != trustedHeader.Height { - err := light.Verify(commonHeader, commonVals, e.ConflictingBlock.SignedHeader, e.ConflictingBlock.ValidatorSet, - trustPeriod, now, 0*time.Second, light.DefaultTrustLevel) + // In the case of lunatic attack there will be a different commonHeader height. Therefore the node perform a single + // verification jump between the common header and the conflicting one + if commonHeader.Height != e.ConflictingBlock.Height { + err := commonVals.VerifyCommitLightTrusting(trustedHeader.ChainID, e.ConflictingBlock.Commit, light.DefaultTrustLevel) if err != nil { - return fmt.Errorf("skipping verification from common to conflicting header failed: %w", err) - } - } else { - // in the case of equivocation and amnesia we expect some header hashes to be correctly derived - if isInvalidHeader(trustedHeader.Header, e.ConflictingBlock.Header) { - return errors.New("common height is the same as conflicting block height so expected the conflicting" + - " block to be correctly derived yet it wasn't") - } - // ensure that 2/3 of the validator set did vote for this block - if err := e.ConflictingBlock.ValidatorSet.VerifyCommitLight(trustedHeader.ChainID, e.ConflictingBlock.Commit.BlockID, - e.ConflictingBlock.Height, e.ConflictingBlock.Commit); err != nil { - return fmt.Errorf("invalid commit from conflicting block: %w", err) + return fmt.Errorf("skipping verification of conflicting block failed: %w", err) } + + // In the case of equivocation and amnesia we expect all header hashes to be correctly derived + } else if isInvalidHeader(trustedHeader.Header, e.ConflictingBlock.Header) { + return errors.New("common height is the same as conflicting block height so expected the conflicting" + + " block to be correctly derived yet it wasn't") } + // Verify that the 2/3+ commits from the conflicting validator set were for the conflicting header + if err := e.ConflictingBlock.ValidatorSet.VerifyCommitLight(trustedHeader.ChainID, e.ConflictingBlock.Commit.BlockID, + e.ConflictingBlock.Height, e.ConflictingBlock.Commit); err != nil { + return fmt.Errorf("invalid commit from conflicting block: %w", err) + } + + // Assert the correct amount of voting power of the validator set if evTotal, valsTotal := e.TotalVotingPower, commonVals.TotalVotingPower(); evTotal != valsTotal { return fmt.Errorf("total voting power from the evidence and our validator set does not match (%d != %d)", evTotal, valsTotal) } - if bytes.Equal(trustedHeader.Hash(), e.ConflictingBlock.Hash()) { + // check in the case of a forward lunatic attack that monotonically increasing time has been violated + if e.ConflictingBlock.Height > trustedHeader.Height && e.ConflictingBlock.Time.After(trustedHeader.Time) { + return fmt.Errorf("conflicting block doesn't violate monotonically increasing time (%v is after %v)", + e.ConflictingBlock.Time, trustedHeader.Time, + ) + + // In all other cases check that the hashes of the conflicting header and the trusted header are different + } else if bytes.Equal(trustedHeader.Hash(), e.ConflictingBlock.Hash()) { return fmt.Errorf("trusted header hash matches the evidence's conflicting header hash: %X", trustedHeader.Hash()) } diff --git a/evidence/verify_test.go b/evidence/verify_test.go index 0e72582b2..5b3adf7f1 100644 --- a/evidence/verify_test.go +++ b/evidence/verify_test.go @@ -34,6 +34,7 @@ func TestVerifyLightClientAttack_Lunatic(t *testing.T) { commonHeader := makeHeaderRandom(4) commonHeader.Time = defaultEvidenceTime trustedHeader := makeHeaderRandom(10) + trustedHeader.Time = defaultEvidenceTime.Add(1 * time.Hour) conflictingHeader := makeHeaderRandom(10) conflictingHeader.Time = defaultEvidenceTime.Add(1 * time.Hour) @@ -89,6 +90,30 @@ func TestVerifyLightClientAttack_Lunatic(t *testing.T) { assert.Error(t, err) ev.TotalVotingPower = 20 + forwardConflictingHeader := makeHeaderRandom(11) + forwardConflictingHeader.Time = defaultEvidenceTime.Add(30 * time.Minute) + forwardConflictingHeader.ValidatorsHash = conflictingVals.Hash() + forwardBlockID := makeBlockID(forwardConflictingHeader.Hash(), 1000, []byte("partshash")) + forwardVoteSet := types.NewVoteSet(evidenceChainID, 11, 1, tmproto.SignedMsgType(2), conflictingVals) + forwardCommit, err := types.MakeCommit(forwardBlockID, 11, 1, forwardVoteSet, conflictingPrivVals, defaultEvidenceTime) + require.NoError(t, err) + forwardLunaticEv := &types.LightClientAttackEvidence{ + ConflictingBlock: &types.LightBlock{ + SignedHeader: &types.SignedHeader{ + Header: forwardConflictingHeader, + Commit: forwardCommit, + }, + ValidatorSet: conflictingVals, + }, + CommonHeight: 4, + TotalVotingPower: 20, + ByzantineValidators: commonVals.Validators, + Timestamp: defaultEvidenceTime, + } + err = evidence.VerifyLightClientAttack(forwardLunaticEv, commonSignedHeader, trustedSignedHeader, commonVals, + defaultEvidenceTime.Add(2*time.Hour), 3*time.Hour) + assert.NoError(t, err) + state := sm.State{ LastBlockTime: defaultEvidenceTime.Add(2 * time.Hour), LastBlockHeight: 11, @@ -100,8 +125,10 @@ func TestVerifyLightClientAttack_Lunatic(t *testing.T) { blockStore := &mocks.BlockStore{} blockStore.On("LoadBlockMeta", int64(4)).Return(&types.BlockMeta{Header: *commonHeader}) blockStore.On("LoadBlockMeta", int64(10)).Return(&types.BlockMeta{Header: *trustedHeader}) + blockStore.On("LoadBlockMeta", int64(11)).Return(nil) blockStore.On("LoadBlockCommit", int64(4)).Return(commit) blockStore.On("LoadBlockCommit", int64(10)).Return(trustedCommit) + blockStore.On("Height").Return(int64(10)) pool, err := evidence.NewPool(dbm.NewMemDB(), stateStore, blockStore) require.NoError(t, err) @@ -126,6 +153,9 @@ func TestVerifyLightClientAttack_Lunatic(t *testing.T) { err = pool.CheckEvidence(evList) assert.Error(t, err) + evList = types.EvidenceList{forwardLunaticEv} + err = pool.CheckEvidence(evList) + assert.NoError(t, err) } func TestVerifyLightClientAttack_Equivocation(t *testing.T) { diff --git a/light/client.go b/light/client.go index 8399aabfe..d835c1b99 100644 --- a/light/client.go +++ b/light/client.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "sort" + "sync" "time" "github.com/tendermint/tendermint/libs/log" @@ -35,6 +36,9 @@ const ( // - http://vancouver-webpages.com/time/web.html // - https://blog.codinghorror.com/keeping-time-on-the-pc/ defaultMaxClockDrift = 10 * time.Second + + // 10s is sufficient for most networks. + defaultMaxBlockLag = 10 * time.Second ) // Option sets a parameter for the light client. @@ -100,13 +104,27 @@ func MaxRetryAttempts(max uint16) Option { } // MaxClockDrift defines how much new header's time can drift into -// the future. Default: 10s. +// the future relative to the light clients local time. Default: 10s. func MaxClockDrift(d time.Duration) Option { return func(c *Client) { c.maxClockDrift = d } } +// MaxBlockLag represents the maximum time difference between the realtime +// that a block is received and the timestamp of that block. +// One can approximate it to the maximum block production time +// +// As an example, say the light client received block B at a time +// 12:05 (this is the real time) and the time on the block +// was 12:00. Then the lag here is 5 minutes. +// Default: 10s +func MaxBlockLag(d time.Duration) Option { + return func(c *Client) { + c.maxBlockLag = d + } +} + // Client represents a light client, connected to a single chain, which gets // light blocks from a primary provider, verifies them either sequentially or by // skipping some and stores them in a trusted store (usually, a local FS). @@ -119,6 +137,7 @@ type Client struct { trustLevel tmmath.Fraction maxRetryAttempts uint16 // see MaxRetryAttempts option maxClockDrift time.Duration + maxBlockLag time.Duration // Mutex for locking during changes of the light clients providers providerMutex tmsync.Mutex @@ -205,6 +224,7 @@ func NewClientFromTrustedStore( trustLevel: DefaultTrustLevel, maxRetryAttempts: defaultMaxRetryAttempts, maxClockDrift: defaultMaxClockDrift, + maxBlockLag: defaultMaxBlockLag, primary: primary, witnesses: witnesses, trustedStore: trustedStore, @@ -636,17 +656,10 @@ func (c *Client) verifySequential( // If some intermediate header is invalid, replace the primary and try // again. c.logger.Error("primary sent invalid header -> replacing", "err", err) - replaceErr := c.replacePrimaryProvider() - if replaceErr != nil { - c.logger.Error("Can't replace primary", "err", replaceErr) - // return original error - return err - } - replacementBlock, fErr := c.lightBlockFromPrimary(ctx, newLightBlock.Height) - if fErr != nil { - c.logger.Error("Can't fetch light block from primary", "err", fErr) - // return original error + replacementBlock, removeErr := c.findNewPrimary(ctx, newLightBlock.Height, true) + if removeErr != nil { + c.logger.Debug("failed to replace primary. Returning original error", "err", removeErr) return err } @@ -770,17 +783,10 @@ func (c *Client) verifySkippingAgainstPrimary( // If some intermediate header is invalid, replace the primary and try // again. c.logger.Error("primary sent invalid header -> replacing", "err", err) - replaceErr := c.replacePrimaryProvider() - if replaceErr != nil { - c.logger.Error("Can't replace primary", "err", replaceErr) - // return original error - return err - } - replacementBlock, fErr := c.lightBlockFromPrimary(ctx, newLightBlock.Height) - if fErr != nil { - c.logger.Error("Can't fetch light block from primary", "err", fErr) - // return original error + replacementBlock, removeErr := c.findNewPrimary(ctx, newLightBlock.Height, true) + if removeErr != nil { + c.logger.Error("failed to replace primary. Returning original error", "err", removeErr) return err } @@ -935,15 +941,25 @@ func (c *Client) backwards( "newHeight", interimHeader.Height, "newHash", interimHeader.Hash()) if err := VerifyBackwards(interimHeader, verifiedHeader); err != nil { - c.logger.Error("primary sent invalid header -> replacing", "err", err) - if replaceErr := c.replacePrimaryProvider(); replaceErr != nil { - c.logger.Error("Can't replace primary", "err", replaceErr) - // return original error - return fmt.Errorf("verify backwards from %d to %d failed: %w", - verifiedHeader.Height, interimHeader.Height, err) + // verification has failed + c.logger.Error("backwards verification failed, replacing primary...", "err", err, "primary", c.primary) + + // the client tries to see if it can get a witness to continue with the request + newPrimarysBlock, replaceErr := c.findNewPrimary(ctx, newHeader.Height, true) + if replaceErr != nil { + c.logger.Debug("failed to replace primary. Returning original error", "err", replaceErr) + return err } - // we need to verify the header at the same height again - continue + + // before continuing we must check that they have the same target header to validate + if !bytes.Equal(newPrimarysBlock.Hash(), newHeader.Hash()) { + c.logger.Debug("replaced primary but new primary has a different block to the initial one") + // return the original error + return err + } + + // try again with the new primary + return c.backwards(ctx, verifiedHeader, newPrimarysBlock.Header) } verifiedHeader = interimHeader } @@ -951,52 +967,144 @@ func (c *Client) backwards( return nil } -// NOTE: requires a providerMutex locked. -func (c *Client) removeWitness(idx int) { - switch len(c.witnesses) { - case 0: - panic(fmt.Sprintf("wanted to remove %d element from empty witnesses slice", idx)) - case 1: - c.witnesses = make([]provider.Provider, 0) +// lightBlockFromPrimary retrieves the lightBlock from the primary provider +// at the specified height. This method also handles provider behavior as follows: +// +// 1. If the provider does not respond or does not have the block, it tries again +// with a different provider +// 2. If all providers return the same error, the light client forwards the error to +// where the initial request came from +// 3. If the provider provides an invalid light block, is deemed unreliable or returns +// any other error, the primary is permanently dropped and is replaced by a witness. +func (c *Client) lightBlockFromPrimary(ctx context.Context, height int64) (*types.LightBlock, error) { + c.providerMutex.Lock() + l, err := c.primary.LightBlock(ctx, height) + c.providerMutex.Unlock() + + switch err { + case nil: + // Everything went smoothly. We reset the lightBlockRequests and return the light block + return l, nil + + case provider.ErrNoResponse, provider.ErrLightBlockNotFound: + // we find a new witness to replace the primary + c.logger.Debug("error from light block request from primary, replacing...", + "error", err, "height", height, "primary", c.primary) + return c.findNewPrimary(ctx, height, false) + default: - c.witnesses[idx] = c.witnesses[len(c.witnesses)-1] + // The light client has most likely received either provider.ErrUnreliableProvider or provider.ErrBadLightBlock + // These errors mean that the light client should drop the primary and try with another provider instead + c.logger.Error("error from light block request from primary, removing...", + "error", err, "height", height, "primary", c.primary) + return c.findNewPrimary(ctx, height, true) + } +} + +// NOTE: requires a providerMutex lock +func (c *Client) removeWitnesses(indexes []int) error { + // check that we will still have witnesses remaining + if len(c.witnesses) <= len(indexes) { + return ErrNoWitnesses + } + + // we need to make sure that we remove witnesses by index in the reverse + // order so as to not affect the indexes themselves + sort.Ints(indexes) + for i := len(indexes) - 1; i >= 0; i-- { + c.witnesses[indexes[i]] = c.witnesses[len(c.witnesses)-1] c.witnesses = c.witnesses[:len(c.witnesses)-1] } + + return nil } -// replaceProvider takes the first alternative provider and promotes it as the -// primary provider. -func (c *Client) replacePrimaryProvider() error { +type witnessResponse struct { + lb *types.LightBlock + witnessIndex int + err error +} + +// findNewPrimary concurrently sends a light block request, promoting the first witness to return +// a valid light block as the new primary. The remove option indicates whether the primary should be +// entire removed or just appended to the back of the witnesses list. This method also handles witness +// errors. If no witness is available, it returns the last error of the witness. +func (c *Client) findNewPrimary(ctx context.Context, height int64, remove bool) (*types.LightBlock, error) { c.providerMutex.Lock() defer c.providerMutex.Unlock() if len(c.witnesses) <= 1 { - return errNoWitnesses{} + return nil, ErrNoWitnesses } - c.primary = c.witnesses[0] - c.witnesses = c.witnesses[1:] - c.logger.Info("Replacing primary with the first witness", "new_primary", c.primary) - return nil -} + var ( + witnessResponsesC = make(chan witnessResponse, len(c.witnesses)) + witnessesToRemove []int + lastError error + wg sync.WaitGroup + ) -// lightBlockFromPrimary retrieves the lightBlock from the primary provider -// at the specified height. Handles dropout by the primary provider by swapping -// with an alternative provider. -func (c *Client) lightBlockFromPrimary(ctx context.Context, height int64) (*types.LightBlock, error) { - c.providerMutex.Lock() - l, err := c.primary.LightBlock(ctx, height) - c.providerMutex.Unlock() - if err != nil { - c.logger.Debug("Error on light block request from primary", "error", err) - replaceErr := c.replacePrimaryProvider() - if replaceErr != nil { - return nil, fmt.Errorf("%v. Tried to replace primary but: %w", err.Error(), replaceErr) + // send out a light block request to all witnesses + subctx, cancel := context.WithCancel(ctx) + defer cancel() + for index := range c.witnesses { + wg.Add(1) + go func(witnessIndex int, witnessResponsesC chan witnessResponse) { + defer wg.Done() + + lb, err := c.witnesses[witnessIndex].LightBlock(subctx, height) + witnessResponsesC <- witnessResponse{lb, witnessIndex, err} + }(index, witnessResponsesC) + } + + // process all the responses as they come in + for i := 0; i < cap(witnessResponsesC); i++ { + response := <-witnessResponsesC + switch response.err { + // success! We have found a new primary + case nil: + cancel() // cancel all remaining requests to other witnesses + + wg.Wait() // wait for all goroutines to finish + + // if we are not intending on removing the primary then append the old primary to the end of the witness slice + if !remove { + c.witnesses = append(c.witnesses, c.primary) + } + + // promote respondent as the new primary + c.logger.Debug("found new primary", "primary", c.witnesses[response.witnessIndex]) + c.primary = c.witnesses[response.witnessIndex] + + // add promoted witness to the list of witnesses to be removed + witnessesToRemove = append(witnessesToRemove, response.witnessIndex) + + // remove witnesses marked as bad (the client must do this before we alter the witness slice and change the indexes + // of witnesses). Removal is done in descending order + if err := c.removeWitnesses(witnessesToRemove); err != nil { + return nil, err + } + + // return the light block that new primary responded with + return response.lb, nil + + // process benign errors by logging them only + case provider.ErrNoResponse, provider.ErrLightBlockNotFound: + lastError = response.err + c.logger.Debug("error on light block request from witness", + "error", response.err, "primary", c.witnesses[response.witnessIndex]) + continue + + // process malevolent errors like ErrUnreliableProvider and ErrBadLightBlock by removing the witness + default: + lastError = response.err + c.logger.Error("error on light block request from witness, removing...", + "error", response.err, "primary", c.witnesses[response.witnessIndex]) + witnessesToRemove = append(witnessesToRemove, response.witnessIndex) } - // replace primary and request a light block again - return c.lightBlockFromPrimary(ctx, height) } - return l, err + + return nil, lastError } // compareFirstHeaderWithWitnesses compares h with all witnesses. If any @@ -1043,11 +1151,9 @@ and remove witness. Otherwise, use the different primary`, e.WitnessIndex), "wit } - // we need to make sure that we remove witnesses by index in the reverse - // order so as to not affect the indexes themselves - sort.Ints(witnessesToRemove) - for i := len(witnessesToRemove) - 1; i >= 0; i-- { - c.removeWitness(witnessesToRemove[i]) + // remove witnesses that have misbehaved + if err := c.removeWitnesses(witnessesToRemove); err != nil { + return err } return nil diff --git a/light/detector.go b/light/detector.go index b0e0a129a..2dbf15134 100644 --- a/light/detector.go +++ b/light/detector.go @@ -5,14 +5,13 @@ import ( "context" "errors" "fmt" - "sort" "time" "github.com/tendermint/tendermint/light/provider" "github.com/tendermint/tendermint/types" ) -// The detector component of the light client detect and handles attacks on the light client. +// The detector component of the light client detects and handles attacks on the light client. // More info here: // tendermint/docs/architecture/adr-047-handling-evidence-from-light-client.md @@ -21,7 +20,7 @@ import ( // It takes the target verified header and compares it with the headers of a set of // witness providers that the light client is connected to. If a conflicting header // is returned it verifies and examines the conflicting header against the verified -// trace that was produced from the primary. If successful it produces two sets of evidence +// trace that was produced from the primary. If successful, it produces two sets of evidence // and sends them to the opposite provider before halting. // // If there are no conflictinge headers, the light client deems the verified target header @@ -65,50 +64,14 @@ func (c *Client) detectDivergence(ctx context.Context, primaryTrace []*types.Lig // need to find the point that the headers diverge and examine this for any evidence of an attack. // // We combine these actions together, verifying the witnesses headers and outputting the trace - // which captures the bifurcation point and if successful provides the information to create - supportingWitness := c.witnesses[e.WitnessIndex] - witnessTrace, primaryBlock, err := c.examineConflictingHeaderAgainstTrace( - ctx, - primaryTrace, - e.Block.SignedHeader, - supportingWitness, - now, - ) + // which captures the bifurcation point and if successful provides the information to create valid evidence. + err := c.handleConflictingHeaders(ctx, primaryTrace, e.Block, e.WitnessIndex, now) if err != nil { - c.logger.Info("Error validating witness's divergent header", "witness", supportingWitness, "err", err) - witnessesToRemove = append(witnessesToRemove, e.WitnessIndex) - continue + // return information of the attack + return err } - - // We are suspecting that the primary is faulty, hence we hold the witness as the source of truth - // and generate evidence against the primary that we can send to the witness - primaryEv := newLightClientAttackEvidence(primaryBlock, witnessTrace[len(witnessTrace)-1], witnessTrace[0]) - c.logger.Error("Attempted attack detected. Sending evidence againt primary by witness", "ev", primaryEv, - "primary", c.primary, "witness", supportingWitness) - c.sendEvidence(ctx, primaryEv, supportingWitness) - - // This may not be valid because the witness itself is at fault. So now we reverse it, examining the - // trace provided by the witness and holding the primary as the source of truth. Note: primary may not - // respond but this is okay as we will halt anyway. - primaryTrace, witnessBlock, err := c.examineConflictingHeaderAgainstTrace( - ctx, - witnessTrace, - primaryBlock.SignedHeader, - c.primary, - now, - ) - if err != nil { - c.logger.Info("Error validating primary's divergent header", "primary", c.primary, "err", err) - continue - } - - // We now use the primary trace to create evidence against the witness and send it to the primary - witnessEv := newLightClientAttackEvidence(witnessBlock, primaryTrace[len(primaryTrace)-1], primaryTrace[0]) - c.logger.Error("Sending evidence against witness by primary", "ev", witnessEv, - "primary", c.primary, "witness", supportingWitness) - c.sendEvidence(ctx, witnessEv, c.primary) - // We return the error and don't process anymore witnesses - return e + // if attempt to generate conflicting headers failed then remove witness + witnessesToRemove = append(witnessesToRemove, e.WitnessIndex) case errBadWitness: c.logger.Info("Witness returned an error during header comparison", "witness", c.witnesses[e.WitnessIndex], @@ -122,11 +85,9 @@ func (c *Client) detectDivergence(ctx context.Context, primaryTrace []*types.Lig } } - // we need to make sure that we remove witnesses by index in the reverse - // order so as to not affect the indexes themselves - sort.Ints(witnessesToRemove) - for i := len(witnessesToRemove) - 1; i >= 0; i-- { - c.removeWitness(witnessesToRemove[i]) + // remove witnesses that have misbehaved + if err := c.removeWitnesses(witnessesToRemove); err != nil { + return err } // 1. If we had at least one witness that returned the same header then we @@ -135,7 +96,7 @@ func (c *Client) detectDivergence(ctx context.Context, primaryTrace []*types.Lig return nil } - // 2. ELse all witnesses have either not responded, don't have the block or sent invalid blocks. + // 2. Else all witnesses have either not responded, don't have the block or sent invalid blocks. return ErrFailedHeaderCrossReferencing } @@ -150,7 +111,77 @@ func (c *Client) compareNewHeaderWithWitness(ctx context.Context, errc chan erro witness provider.Provider, witnessIndex int) { lightBlock, err := witness.LightBlock(ctx, h.Height) - if err != nil { + switch err { + // no error means we move on to checking the hash of the two headers + case nil: + break + + // the witness hasn't been helpful in comparing headers, we mark the response and continue + // comparing with the rest of the witnesses + case provider.ErrNoResponse, provider.ErrLightBlockNotFound: + errc <- err + return + + // the witness' head of the blockchain is lower than the height of the primary. This could be one of + // two things: + // 1) The witness is lagging behind + // 2) The primary may be performing a lunatic attack with a height and time in the future + case provider.ErrHeightTooHigh: + // The light client now asks for the latest header that the witness has + var isTargetHeight bool + isTargetHeight, lightBlock, err = c.getTargetBlockOrLatest(ctx, h.Height, witness) + if err != nil { + errc <- err + return + } + + // if the witness caught up and has returned a block of the target height then we can + // break from this switch case and continue to verify the hashes + if isTargetHeight { + break + } + + // witness' last header is below the primary's header. We check the times to see if the blocks + // have conflicting times + if !lightBlock.Time.Before(h.Time) { + errc <- errConflictingHeaders{Block: lightBlock, WitnessIndex: witnessIndex} + return + } + + // the witness is behind. We wait for a period WAITING = 2 * DRIFT + LAG. + // This should give the witness ample time if it is a participating member + // of consensus to produce a block that has a time that is after the primary's + // block time. If not the witness is too far behind and the light client removes it + time.Sleep(2*c.maxClockDrift + c.maxBlockLag) + isTargetHeight, lightBlock, err = c.getTargetBlockOrLatest(ctx, h.Height, witness) + if err != nil { + errc <- errBadWitness{Reason: err, WitnessIndex: witnessIndex} + return + } + if isTargetHeight { + break + } + + // the witness still doesn't have a block at the height of the primary. + // Check if there is a conflicting time + if !lightBlock.Time.Before(h.Time) { + errc <- errConflictingHeaders{Block: lightBlock, WitnessIndex: witnessIndex} + return + } + + // Following this request response procedure, the witness has been unable to produce a block + // that can somehow conflict with the primary's block. We thus conclude that the witness + // is too far behind and thus we return a no response error. + // + // NOTE: If the clock drift / lag has been miscalibrated it is feasible that the light client has + // drifted too far ahead for any witness to be able provide a comparable block and thus may allow + // for a malicious primary to attack it + errc <- provider.ErrNoResponse + return + + default: + // all other errors (i.e. invalid block, closed connection or unreliable provider) we mark the + // witness as bad and remove it errc <- errBadWitness{Reason: err, WitnessIndex: witnessIndex} return } @@ -171,6 +202,67 @@ func (c *Client) sendEvidence(ctx context.Context, ev *types.LightClientAttackEv } } +// handleConflictingHeaders handles the primary style of attack, which is where a primary and witness have +// two headers of the same height but with different hashes +func (c *Client) handleConflictingHeaders( + ctx context.Context, + primaryTrace []*types.LightBlock, + challendingBlock *types.LightBlock, + witnessIndex int, + now time.Time, +) error { + supportingWitness := c.witnesses[witnessIndex] + witnessTrace, primaryBlock, err := c.examineConflictingHeaderAgainstTrace( + ctx, + primaryTrace, + challendingBlock, + supportingWitness, + now, + ) + if err != nil { + c.logger.Info("error validating witness's divergent header", "witness", supportingWitness, "err", err) + return nil + } + + // We are suspecting that the primary is faulty, hence we hold the witness as the source of truth + // and generate evidence against the primary that we can send to the witness + commonBlock, trustedBlock := witnessTrace[0], witnessTrace[len(witnessTrace)-1] + evidenceAgainstPrimary := newLightClientAttackEvidence(primaryBlock, trustedBlock, commonBlock) + c.logger.Error("ATTEMPTED ATTACK DETECTED. Sending evidence againt primary by witness", "ev", evidenceAgainstPrimary, + "primary", c.primary, "witness", supportingWitness) + c.sendEvidence(ctx, evidenceAgainstPrimary, supportingWitness) + + if primaryBlock.Commit.Round != witnessTrace[len(witnessTrace)-1].Commit.Round { + c.logger.Info("The light client has detected, and prevented, an attempted amnesia attack." + + " We think this attack is pretty unlikely, so if you see it, that's interesting to us." + + " Can you let us know by opening an issue through https://github.com/tendermint/tendermint/issues/new?") + } + + // This may not be valid because the witness itself is at fault. So now we reverse it, examining the + // trace provided by the witness and holding the primary as the source of truth. Note: primary may not + // respond but this is okay as we will halt anyway. + primaryTrace, witnessBlock, err := c.examineConflictingHeaderAgainstTrace( + ctx, + witnessTrace, + primaryBlock, + c.primary, + now, + ) + if err != nil { + c.logger.Info("Error validating primary's divergent header", "primary", c.primary, "err", err) + return ErrLightClientAttack + } + + // We now use the primary trace to create evidence against the witness and send it to the primary + commonBlock, trustedBlock = primaryTrace[0], primaryTrace[len(primaryTrace)-1] + evidenceAgainstWitness := newLightClientAttackEvidence(witnessBlock, trustedBlock, commonBlock) + c.logger.Error("Sending evidence against witness by primary", "ev", evidenceAgainstWitness, + "primary", c.primary, "witness", supportingWitness) + c.sendEvidence(ctx, evidenceAgainstWitness, c.primary) + // We return the error and don't process anymore witnesses + return ErrLightClientAttack +} + // examineConflictingHeaderAgainstTrace takes a trace from one provider and a divergent header that // it has received from another and preforms verifySkipping at the heights of each of the intermediate // headers in the trace until it reaches the divergentHeader. 1 of 2 things can happen. @@ -179,22 +271,66 @@ func (c *Client) sendEvidence(ctx context.Context, ev *types.LightClientAttackEv // is the bifurcation point and the light client can create evidence from it // 2. The source stops responding, doesn't have the block or sends an invalid header in which case we // return the error and remove the witness +// +// CONTRACT: +// 1. Trace can not be empty len(trace) > 0 +// 2. The last block in the trace can not be of a lower height than the target block +// trace[len(trace)-1].Height >= targetBlock.Height +// 3. The func (c *Client) examineConflictingHeaderAgainstTrace( ctx context.Context, trace []*types.LightBlock, - divergentHeader *types.SignedHeader, - source provider.Provider, now time.Time) ([]*types.LightBlock, *types.LightBlock, error) { + targetBlock *types.LightBlock, + source provider.Provider, now time.Time, +) ([]*types.LightBlock, *types.LightBlock, error) { - var previouslyVerifiedBlock *types.LightBlock + var ( + previouslyVerifiedBlock, sourceBlock *types.LightBlock + sourceTrace []*types.LightBlock + err error + ) + + if targetBlock.Height < trace[0].Height { + return nil, nil, fmt.Errorf("target block has a height lower than the trusted height (%d < %d)", + targetBlock.Height, trace[0].Height) + } for idx, traceBlock := range trace { - // The first block in the trace MUST be the same to the light block that the source produces - // else we cannot continue with verification. - sourceBlock, err := source.LightBlock(ctx, traceBlock.Height) - if err != nil { - return nil, nil, err + // this case only happens in a forward lunatic attack. We treat the block with the + // height directly after the targetBlock as the divergent block + if traceBlock.Height > targetBlock.Height { + // sanity check that the time of the traceBlock is indeed less than that of the targetBlock. If the trace + // was correctly verified we should expect monotonically increasing time. This means that if the block at + // the end of the trace has a lesser time than the target block then all blocks in the trace should have a + // lesser time + if traceBlock.Time.After(targetBlock.Time) { + return nil, nil, + errors.New("sanity check failed: expected traceblock to have a lesser time than the target block") + } + + // before sending back the divergent block and trace we need to ensure we have verified + // the final gap between the previouslyVerifiedBlock and the targetBlock + if previouslyVerifiedBlock.Height != targetBlock.Height { + sourceTrace, err = c.verifySkipping(ctx, source, previouslyVerifiedBlock, targetBlock, now) + if err != nil { + return nil, nil, fmt.Errorf("verifySkipping of conflicting header failed: %w", err) + } + } + return sourceTrace, traceBlock, nil } + // get the corresponding block from the source to verify and match up against the traceBlock + if traceBlock.Height == targetBlock.Height { + sourceBlock = targetBlock + } else { + sourceBlock, err = source.LightBlock(ctx, traceBlock.Height) + if err != nil { + return nil, nil, fmt.Errorf("failed to examine trace: %w", err) + } + } + + // The first block in the trace MUST be the same to the light block that the source produces + // else we cannot continue with verification. if idx == 0 { if shash, thash := sourceBlock.Hash(), traceBlock.Hash(); !bytes.Equal(shash, thash) { return nil, nil, fmt.Errorf("trusted block is different to the source's first block (%X = %X)", @@ -206,25 +342,55 @@ func (c *Client) examineConflictingHeaderAgainstTrace( // we check that the source provider can verify a block at the same height of the // intermediate height - trace, err := c.verifySkipping(ctx, source, previouslyVerifiedBlock, sourceBlock, now) + sourceTrace, err = c.verifySkipping(ctx, source, previouslyVerifiedBlock, sourceBlock, now) if err != nil { return nil, nil, fmt.Errorf("verifySkipping of conflicting header failed: %w", err) } // check if the headers verified by the source has diverged from the trace if shash, thash := sourceBlock.Hash(), traceBlock.Hash(); !bytes.Equal(shash, thash) { // Bifurcation point found! - return trace, traceBlock, nil + return sourceTrace, traceBlock, nil } // headers are still the same. update the previouslyVerifiedBlock previouslyVerifiedBlock = sourceBlock } - // We have reached the end of the trace without observing a divergence. The last header is thus different - // from the divergent header that the source originally sent us, then we return an error. - return nil, nil, fmt.Errorf("source provided different header to the original header it provided (%X != %X)", - previouslyVerifiedBlock.Hash(), divergentHeader.Hash()) + // We have reached the end of the trace. This should never happen. This can only happen if one of the stated + // prerequisites to this function were not met. Namely that either trace[len(trace)-1].Height < targetBlock.Height + // or that trace[i].Hash() != targetBlock.Hash() + return nil, nil, errNoDivergence + +} + +// getTargetBlockOrLatest gets the latest height, if it is greater than the target height then it queries +// the target heght else it returns the latest. returns true if it successfully managed to acquire the target +// height. +func (c *Client) getTargetBlockOrLatest( + ctx context.Context, + height int64, + witness provider.Provider, +) (bool, *types.LightBlock, error) { + lightBlock, err := witness.LightBlock(ctx, 0) + if err != nil { + return false, nil, err + } + + if lightBlock.Height == height { + // the witness has caught up to the height of the provider's signed header. We + // can resume with checking the hashes. + return true, lightBlock, nil + } + + if lightBlock.Height > height { + // the witness has caught up. We recursively call the function again. However in order + // to avoud a wild goose chase where the witness sends us one header below and one header + // above the height we set a timeout to the context + lightBlock, err := witness.LightBlock(ctx, height) + return true, lightBlock, err + } + return false, lightBlock, nil } // newLightClientAttackEvidence determines the type of attack and then forms the evidence filling out diff --git a/light/detector_test.go b/light/detector_test.go index 4788759e0..caec5e8e2 100644 --- a/light/detector_test.go +++ b/light/detector_test.go @@ -63,7 +63,7 @@ func TestLightClientAttackEvidence_Lunatic(t *testing.T) { // Check verification returns an error. _, err = c.VerifyLightBlockAtHeight(ctx, 10, bTime.Add(1*time.Hour)) if assert.Error(t, err) { - assert.Contains(t, err.Error(), "does not match primary") + assert.Equal(t, light.ErrLightClientAttack, err) } // Check evidence was sent to both full nodes. @@ -146,7 +146,7 @@ func TestLightClientAttackEvidence_Equivocation(t *testing.T) { // Check verification returns an error. _, err = c.VerifyLightBlockAtHeight(ctx, 10, bTime.Add(1*time.Hour)) if assert.Error(t, err) { - assert.Contains(t, err.Error(), "does not match primary") + assert.Equal(t, light.ErrLightClientAttack, err) } // Check evidence was sent to both full nodes. @@ -172,6 +172,139 @@ func TestLightClientAttackEvidence_Equivocation(t *testing.T) { } } +func TestLightClientAttackEvidence_ForwardLunatic(t *testing.T) { + // primary performs a lunatic attack but changes the time of the header to + // something in the future relative to the blockchain + var ( + latestHeight = int64(10) + valSize = 5 + forgedHeight = int64(12) + proofHeight = int64(11) + primaryHeaders = make(map[int64]*types.SignedHeader, forgedHeight) + primaryValidators = make(map[int64]*types.ValidatorSet, forgedHeight) + ) + + witnessHeaders, witnessValidators, chainKeys := genMockNodeWithKeys(chainID, latestHeight, valSize, 2, bTime) + + // primary has the exact same headers except it forges one extra header in the future using keys from 2/5ths of + // the validators + for h := range witnessHeaders { + primaryHeaders[h] = witnessHeaders[h] + primaryValidators[h] = witnessValidators[h] + } + forgedKeys := chainKeys[latestHeight].ChangeKeys(3) // we change 3 out of the 5 validators (still 2/5 remain) + primaryValidators[forgedHeight] = forgedKeys.ToValidators(2, 0) + primaryHeaders[forgedHeight] = forgedKeys.GenSignedHeader( + chainID, + forgedHeight, + bTime.Add(time.Duration(latestHeight+1)*time.Minute), // 11 mins + nil, + primaryValidators[forgedHeight], + primaryValidators[forgedHeight], + hash("app_hash"), + hash("cons_hash"), + hash("results_hash"), + 0, len(forgedKeys), + ) + + witness := mockp.New(chainID, witnessHeaders, witnessValidators) + primary := mockp.New(chainID, primaryHeaders, primaryValidators) + + laggingWitness := witness.Copy(chainID) + + // In order to perform the attack, the primary needs at least one accomplice as a witness to also + // send the forged block + accomplice := primary + + c, err := light.NewClient( + ctx, + chainID, + light.TrustOptions{ + Period: 4 * time.Hour, + Height: 1, + Hash: primaryHeaders[1].Hash(), + }, + primary, + []provider.Provider{witness, accomplice}, + dbs.New(dbm.NewMemDB(), chainID), + light.Logger(log.TestingLogger()), + light.MaxClockDrift(1*time.Second), + light.MaxBlockLag(1*time.Second), + ) + require.NoError(t, err) + + // two seconds later, the supporting withness should receive the header that can be used + // to prove that there was an attack + vals := chainKeys[latestHeight].ToValidators(2, 0) + newLb := &types.LightBlock{ + SignedHeader: chainKeys[latestHeight].GenSignedHeader( + chainID, + proofHeight, + bTime.Add(time.Duration(proofHeight+1)*time.Minute), // 12 mins + nil, + vals, + vals, + hash("app_hash"), + hash("cons_hash"), + hash("results_hash"), + 0, len(chainKeys), + ), + ValidatorSet: vals, + } + go func() { + time.Sleep(2 * time.Second) + witness.AddLightBlock(newLb) + }() + + // Now assert that verification returns an error. We craft the light clients time to be a little ahead of the chain + // to allow a window for the attack to manifest itself. + _, err = c.Update(ctx, bTime.Add(time.Duration(forgedHeight)*time.Minute)) + if assert.Error(t, err) { + assert.Equal(t, light.ErrLightClientAttack, err) + } + + // Check evidence was sent to the witness against the full node + evAgainstPrimary := &types.LightClientAttackEvidence{ + ConflictingBlock: &types.LightBlock{ + SignedHeader: primaryHeaders[forgedHeight], + ValidatorSet: primaryValidators[forgedHeight], + }, + CommonHeight: latestHeight, + } + assert.True(t, witness.HasEvidence(evAgainstPrimary)) + + // We attempt the same call but now the supporting witness has a block which should + // immediately conflict in time with the primary + _, err = c.VerifyLightBlockAtHeight(ctx, forgedHeight, bTime.Add(time.Duration(forgedHeight)*time.Minute)) + if assert.Error(t, err) { + assert.Equal(t, light.ErrLightClientAttack, err) + } + assert.True(t, witness.HasEvidence(evAgainstPrimary)) + + // Lastly we test the unfortunate case where the light clients supporting witness doesn't update + // in enough time + c, err = light.NewClient( + ctx, + chainID, + light.TrustOptions{ + Period: 4 * time.Hour, + Height: 1, + Hash: primaryHeaders[1].Hash(), + }, + primary, + []provider.Provider{laggingWitness, accomplice}, + dbs.New(dbm.NewMemDB(), chainID), + light.Logger(log.TestingLogger()), + light.MaxClockDrift(1*time.Second), + light.MaxBlockLag(1*time.Second), + ) + require.NoError(t, err) + + _, err = c.Update(ctx, bTime.Add(time.Duration(forgedHeight)*time.Minute)) + assert.NoError(t, err) + +} + // 1. Different nodes therefore a divergent header is produced. // => light client returns an error upon creation because primary and witness // have a different view. @@ -258,5 +391,41 @@ func TestClientDivergentTraces3(t *testing.T) { _, err = c.VerifyLightBlockAtHeight(ctx, 10, bTime.Add(1*time.Hour)) assert.Error(t, err) - assert.Equal(t, 0, len(c.Witnesses())) + assert.Equal(t, 1, len(c.Witnesses())) +} + +// 4. Witness has a divergent header but can not produce a valid trace to back it up. +// It should be ignored +func TestClientDivergentTraces4(t *testing.T) { + _, primaryHeaders, primaryVals := genMockNode(chainID, 10, 5, 2, bTime) + primary := mockp.New(chainID, primaryHeaders, primaryVals) + + firstBlock, err := primary.LightBlock(ctx, 1) + require.NoError(t, err) + + _, mockHeaders, mockVals := genMockNode(chainID, 10, 5, 2, bTime) + witness := primary.Copy(chainID) + witness.AddLightBlock(&types.LightBlock{ + SignedHeader: mockHeaders[10], + ValidatorSet: mockVals[10], + }) + + c, err := light.NewClient( + ctx, + chainID, + light.TrustOptions{ + Height: 1, + Hash: firstBlock.Hash(), + Period: 4 * time.Hour, + }, + primary, + []provider.Provider{witness}, + dbs.New(dbm.NewMemDB(), chainID), + light.Logger(log.TestingLogger()), + ) + require.NoError(t, err) + + _, err = c.VerifyLightBlockAtHeight(ctx, 10, bTime.Add(1*time.Hour)) + assert.Error(t, err) + assert.Equal(t, 1, len(c.Witnesses())) } diff --git a/light/errors.go b/light/errors.go index 390d35c58..aba9e63e1 100644 --- a/light/errors.go +++ b/light/errors.go @@ -63,6 +63,18 @@ func (e ErrVerificationFailed) Error() string { return fmt.Sprintf("verify from #%d to #%d failed: %v", e.From, e.To, e.Reason) } +// ErrLightClientAttack is returned when the light client has detected an attempt +// to verify a false header and has sent the evidence to either a witness or primary. +var ErrLightClientAttack = errors.New(`attempted attack detected. + Light client received valid conflicting header from witness. + Unable to verify header. Evidence has been sent to both providers. + Check logs for full evidence and trace`, +) + +// ErrNoWitnesses means that there are not enough witnesses connected to +// continue running the light client. +var ErrNoWitnesses = errors.New("no witnesses connected. please reset light client") + // ----------------------------- INTERNAL ERRORS --------------------------------- // ErrConflictingHeaders is thrown when two conflicting headers are discovered. @@ -95,3 +107,7 @@ type errBadWitness struct { func (e errBadWitness) Error() string { return fmt.Sprintf("Witness %d returned error: %s", e.WitnessIndex, e.Reason.Error()) } + +var errNoDivergence = errors.New( + "sanity check failed: no divergence between the original trace and the provider's new trace", +) diff --git a/light/provider/errors.go b/light/provider/errors.go index 5d24efd73..398647b3e 100644 --- a/light/provider/errors.go +++ b/light/provider/errors.go @@ -6,8 +6,12 @@ import ( ) var ( + // ErrHeightTooHigh is returned when the height is higher than the last + // block that the provider has. The light client will not remove the provider + ErrHeightTooHigh = errors.New("height requested is too high") // ErrLightBlockNotFound is returned when a provider can't find the - // requested header. + // requested header (i.e. it has been pruned). + // The light client will not remove the provider ErrLightBlockNotFound = errors.New("light block not found") // ErrNoResponse is returned if the provider doesn't respond to the // request in a gieven time diff --git a/light/provider/http/http.go b/light/provider/http/http.go index c76e7abbb..e0abbb41f 100644 --- a/light/provider/http/http.go +++ b/light/provider/http/http.go @@ -16,7 +16,8 @@ import ( var ( // This is very brittle, see: https://github.com/tendermint/tendermint/issues/4740 - regexpMissingHeight = regexp.MustCompile(`height \d+ (must be less than or equal to|is not available)`) + regexpMissingHeight = regexp.MustCompile(`height \d+ is not available`) + regexpTooHigh = regexp.MustCompile(`height \d+ must be less than or equal to`) maxRetryAttempts = 10 timeout uint = 5 // sec. @@ -75,6 +76,12 @@ func (p *http) LightBlock(ctx context.Context, height int64) (*types.LightBlock, return nil, err } + if height != 0 && sh.Height != height { + return nil, provider.ErrBadLightBlock{ + Reason: fmt.Errorf("height %d responded doesn't match height %d requested", sh.Height, height), + } + } + vs, err := p.validatorSet(ctx, &sh.Height) if err != nil { return nil, err @@ -117,6 +124,10 @@ func (p *http) validatorSet(ctx context.Context, height *int64) (*types.Validato res, err := p.client.Validators(ctx, height, &page, &perPage) if err != nil { // TODO: standardize errors on the RPC side + if regexpTooHigh.MatchString(err.Error()) { + return nil, provider.ErrHeightTooHigh + } + if regexpMissingHeight.MatchString(err.Error()) { return nil, provider.ErrLightBlockNotFound } @@ -162,6 +173,10 @@ func (p *http) signedHeader(ctx context.Context, height *int64) (*types.SignedHe commit, err := p.client.Commit(ctx, height) if err != nil { // TODO: standardize errors on the RPC side + if regexpTooHigh.MatchString(err.Error()) { + return nil, provider.ErrHeightTooHigh + } + if regexpMissingHeight.MatchString(err.Error()) { return nil, provider.ErrLightBlockNotFound } diff --git a/light/provider/http/http_test.go b/light/provider/http/http_test.go index b6b3989a8..1a0beb339 100644 --- a/light/provider/http/http_test.go +++ b/light/provider/http/http_test.go @@ -80,9 +80,10 @@ func TestProvider(t *testing.T) { assert.Equal(t, lower, sh.Height) // fetching missing heights (both future and pruned) should return appropriate errors - _, err = p.LightBlock(context.Background(), 1000) + lb, err := p.LightBlock(context.Background(), 1000) require.Error(t, err) - assert.Equal(t, provider.ErrLightBlockNotFound, err) + require.Nil(t, lb) + assert.Equal(t, provider.ErrHeightTooHigh, err) _, err = p.LightBlock(context.Background(), 1) require.Error(t, err) diff --git a/light/provider/mock/mock.go b/light/provider/mock/mock.go index cf28846ef..091f0abf8 100644 --- a/light/provider/mock/mock.go +++ b/light/provider/mock/mock.go @@ -5,16 +5,20 @@ import ( "errors" "fmt" "strings" + "sync" "github.com/tendermint/tendermint/light/provider" "github.com/tendermint/tendermint/types" ) type Mock struct { - chainID string + chainID string + + mtx sync.Mutex headers map[int64]*types.SignedHeader vals map[int64]*types.ValidatorSet evidenceToReport map[string]types.Evidence // hash => evidence + latestHeight int64 } var _ provider.Provider = (*Mock)(nil) @@ -22,11 +26,18 @@ var _ provider.Provider = (*Mock)(nil) // New creates a mock provider with the given set of headers and validator // sets. func New(chainID string, headers map[int64]*types.SignedHeader, vals map[int64]*types.ValidatorSet) *Mock { + height := int64(0) + for h := range headers { + if h > height { + height = h + } + } return &Mock{ chainID: chainID, headers: headers, vals: vals, evidenceToReport: make(map[string]types.Evidence), + latestHeight: height, } } @@ -50,16 +61,18 @@ func (p *Mock) String() string { } func (p *Mock) LightBlock(_ context.Context, height int64) (*types.LightBlock, error) { + p.mtx.Lock() + defer p.mtx.Unlock() var lb *types.LightBlock - if height == 0 && len(p.headers) > 0 { - sh := p.headers[int64(len(p.headers))] - vals := p.vals[int64(len(p.vals))] - lb = &types.LightBlock{ - SignedHeader: sh, - ValidatorSet: vals, - } + if height > p.latestHeight { + return nil, provider.ErrHeightTooHigh + } + + if height == 0 && len(p.headers) > 0 { + height = p.latestHeight } + if _, ok := p.headers[height]; ok { sh := p.headers[height] vals := p.vals[height] @@ -89,3 +102,21 @@ func (p *Mock) HasEvidence(ev types.Evidence) bool { _, ok := p.evidenceToReport[string(ev.Hash())] return ok } + +func (p *Mock) AddLightBlock(lb *types.LightBlock) { + p.mtx.Lock() + defer p.mtx.Unlock() + + if err := lb.ValidateBasic(lb.ChainID); err != nil { + panic(fmt.Sprintf("unable to add light block, err: %v", err)) + } + p.headers[lb.Height] = lb.SignedHeader + p.vals[lb.Height] = lb.ValidatorSet + if lb.Height > p.latestHeight { + p.latestHeight = lb.Height + } +} + +func (p *Mock) Copy(id string) *Mock { + return New(id, p.headers, p.vals) +} diff --git a/rpc/core/env.go b/rpc/core/env.go index 41e003a36..c24c3d262 100644 --- a/rpc/core/env.go +++ b/rpc/core/env.go @@ -152,7 +152,7 @@ func getHeight(latestHeight int64, heightPtr *int64) (int64, error) { } base := env.BlockStore.Base() if height < base { - return 0, fmt.Errorf("height %v is not available, lowest height is %v", + return 0, fmt.Errorf("height %d is not available, lowest height is %d", height, base) } return height, nil