diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 92af3dc30..8bcfb307b 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -89,3 +89,4 @@ Friendly reminder: We have a [bug bounty program](https://hackerone.com/tendermi - [blockchain/v1] [\#5701](https://github.com/tendermint/tendermint/pull/5701) Handle peers without blocks (@melekes) - [blockchain/v1] \#5711 Fix deadlock (@melekes) - [rpc/jsonrpc/server] \#6191 Correctly unmarshal `RPCRequest` when data is `null` (@melekes) +- [light/evidence] Adds logic to handle forward lunatic attacks (@cmwaters) diff --git a/docs/architecture/adr-047-handling-evidence-from-light-client.md b/docs/architecture/adr-047-handling-evidence-from-light-client.md index 4de47819a..417d59fb0 100644 --- a/docs/architecture/adr-047-handling-evidence-from-light-client.md +++ b/docs/architecture/adr-047-handling-evidence-from-light-client.md @@ -8,6 +8,7 @@ * 14-08-2020: Introduce light traces (listed now as an alternative approach) * 20-08-2020: Light client produces evidence when detected instead of passing to full node * 16-09-2020: Post-implementation revision +* 15-03-2020: Ammends for the case of a forward lunatic attack ### Glossary of Terms @@ -106,8 +107,10 @@ This is done with: ```golang func (c *Client) examineConflictingHeaderAgainstTrace( trace []*types.LightBlock, - divergentHeader *types.SignedHeader, - source provider.Provider, now time.Time) ([]*types.LightBlock, *types.LightBlock, error) + targetBlock *types.LightBlock, + source provider.Provider, + now time.Time, + ) ([]*types.LightBlock, *types.LightBlock, error) ``` which performs the following @@ -117,16 +120,21 @@ because witnesses cannot be added and removed after the client is initialized. B as a sanity check. If this fails we have to drop the witness. 2. Querying and verifying the witness's headers using bisection at the same heights of all the -intermediary headers of the primary (In the above example this is A, B, C, D, F, H). If bisection fails or the witness stops responding then -we can call the witness faulty and drop it. +intermediary headers of the primary (In the above example this is A, B, C, D, F, H). If bisection fails +or the witness stops responding then we can call the witness faulty and drop it. -3. We eventually reach a verified header by the witness which is not the same as the intermediary header (In the above example this is E). -This is the point of bifurcation (This could also be the last header). +3. We eventually reach a verified header by the witness which is not the same as the intermediary header +(In the above example this is E). This is the point of bifurcation (This could also be the last header). + +4. There is a unique case where the trace that is being examined against has blocks that have a greater +height than the targetBlock. This can occur as part of a forward lunatic attack where the primary has +provided a light block that has a height greater than the head of the chain (see Appendix B). In this +case, the light client will verify the sources blocks up to the targetBlock and return the block in the +trace that is directly after the targetBlock in height as the `ConflictingBlock` This function then returns the trace of blocks from the witness node between the common header and the -divergent header of the primary as it -is likely as seen in the example to the right below that multiple headers where required in order to -verify the divergent one. This trace will +divergent header of the primary as it is likely, as seen in the example to the right, that multiple +headers where required in order to verify the divergent one. This trace will be used later (as is also described later in this document). ![](../imgs/bifurcation-point.png) @@ -225,3 +233,22 @@ would be validators that currently still have something staked. Not only this but there was a large degree of extra computation required in storing all the currently staked validators that could possibly fall into the group of being a phantom validator. Given this, it was removed. + +## Appendix B + +A unique flavor of lunatic attack is a forward lunatic attack. This is where a malicious +node provides a header with a height greater than the height of the blockchain. Thus there +are no witnesses capable of rebutting the malicious header. Such an attack will also +require an accomplice, i.e. at least one other witness to also return the same forged header. +Although such attacks can be any arbitrary height ahead, they must still remain within the +clock drift of the light clients real time. Therefore, to detect such an attack, a light +client will wait for a time + +``` +2 * MAX_CLOCK_DRIFT + LAG +``` + +for a witness to provide the latest block it has. Given the time constraints, if the witness +is operating at the head of the blockchain, it will have a header with an earlier height but +a later timestamp. This can be used to prove that the primary has submitted a lunatic header +which violates monotonically increasing time. diff --git a/docs/architecture/adr-059-evidence-composition-and-lifecycle.md b/docs/architecture/adr-059-evidence-composition-and-lifecycle.md index 707a18bfb..5b86164b2 100644 --- a/docs/architecture/adr-059-evidence-composition-and-lifecycle.md +++ b/docs/architecture/adr-059-evidence-composition-and-lifecycle.md @@ -4,6 +4,7 @@ - 04/09/2020: Initial Draft (Unabridged) - 07/09/2020: First Version +- 13.03.21: Ammendment to accomodate forward lunatic attack ## Scope @@ -159,7 +160,7 @@ For `LightClientAttack` - Fetch the common signed header and val set from the common height and use skipping verification to verify the conflicting header -- Fetch the trusted signed header at the same height as the conflicting header and compare with the conflicting header to work out which type of attack it is and in doing so return the malicious validators. +- Fetch the trusted signed header at the same height as the conflicting header and compare with the conflicting header to work out which type of attack it is and in doing so return the malicious validators. NOTE: If the node doesn't have the signed header at the height of the conflicting header, it instead fetches the latest header it has and checks to see if it can prove the evidence based on a violation of header time. This is known as forward lunatic attack. - If equivocation, return the validators that signed for the commits of both the trusted and signed header @@ -167,7 +168,11 @@ For `LightClientAttack` - If amnesia, return no validators (since we can't know which validators are malicious). This also means that we don't currently send amnesia evidence to the application, although we will introduce more robust amnesia evidence handling in future Tendermint Core releases -- For each validator, check the look up table to make sure there already isn't evidence against this validator +- Check that the hashes of the conflicting header and the trusted header are different + +- In the case of a forward lunatic attack, where the trusted header height is less than the conflicting header height, the node checks that the time of the trusted header is later than the time of conflicting header. This proves that the conflicting header breaks monotonically increasing time. If the node doesn't have a trusted header with a later time then it is unable to validate the evidence for now. + +- Lastly, for each validator, check the look up table to make sure there already isn't evidence against this validator After verification we persist the evidence with the key `height/hash` to the pending evidence database in the evidence pool with the following format: diff --git a/evidence/mocks/block_store.go b/evidence/mocks/block_store.go index cdf316d00..e6205939a 100644 --- a/evidence/mocks/block_store.go +++ b/evidence/mocks/block_store.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.5.1. DO NOT EDIT. +// Code generated by mockery v0.0.0-dev. DO NOT EDIT. package mocks @@ -12,6 +12,20 @@ type BlockStore struct { mock.Mock } +// Height provides a mock function with given fields: +func (_m *BlockStore) Height() int64 { + ret := _m.Called() + + var r0 int64 + if rf, ok := ret.Get(0).(func() int64); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(int64) + } + + return r0 +} + // LoadBlockCommit provides a mock function with given fields: height func (_m *BlockStore) LoadBlockCommit(height int64) *types.Commit { ret := _m.Called(height) diff --git a/evidence/services.go b/evidence/services.go index 5e15b38b4..274433cbe 100644 --- a/evidence/services.go +++ b/evidence/services.go @@ -9,4 +9,5 @@ import ( type BlockStore interface { LoadBlockMeta(height int64) *types.BlockMeta LoadBlockCommit(height int64) *types.Commit + Height() int64 } diff --git a/evidence/verify.go b/evidence/verify.go index dabd248c8..406f6b8da 100644 --- a/evidence/verify.go +++ b/evidence/verify.go @@ -100,7 +100,21 @@ func (evpool *Pool) verify(evidence types.Evidence) error { if evidence.Height() != ev.ConflictingBlock.Height { trustedHeader, err = getSignedHeader(evpool.blockStore, ev.ConflictingBlock.Height) if err != nil { - return err + // FIXME: This multi step process is a bit unergonomic. We may want to consider a more efficient process + // that doesn't require as much io and is atomic. + + // If the node doesn't have a block at the height of the conflicting block, then this could be + // a forward lunatic attack. Thus the node must get the latest height it has + latestHeight := evpool.blockStore.Height() + trustedHeader, err = getSignedHeader(evpool.blockStore, latestHeight) + if err != nil { + return err + } + if trustedHeader.Time.Before(ev.ConflictingBlock.Time) { + return fmt.Errorf("latest block time (%v) is before conflicting block time (%v)", + trustedHeader.Time, ev.ConflictingBlock.Time, + ) + } } } @@ -176,36 +190,47 @@ func (evpool *Pool) verify(evidence types.Evidence) error { // the following checks: // - the common header from the full node has at least 1/3 voting power which is also present in // the conflicting header's commit +// - 2/3+ of the conflicting validator set correctly signed the conflicting block // - the nodes trusted header at the same height as the conflicting header has a different hash +// +// CONTRACT: must run ValidateBasic() on the evidence before verifying +// must check that the evidence has not expired (i.e. is outside the maximum age threshold) func VerifyLightClientAttack(e *types.LightClientAttackEvidence, commonHeader, trustedHeader *types.SignedHeader, commonVals *types.ValidatorSet, now time.Time, trustPeriod time.Duration) error { - // In the case of lunatic attack we need to perform a single verification jump between the - // common header and the conflicting one - if commonHeader.Height != trustedHeader.Height { - err := light.Verify(commonHeader, commonVals, e.ConflictingBlock.SignedHeader, e.ConflictingBlock.ValidatorSet, - trustPeriod, now, 0*time.Second, light.DefaultTrustLevel) + // In the case of lunatic attack there will be a different commonHeader height. Therefore the node perform a single + // verification jump between the common header and the conflicting one + if commonHeader.Height != e.ConflictingBlock.Height { + err := commonVals.VerifyCommitLightTrusting(trustedHeader.ChainID, e.ConflictingBlock.Commit, light.DefaultTrustLevel) if err != nil { - return fmt.Errorf("skipping verification from common to conflicting header failed: %w", err) - } - } else { - // in the case of equivocation and amnesia we expect some header hashes to be correctly derived - if isInvalidHeader(trustedHeader.Header, e.ConflictingBlock.Header) { - return errors.New("common height is the same as conflicting block height so expected the conflicting" + - " block to be correctly derived yet it wasn't") - } - // ensure that 2/3 of the validator set did vote for this block - if err := e.ConflictingBlock.ValidatorSet.VerifyCommitLight(trustedHeader.ChainID, e.ConflictingBlock.Commit.BlockID, - e.ConflictingBlock.Height, e.ConflictingBlock.Commit); err != nil { - return fmt.Errorf("invalid commit from conflicting block: %w", err) + return fmt.Errorf("skipping verification of conflicting block failed: %w", err) } + + // In the case of equivocation and amnesia we expect all header hashes to be correctly derived + } else if isInvalidHeader(trustedHeader.Header, e.ConflictingBlock.Header) { + return errors.New("common height is the same as conflicting block height so expected the conflicting" + + " block to be correctly derived yet it wasn't") } + // Verify that the 2/3+ commits from the conflicting validator set were for the conflicting header + if err := e.ConflictingBlock.ValidatorSet.VerifyCommitLight(trustedHeader.ChainID, e.ConflictingBlock.Commit.BlockID, + e.ConflictingBlock.Height, e.ConflictingBlock.Commit); err != nil { + return fmt.Errorf("invalid commit from conflicting block: %w", err) + } + + // Assert the correct amount of voting power of the validator set if evTotal, valsTotal := e.TotalVotingPower, commonVals.TotalVotingPower(); evTotal != valsTotal { return fmt.Errorf("total voting power from the evidence and our validator set does not match (%d != %d)", evTotal, valsTotal) } - if bytes.Equal(trustedHeader.Hash(), e.ConflictingBlock.Hash()) { + // check in the case of a forward lunatic attack that monotonically increasing time has been violated + if e.ConflictingBlock.Height > trustedHeader.Height && e.ConflictingBlock.Time.After(trustedHeader.Time) { + return fmt.Errorf("conflicting block doesn't violate monotonically increasing time (%v is after %v)", + e.ConflictingBlock.Time, trustedHeader.Time, + ) + + // In all other cases check that the hashes of the conflicting header and the trusted header are different + } else if bytes.Equal(trustedHeader.Hash(), e.ConflictingBlock.Hash()) { return fmt.Errorf("trusted header hash matches the evidence's conflicting header hash: %X", trustedHeader.Hash()) } diff --git a/evidence/verify_test.go b/evidence/verify_test.go index 30e5e4e71..dfb8cec18 100644 --- a/evidence/verify_test.go +++ b/evidence/verify_test.go @@ -34,6 +34,7 @@ func TestVerifyLightClientAttack_Lunatic(t *testing.T) { commonHeader := makeHeaderRandom(4) commonHeader.Time = defaultEvidenceTime trustedHeader := makeHeaderRandom(10) + trustedHeader.Time = defaultEvidenceTime.Add(1 * time.Hour) conflictingHeader := makeHeaderRandom(10) conflictingHeader.Time = defaultEvidenceTime.Add(1 * time.Hour) @@ -89,6 +90,30 @@ func TestVerifyLightClientAttack_Lunatic(t *testing.T) { assert.Error(t, err) ev.TotalVotingPower = 20 + forwardConflictingHeader := makeHeaderRandom(11) + forwardConflictingHeader.Time = defaultEvidenceTime.Add(30 * time.Minute) + forwardConflictingHeader.ValidatorsHash = conflictingVals.Hash() + forwardBlockID := makeBlockID(forwardConflictingHeader.Hash(), 1000, []byte("partshash")) + forwardVoteSet := types.NewVoteSet(evidenceChainID, 11, 1, tmproto.SignedMsgType(2), conflictingVals) + forwardCommit, err := types.MakeCommit(forwardBlockID, 11, 1, forwardVoteSet, conflictingPrivVals, defaultEvidenceTime) + require.NoError(t, err) + forwardLunaticEv := &types.LightClientAttackEvidence{ + ConflictingBlock: &types.LightBlock{ + SignedHeader: &types.SignedHeader{ + Header: forwardConflictingHeader, + Commit: forwardCommit, + }, + ValidatorSet: conflictingVals, + }, + CommonHeight: 4, + TotalVotingPower: 20, + ByzantineValidators: commonVals.Validators, + Timestamp: defaultEvidenceTime, + } + err = evidence.VerifyLightClientAttack(forwardLunaticEv, commonSignedHeader, trustedSignedHeader, commonVals, + defaultEvidenceTime.Add(2*time.Hour), 3*time.Hour) + assert.NoError(t, err) + state := sm.State{ LastBlockTime: defaultEvidenceTime.Add(2 * time.Hour), LastBlockHeight: 11, @@ -100,8 +125,10 @@ func TestVerifyLightClientAttack_Lunatic(t *testing.T) { blockStore := &mocks.BlockStore{} blockStore.On("LoadBlockMeta", int64(4)).Return(&types.BlockMeta{Header: *commonHeader}) blockStore.On("LoadBlockMeta", int64(10)).Return(&types.BlockMeta{Header: *trustedHeader}) + blockStore.On("LoadBlockMeta", int64(11)).Return(nil) blockStore.On("LoadBlockCommit", int64(4)).Return(commit) blockStore.On("LoadBlockCommit", int64(10)).Return(trustedCommit) + blockStore.On("Height").Return(int64(10)) pool, err := evidence.NewPool(log.TestingLogger(), dbm.NewMemDB(), stateStore, blockStore) require.NoError(t, err) @@ -125,6 +152,9 @@ func TestVerifyLightClientAttack_Lunatic(t *testing.T) { err = pool.CheckEvidence(evList) assert.Error(t, err) + evList = types.EvidenceList{forwardLunaticEv} + err = pool.CheckEvidence(evList) + assert.NoError(t, err) } func TestVerifyLightClientAttack_Equivocation(t *testing.T) { diff --git a/light/client.go b/light/client.go index b6785f768..459d406bd 100644 --- a/light/client.go +++ b/light/client.go @@ -35,6 +35,9 @@ const ( // - http://vancouver-webpages.com/time/web.html // - https://blog.codinghorror.com/keeping-time-on-the-pc/ defaultMaxClockDrift = 10 * time.Second + + // 10s is sufficient for most networks. + defaultMaxBlockLag = 10 * time.Second ) // Option sets a parameter for the light client. @@ -92,13 +95,27 @@ func Logger(l log.Logger) Option { } // MaxClockDrift defines how much new header's time can drift into -// the future. Default: 10s. +// the future relative to the light clients local time. Default: 10s. func MaxClockDrift(d time.Duration) Option { return func(c *Client) { c.maxClockDrift = d } } +// MaxBlockLag represents the maximum time difference between the realtime +// that a block is received and the timestamp of that block. +// One can approximate it to the maximum block production time +// +// As an example, say the light client received block B at a time +// 12:05 (this is the real time) and the time on the block +// was 12:00. Then the lag here is 5 minutes. +// Default: 10s +func MaxBlockLag(d time.Duration) Option { + return func(c *Client) { + c.maxBlockLag = d + } +} + // Client represents a light client, connected to a single chain, which gets // light blocks from a primary provider, verifies them either sequentially or by // skipping some and stores them in a trusted store (usually, a local FS). @@ -110,6 +127,7 @@ type Client struct { verificationMode mode trustLevel tmmath.Fraction maxClockDrift time.Duration + maxBlockLag time.Duration // Mutex for locking during changes of the light clients providers providerMutex tmsync.Mutex @@ -197,6 +215,7 @@ func NewClientFromTrustedStore( verificationMode: skipping, trustLevel: DefaultTrustLevel, maxClockDrift: defaultMaxClockDrift, + maxBlockLag: defaultMaxBlockLag, primary: primary, witnesses: witnesses, trustedStore: trustedStore, @@ -952,13 +971,15 @@ func (c *Client) lightBlockFromPrimary(ctx context.Context, height int64) (*type case provider.ErrNoResponse, provider.ErrLightBlockNotFound: // we find a new witness to replace the primary - c.logger.Debug("error from light block request from primary, replacing...", "error", err, "primary", c.primary) + c.logger.Debug("error from light block request from primary, replacing...", + "error", err, "height", height, "primary", c.primary) return c.findNewPrimary(ctx, height, false) default: // The light client has most likely received either provider.ErrUnreliableProvider or provider.ErrBadLightBlock // These errors mean that the light client should drop the primary and try with another provider instead - c.logger.Error("error from light block request from primary, removing...", "error", err, "primary", c.primary) + c.logger.Error("error from light block request from primary, removing...", + "error", err, "height", height, "primary", c.primary) return c.findNewPrimary(ctx, height, true) } } diff --git a/light/detector.go b/light/detector.go index 50b9f671b..b63763cf2 100644 --- a/light/detector.go +++ b/light/detector.go @@ -11,7 +11,7 @@ import ( "github.com/tendermint/tendermint/types" ) -// The detector component of the light client detect and handles attacks on the light client. +// The detector component of the light client detects and handles attacks on the light client. // More info here: // tendermint/docs/architecture/adr-047-handling-evidence-from-light-client.md @@ -20,7 +20,7 @@ import ( // It takes the target verified header and compares it with the headers of a set of // witness providers that the light client is connected to. If a conflicting header // is returned it verifies and examines the conflicting header against the verified -// trace that was produced from the primary. If successful it produces two sets of evidence +// trace that was produced from the primary. If successful, it produces two sets of evidence // and sends them to the opposite provider before halting. // // If there are no conflictinge headers, the light client deems the verified target header @@ -64,56 +64,14 @@ func (c *Client) detectDivergence(ctx context.Context, primaryTrace []*types.Lig // need to find the point that the headers diverge and examine this for any evidence of an attack. // // We combine these actions together, verifying the witnesses headers and outputting the trace - // which captures the bifurcation point and if successful provides the information to create - supportingWitness := c.witnesses[e.WitnessIndex] - witnessTrace, primaryBlock, err := c.examineConflictingHeaderAgainstTrace( - ctx, - primaryTrace, - e.Block.SignedHeader, - supportingWitness, - now, - ) + // which captures the bifurcation point and if successful provides the information to create valid evidence. + err := c.handleConflictingHeaders(ctx, primaryTrace, e.Block, e.WitnessIndex, now) if err != nil { - c.logger.Info("error validating witness's divergent header", "witness", supportingWitness, "err", err) - witnessesToRemove = append(witnessesToRemove, e.WitnessIndex) - continue + // return information of the attack + return err } - - // We are suspecting that the primary is faulty, hence we hold the witness as the source of truth - // and generate evidence against the primary that we can send to the witness - primaryEv := newLightClientAttackEvidence(primaryBlock, witnessTrace[len(witnessTrace)-1], witnessTrace[0]) - c.logger.Error("ATTEMPTED ATTACK DETECTED. Sending evidence againt primary by witness", "ev", primaryEv, - "primary", c.primary, "witness", supportingWitness) - c.sendEvidence(ctx, primaryEv, supportingWitness) - - if primaryBlock.Commit.Round != witnessTrace[len(witnessTrace)-1].Commit.Round { - c.logger.Info("The light client has detected, and prevented, an attempted amnesia attack." + - " We think this attack is pretty unlikely, so if you see it, that's interesting to us." + - " Can you let us know by opening an issue through https://github.com/tendermint/tendermint/issues/new?") - } - - // This may not be valid because the witness itself is at fault. So now we reverse it, examining the - // trace provided by the witness and holding the primary as the source of truth. Note: primary may not - // respond but this is okay as we will halt anyway. - primaryTrace, witnessBlock, err := c.examineConflictingHeaderAgainstTrace( - ctx, - witnessTrace, - primaryBlock.SignedHeader, - c.primary, - now, - ) - if err != nil { - c.logger.Info("Error validating primary's divergent header", "primary", c.primary, "err", err) - return ErrLightClientAttack - } - - // We now use the primary trace to create evidence against the witness and send it to the primary - witnessEv := newLightClientAttackEvidence(witnessBlock, primaryTrace[len(primaryTrace)-1], primaryTrace[0]) - c.logger.Error("Sending evidence against witness by primary", "ev", witnessEv, - "primary", c.primary, "witness", supportingWitness) - c.sendEvidence(ctx, witnessEv, c.primary) - // We return the error and don't process anymore witnesses - return ErrLightClientAttack + // if attempt to generate conflicting headers failed then remove witness + witnessesToRemove = append(witnessesToRemove, e.WitnessIndex) case errBadWitness: c.logger.Info("witness returned an error during header comparison, removing...", @@ -135,7 +93,7 @@ func (c *Client) detectDivergence(ctx context.Context, primaryTrace []*types.Lig return nil } - // 2. ELse all witnesses have either not responded, don't have the block or sent invalid blocks. + // 2. Else all witnesses have either not responded, don't have the block or sent invalid blocks. return ErrFailedHeaderCrossReferencing } @@ -151,16 +109,76 @@ func (c *Client) compareNewHeaderWithWitness(ctx context.Context, errc chan erro lightBlock, err := witness.LightBlock(ctx, h.Height) switch err { + // no error means we move on to checking the hash of the two headers case nil: break + // the witness hasn't been helpful in comparing headers, we mark the response and continue + // comparing with the rest of the witnesses case provider.ErrNoResponse, provider.ErrLightBlockNotFound: errc <- err return + // the witness' head of the blockchain is lower than the height of the primary. This could be one of + // two things: + // 1) The witness is lagging behind + // 2) The primary may be performing a lunatic attack with a height and time in the future + case provider.ErrHeightTooHigh: + // The light client now asks for the latest header that the witness has + var isTargetHeight bool + isTargetHeight, lightBlock, err = c.getTargetBlockOrLatest(ctx, h.Height, witness) + if err != nil { + errc <- err + return + } + + // if the witness caught up and has returned a block of the target height then we can + // break from this switch case and continue to verify the hashes + if isTargetHeight { + break + } + + // witness' last header is below the primary's header. We check the times to see if the blocks + // have conflicting times + if !lightBlock.Time.Before(h.Time) { + errc <- errConflictingHeaders{Block: lightBlock, WitnessIndex: witnessIndex} + return + } + + // the witness is behind. We wait for a period WAITING = 2 * DRIFT + LAG. + // This should give the witness ample time if it is a participating member + // of consensus to produce a block that has a time that is after the primary's + // block time. If not the witness is too far behind and the light client removes it + time.Sleep(2*c.maxClockDrift + c.maxBlockLag) + isTargetHeight, lightBlock, err = c.getTargetBlockOrLatest(ctx, h.Height, witness) + if err != nil { + errc <- errBadWitness{Reason: err, WitnessIndex: witnessIndex} + return + } + if isTargetHeight { + break + } + + // the witness still doesn't have a block at the height of the primary. + // Check if there is a conflicting time + if !lightBlock.Time.Before(h.Time) { + errc <- errConflictingHeaders{Block: lightBlock, WitnessIndex: witnessIndex} + return + } + + // Following this request response procedure, the witness has been unable to produce a block + // that can somehow conflict with the primary's block. We thus conclude that the witness + // is too far behind and thus we return a no response error. + // + // NOTE: If the clock drift / lag has been miscalibrated it is feasible that the light client has + // drifted too far ahead for any witness to be able provide a comparable block and thus may allow + // for a malicious primary to attack it + errc <- provider.ErrNoResponse + return + default: - // all other errors (i.e. invalid block or unreliable provider) we mark the witness as bad - // and remove it + // all other errors (i.e. invalid block, closed connection or unreliable provider) we mark the + // witness as bad and remove it errc <- errBadWitness{Reason: err, WitnessIndex: witnessIndex} return } @@ -181,6 +199,67 @@ func (c *Client) sendEvidence(ctx context.Context, ev *types.LightClientAttackEv } } +// handleConflictingHeaders handles the primary style of attack, which is where a primary and witness have +// two headers of the same height but with different hashes +func (c *Client) handleConflictingHeaders( + ctx context.Context, + primaryTrace []*types.LightBlock, + challendingBlock *types.LightBlock, + witnessIndex int, + now time.Time, +) error { + supportingWitness := c.witnesses[witnessIndex] + witnessTrace, primaryBlock, err := c.examineConflictingHeaderAgainstTrace( + ctx, + primaryTrace, + challendingBlock, + supportingWitness, + now, + ) + if err != nil { + c.logger.Info("error validating witness's divergent header", "witness", supportingWitness, "err", err) + return nil + } + + // We are suspecting that the primary is faulty, hence we hold the witness as the source of truth + // and generate evidence against the primary that we can send to the witness + commonBlock, trustedBlock := witnessTrace[0], witnessTrace[len(witnessTrace)-1] + evidenceAgainstPrimary := newLightClientAttackEvidence(primaryBlock, trustedBlock, commonBlock) + c.logger.Error("ATTEMPTED ATTACK DETECTED. Sending evidence againt primary by witness", "ev", evidenceAgainstPrimary, + "primary", c.primary, "witness", supportingWitness) + c.sendEvidence(ctx, evidenceAgainstPrimary, supportingWitness) + + if primaryBlock.Commit.Round != witnessTrace[len(witnessTrace)-1].Commit.Round { + c.logger.Info("The light client has detected, and prevented, an attempted amnesia attack." + + " We think this attack is pretty unlikely, so if you see it, that's interesting to us." + + " Can you let us know by opening an issue through https://github.com/tendermint/tendermint/issues/new?") + } + + // This may not be valid because the witness itself is at fault. So now we reverse it, examining the + // trace provided by the witness and holding the primary as the source of truth. Note: primary may not + // respond but this is okay as we will halt anyway. + primaryTrace, witnessBlock, err := c.examineConflictingHeaderAgainstTrace( + ctx, + witnessTrace, + primaryBlock, + c.primary, + now, + ) + if err != nil { + c.logger.Info("Error validating primary's divergent header", "primary", c.primary, "err", err) + return ErrLightClientAttack + } + + // We now use the primary trace to create evidence against the witness and send it to the primary + commonBlock, trustedBlock = primaryTrace[0], primaryTrace[len(primaryTrace)-1] + evidenceAgainstWitness := newLightClientAttackEvidence(witnessBlock, trustedBlock, commonBlock) + c.logger.Error("Sending evidence against witness by primary", "ev", evidenceAgainstWitness, + "primary", c.primary, "witness", supportingWitness) + c.sendEvidence(ctx, evidenceAgainstWitness, c.primary) + // We return the error and don't process anymore witnesses + return ErrLightClientAttack +} + // examineConflictingHeaderAgainstTrace takes a trace from one provider and a divergent header that // it has received from another and preforms verifySkipping at the heights of each of the intermediate // headers in the trace until it reaches the divergentHeader. 1 of 2 things can happen. @@ -189,22 +268,66 @@ func (c *Client) sendEvidence(ctx context.Context, ev *types.LightClientAttackEv // is the bifurcation point and the light client can create evidence from it // 2. The source stops responding, doesn't have the block or sends an invalid header in which case we // return the error and remove the witness +// +// CONTRACT: +// 1. Trace can not be empty len(trace) > 0 +// 2. The last block in the trace can not be of a lower height than the target block +// trace[len(trace)-1].Height >= targetBlock.Height +// 3. The func (c *Client) examineConflictingHeaderAgainstTrace( ctx context.Context, trace []*types.LightBlock, - divergentHeader *types.SignedHeader, - source provider.Provider, now time.Time) ([]*types.LightBlock, *types.LightBlock, error) { + targetBlock *types.LightBlock, + source provider.Provider, now time.Time, +) ([]*types.LightBlock, *types.LightBlock, error) { + + var ( + previouslyVerifiedBlock, sourceBlock *types.LightBlock + sourceTrace []*types.LightBlock + err error + ) - var previouslyVerifiedBlock *types.LightBlock + if targetBlock.Height < trace[0].Height { + return nil, nil, fmt.Errorf("target block has a height lower than the trusted height (%d < %d)", + targetBlock.Height, trace[0].Height) + } for idx, traceBlock := range trace { - // The first block in the trace MUST be the same to the light block that the source produces - // else we cannot continue with verification. - sourceBlock, err := source.LightBlock(ctx, traceBlock.Height) - if err != nil { - return nil, nil, err + // this case only happens in a forward lunatic attack. We treat the block with the + // height directly after the targetBlock as the divergent block + if traceBlock.Height > targetBlock.Height { + // sanity check that the time of the traceBlock is indeed less than that of the targetBlock. If the trace + // was correctly verified we should expect monotonically increasing time. This means that if the block at + // the end of the trace has a lesser time than the target block then all blocks in the trace should have a + // lesser time + if traceBlock.Time.After(targetBlock.Time) { + return nil, nil, + errors.New("sanity check failed: expected traceblock to have a lesser time than the target block") + } + + // before sending back the divergent block and trace we need to ensure we have verified + // the final gap between the previouslyVerifiedBlock and the targetBlock + if previouslyVerifiedBlock.Height != targetBlock.Height { + sourceTrace, err = c.verifySkipping(ctx, source, previouslyVerifiedBlock, targetBlock, now) + if err != nil { + return nil, nil, fmt.Errorf("verifySkipping of conflicting header failed: %w", err) + } + } + return sourceTrace, traceBlock, nil } + // get the corresponding block from the source to verify and match up against the traceBlock + if traceBlock.Height == targetBlock.Height { + sourceBlock = targetBlock + } else { + sourceBlock, err = source.LightBlock(ctx, traceBlock.Height) + if err != nil { + return nil, nil, fmt.Errorf("failed to examine trace: %w", err) + } + } + + // The first block in the trace MUST be the same to the light block that the source produces + // else we cannot continue with verification. if idx == 0 { if shash, thash := sourceBlock.Hash(), traceBlock.Hash(); !bytes.Equal(shash, thash) { return nil, nil, fmt.Errorf("trusted block is different to the source's first block (%X = %X)", @@ -216,25 +339,55 @@ func (c *Client) examineConflictingHeaderAgainstTrace( // we check that the source provider can verify a block at the same height of the // intermediate height - trace, err := c.verifySkipping(ctx, source, previouslyVerifiedBlock, sourceBlock, now) + sourceTrace, err = c.verifySkipping(ctx, source, previouslyVerifiedBlock, sourceBlock, now) if err != nil { return nil, nil, fmt.Errorf("verifySkipping of conflicting header failed: %w", err) } // check if the headers verified by the source has diverged from the trace if shash, thash := sourceBlock.Hash(), traceBlock.Hash(); !bytes.Equal(shash, thash) { // Bifurcation point found! - return trace, traceBlock, nil + return sourceTrace, traceBlock, nil } // headers are still the same. update the previouslyVerifiedBlock previouslyVerifiedBlock = sourceBlock } - // We have reached the end of the trace without observing a divergence. The last header is thus different - // from the divergent header that the source originally sent us, then we return an error. - return nil, nil, fmt.Errorf("source provided different header to the original header it provided (%X != %X)", - previouslyVerifiedBlock.Hash(), divergentHeader.Hash()) + // We have reached the end of the trace. This should never happen. This can only happen if one of the stated + // prerequisites to this function were not met. Namely that either trace[len(trace)-1].Height < targetBlock.Height + // or that trace[i].Hash() != targetBlock.Hash() + return nil, nil, errNoDivergence + +} + +// getTargetBlockOrLatest gets the latest height, if it is greater than the target height then it queries +// the target heght else it returns the latest. returns true if it successfully managed to acquire the target +// height. +func (c *Client) getTargetBlockOrLatest( + ctx context.Context, + height int64, + witness provider.Provider, +) (bool, *types.LightBlock, error) { + lightBlock, err := witness.LightBlock(ctx, 0) + if err != nil { + return false, nil, err + } + + if lightBlock.Height == height { + // the witness has caught up to the height of the provider's signed header. We + // can resume with checking the hashes. + return true, lightBlock, nil + } + + if lightBlock.Height > height { + // the witness has caught up. We recursively call the function again. However in order + // to avoud a wild goose chase where the witness sends us one header below and one header + // above the height we set a timeout to the context + lightBlock, err := witness.LightBlock(ctx, height) + return true, lightBlock, err + } + return false, lightBlock, nil } // newLightClientAttackEvidence determines the type of attack and then forms the evidence filling out diff --git a/light/detector_test.go b/light/detector_test.go index 02e7bb249..97ae16e59 100644 --- a/light/detector_test.go +++ b/light/detector_test.go @@ -170,6 +170,139 @@ func TestLightClientAttackEvidence_Equivocation(t *testing.T) { } } +func TestLightClientAttackEvidence_ForwardLunatic(t *testing.T) { + // primary performs a lunatic attack but changes the time of the header to + // something in the future relative to the blockchain + var ( + latestHeight = int64(10) + valSize = 5 + forgedHeight = int64(12) + proofHeight = int64(11) + primaryHeaders = make(map[int64]*types.SignedHeader, forgedHeight) + primaryValidators = make(map[int64]*types.ValidatorSet, forgedHeight) + ) + + witnessHeaders, witnessValidators, chainKeys := genMockNodeWithKeys(chainID, latestHeight, valSize, 2, bTime) + + // primary has the exact same headers except it forges one extra header in the future using keys from 2/5ths of + // the validators + for h := range witnessHeaders { + primaryHeaders[h] = witnessHeaders[h] + primaryValidators[h] = witnessValidators[h] + } + forgedKeys := chainKeys[latestHeight].ChangeKeys(3) // we change 3 out of the 5 validators (still 2/5 remain) + primaryValidators[forgedHeight] = forgedKeys.ToValidators(2, 0) + primaryHeaders[forgedHeight] = forgedKeys.GenSignedHeader( + chainID, + forgedHeight, + bTime.Add(time.Duration(latestHeight+1)*time.Minute), // 11 mins + nil, + primaryValidators[forgedHeight], + primaryValidators[forgedHeight], + hash("app_hash"), + hash("cons_hash"), + hash("results_hash"), + 0, len(forgedKeys), + ) + + witness := mockp.New(chainID, witnessHeaders, witnessValidators) + primary := mockp.New(chainID, primaryHeaders, primaryValidators) + + laggingWitness := witness.Copy("laggingWitness") + + // In order to perform the attack, the primary needs at least one accomplice as a witness to also + // send the forged block + accomplice := primary + + c, err := light.NewClient( + ctx, + chainID, + light.TrustOptions{ + Period: 4 * time.Hour, + Height: 1, + Hash: primaryHeaders[1].Hash(), + }, + primary, + []provider.Provider{witness, accomplice}, + dbs.New(dbm.NewMemDB()), + light.Logger(log.TestingLogger()), + light.MaxClockDrift(1*time.Second), + light.MaxBlockLag(1*time.Second), + ) + require.NoError(t, err) + + // two seconds later, the supporting withness should receive the header that can be used + // to prove that there was an attack + go func() { + time.Sleep(2 * time.Second) + vals := chainKeys[latestHeight].ToValidators(2, 0) + newLb := &types.LightBlock{ + SignedHeader: chainKeys[latestHeight].GenSignedHeader( + chainID, + proofHeight, + bTime.Add(time.Duration(proofHeight+1)*time.Minute), // 12 mins + nil, + vals, + vals, + hash("app_hash"), + hash("cons_hash"), + hash("results_hash"), + 0, len(chainKeys), + ), + ValidatorSet: vals, + } + witness.AddLightBlock(newLb) + }() + + // Now assert that verification returns an error. We craft the light clients time to be a little ahead of the chain + // to allow a window for the attack to manifest itself. + _, err = c.Update(ctx, bTime.Add(time.Duration(forgedHeight)*time.Minute)) + if assert.Error(t, err) { + assert.Equal(t, light.ErrLightClientAttack, err) + } + + // Check evidence was sent to the witness against the full node + evAgainstPrimary := &types.LightClientAttackEvidence{ + ConflictingBlock: &types.LightBlock{ + SignedHeader: primaryHeaders[forgedHeight], + ValidatorSet: primaryValidators[forgedHeight], + }, + CommonHeight: latestHeight, + } + assert.True(t, witness.HasEvidence(evAgainstPrimary)) + + // We attempt the same call but now the supporting witness has a block which should + // immediately conflict in time with the primary + _, err = c.VerifyLightBlockAtHeight(ctx, forgedHeight, bTime.Add(time.Duration(forgedHeight)*time.Minute)) + if assert.Error(t, err) { + assert.Equal(t, light.ErrLightClientAttack, err) + } + assert.True(t, witness.HasEvidence(evAgainstPrimary)) + + // Lastly we test the unfortunate case where the light clients supporting witness doesn't update + // in enough time + c, err = light.NewClient( + ctx, + chainID, + light.TrustOptions{ + Period: 4 * time.Hour, + Height: 1, + Hash: primaryHeaders[1].Hash(), + }, + primary, + []provider.Provider{laggingWitness, accomplice}, + dbs.New(dbm.NewMemDB()), + light.Logger(log.TestingLogger()), + light.MaxClockDrift(1*time.Second), + light.MaxBlockLag(1*time.Second), + ) + require.NoError(t, err) + + _, err = c.Update(ctx, bTime.Add(time.Duration(forgedHeight)*time.Minute)) + assert.NoError(t, err) + +} + // 1. Different nodes therefore a divergent header is produced. // => light client returns an error upon creation because primary and witness // have a different view. @@ -255,3 +388,39 @@ func TestClientDivergentTraces3(t *testing.T) { assert.Error(t, err) assert.Equal(t, 1, len(c.Witnesses())) } + +// 4. Witness has a divergent header but can not produce a valid trace to back it up. +// It should be ignored +func TestClientDivergentTraces4(t *testing.T) { + _, primaryHeaders, primaryVals := genMockNode(chainID, 10, 5, 2, bTime) + primary := mockp.New(chainID, primaryHeaders, primaryVals) + + firstBlock, err := primary.LightBlock(ctx, 1) + require.NoError(t, err) + + _, mockHeaders, mockVals := genMockNode(chainID, 10, 5, 2, bTime) + witness := primary.Copy("witness") + witness.AddLightBlock(&types.LightBlock{ + SignedHeader: mockHeaders[10], + ValidatorSet: mockVals[10], + }) + + c, err := light.NewClient( + ctx, + chainID, + light.TrustOptions{ + Height: 1, + Hash: firstBlock.Hash(), + Period: 4 * time.Hour, + }, + primary, + []provider.Provider{witness}, + dbs.New(dbm.NewMemDB()), + light.Logger(log.TestingLogger()), + ) + require.NoError(t, err) + + _, err = c.VerifyLightBlockAtHeight(ctx, 10, bTime.Add(1*time.Hour)) + assert.Error(t, err) + assert.Equal(t, 1, len(c.Witnesses())) +} diff --git a/light/errors.go b/light/errors.go index 3669e6fcb..c06ff1a94 100644 --- a/light/errors.go +++ b/light/errors.go @@ -100,3 +100,7 @@ type errBadWitness struct { func (e errBadWitness) Error() string { return fmt.Sprintf("Witness %d returned error: %s", e.WitnessIndex, e.Reason.Error()) } + +var errNoDivergence = errors.New( + "sanity check failed: no divergence between the original trace and the provider's new trace", +) diff --git a/light/provider/errors.go b/light/provider/errors.go index 40e0c6fc8..355ec3475 100644 --- a/light/provider/errors.go +++ b/light/provider/errors.go @@ -6,12 +6,19 @@ import ( ) var ( + // ErrHeightTooHigh is returned when the height is higher than the last + // block that the provider has. The light client will not remove the provider + ErrHeightTooHigh = errors.New("height requested is too high") // ErrLightBlockNotFound is returned when a provider can't find the - // requested header. The light client will not remove the provider + // requested header (i.e. it has been pruned). + // The light client will not remove the provider ErrLightBlockNotFound = errors.New("light block not found") // ErrNoResponse is returned if the provider doesn't respond to the // request in a given time. The light client will not remove the provider ErrNoResponse = errors.New("client failed to respond") + // ErrConnectionClosed is returned if the provider closes the connection. + // In this case we remove the provider. + ErrConnectionClosed = errors.New("client closed connection") ) // ErrBadLightBlock is returned when a provider returns an invalid diff --git a/light/provider/http/http.go b/light/provider/http/http.go index 125589b6b..a5e2f02d1 100644 --- a/light/provider/http/http.go +++ b/light/provider/http/http.go @@ -2,7 +2,6 @@ package http import ( "context" - "errors" "fmt" "math/rand" "net/url" @@ -117,6 +116,12 @@ func (p *http) LightBlock(ctx context.Context, height int64) (*types.LightBlock, return nil, err } + if height != 0 && sh.Height != height { + return nil, provider.ErrBadLightBlock{ + Reason: fmt.Errorf("height %d responded doesn't match height %d requested", sh.Height, height), + } + } + vs, err := p.validatorSet(ctx, &sh.Height) if err != nil { return nil, err @@ -189,17 +194,13 @@ func (p *http) validatorSet(ctx context.Context, height *int64) (*types.Validato return nil, provider.ErrBadLightBlock{Reason: e} case *rpctypes.RPCError: - // check if the error indicates that the peer doesn't have the block - if strings.Contains(e.Data, ctypes.ErrHeightNotAvailable.Error()) || - strings.Contains(e.Data, ctypes.ErrHeightExceedsChainHead.Error()) { - return nil, provider.ErrLightBlockNotFound - } - return nil, provider.ErrBadLightBlock{Reason: e} + // process the rpc error and return the corresponding error to the light client + return nil, p.parseRPCError(e) default: - // If we don't know the error then by default we return a bad light block error and + // If we don't know the error then by default we return an unreliable provider error and // terminate the connection with the peer. - return nil, provider.ErrBadLightBlock{Reason: e} + return nil, provider.ErrUnreliableProvider{Reason: e.Error()} } // update the total and increment the page index so we can fetch the @@ -236,19 +237,13 @@ func (p *http) signedHeader(ctx context.Context, height *int64) (*types.SignedHe return nil, provider.ErrBadLightBlock{Reason: e} case *rpctypes.RPCError: - // check if the error indicates that the peer doesn't have the block - if strings.Contains(e.Data, ctypes.ErrHeightNotAvailable.Error()) || - strings.Contains(e.Data, ctypes.ErrHeightExceedsChainHead.Error()) { - return nil, p.noBlock() - } - - // for every other error, the provider returns a bad block - return nil, provider.ErrBadLightBlock{Reason: errors.New(e.Data)} + // process the rpc error and return the corresponding error to the light client + return nil, p.parseRPCError(e) default: - // If we don't know the error then by default we return a bad light block error and + // If we don't know the error then by default we return an unreliable provider error and // terminate the connection with the peer. - return nil, provider.ErrBadLightBlock{Reason: e} + return nil, provider.ErrUnreliableProvider{Reason: e.Error()} } } return nil, p.noResponse() @@ -264,14 +259,37 @@ func (p *http) noResponse() error { return provider.ErrNoResponse } -func (p *http) noBlock() error { +func (p *http) noBlock(e error) error { p.noBlockCount++ if p.noBlockCount > p.noBlockThreshold { return provider.ErrUnreliableProvider{ Reason: fmt.Sprintf("failed to provide a block after %d attempts", p.noBlockCount), } } - return provider.ErrLightBlockNotFound + return e +} + +// parseRPCError process the error and return the corresponding error to the light clent +// NOTE: When an error is sent over the wire it gets "flattened" hence we are unable to use error +// checking functions like errors.Is() to unwrap the error. +func (p *http) parseRPCError(e *rpctypes.RPCError) error { + switch { + // 1) check if the error indicates that the peer doesn't have the block + case strings.Contains(e.Data, ctypes.ErrHeightNotAvailable.Error()): + return p.noBlock(provider.ErrLightBlockNotFound) + + // 2) check if the height requested is too high + case strings.Contains(e.Data, ctypes.ErrHeightExceedsChainHead.Error()): + return p.noBlock(provider.ErrHeightTooHigh) + + // 3) check if the provider closed the connection + case strings.Contains(e.Data, "connection refused"): + return provider.ErrConnectionClosed + + // 4) else return a generic error + default: + return provider.ErrBadLightBlock{Reason: e} + } } func validateHeight(height int64) (*int64, error) { diff --git a/light/provider/http/http_test.go b/light/provider/http/http_test.go index 240c97973..728ae3c89 100644 --- a/light/provider/http/http_test.go +++ b/light/provider/http/http_test.go @@ -79,11 +79,11 @@ func TestProvider(t *testing.T) { require.NoError(t, err) assert.Equal(t, lower, lb.Height) - // // fetching missing heights (both future and pruned) should return appropriate errors + // fetching missing heights (both future and pruned) should return appropriate errors lb, err = p.LightBlock(context.Background(), 1000) require.Error(t, err) require.Nil(t, lb) - assert.Equal(t, provider.ErrLightBlockNotFound, err) + assert.Equal(t, provider.ErrHeightTooHigh, err) _, err = p.LightBlock(context.Background(), 1) require.Error(t, err) diff --git a/light/provider/mock/mock.go b/light/provider/mock/mock.go index 310646de0..f527ef128 100644 --- a/light/provider/mock/mock.go +++ b/light/provider/mock/mock.go @@ -15,6 +15,7 @@ type Mock struct { headers map[int64]*types.SignedHeader vals map[int64]*types.ValidatorSet evidenceToReport map[string]types.Evidence // hash => evidence + latestHeight int64 } var _ provider.Provider = (*Mock)(nil) @@ -22,11 +23,18 @@ var _ provider.Provider = (*Mock)(nil) // New creates a mock provider with the given set of headers and validator // sets. func New(id string, headers map[int64]*types.SignedHeader, vals map[int64]*types.ValidatorSet) *Mock { + height := int64(0) + for h := range headers { + if h > height { + height = h + } + } return &Mock{ id: id, headers: headers, vals: vals, evidenceToReport: make(map[string]types.Evidence), + latestHeight: height, } } @@ -46,15 +54,15 @@ func (p *Mock) String() string { func (p *Mock) LightBlock(_ context.Context, height int64) (*types.LightBlock, error) { var lb *types.LightBlock - if height == 0 && len(p.headers) > 0 { - sh := p.headers[int64(len(p.headers))] - vals := p.vals[int64(len(p.vals))] - lb = &types.LightBlock{ - SignedHeader: sh, - ValidatorSet: vals, - } + if height > p.latestHeight { + return nil, provider.ErrHeightTooHigh + } + + if height == 0 && len(p.headers) > 0 { + height = p.latestHeight } + if _, ok := p.headers[height]; ok { sh := p.headers[height] vals := p.vals[height] @@ -84,3 +92,18 @@ func (p *Mock) HasEvidence(ev types.Evidence) bool { _, ok := p.evidenceToReport[string(ev.Hash())] return ok } + +func (p *Mock) AddLightBlock(lb *types.LightBlock) { + if err := lb.ValidateBasic(lb.ChainID); err != nil { + panic(fmt.Sprintf("unable to add light block, err: %v", err)) + } + p.headers[lb.Height] = lb.SignedHeader + p.vals[lb.Height] = lb.ValidatorSet + if lb.Height > p.latestHeight { + p.latestHeight = lb.Height + } +} + +func (p *Mock) Copy(id string) *Mock { + return New(id, p.headers, p.vals) +} diff --git a/rpc/core/env.go b/rpc/core/env.go index 889c1df14..aa9ce4ba4 100644 --- a/rpc/core/env.go +++ b/rpc/core/env.go @@ -154,7 +154,7 @@ func getHeight(latestHeight int64, heightPtr *int64) (int64, error) { } base := env.BlockStore.Base() if height < base { - return 0, fmt.Errorf("%w (requested height: %d, base height: %d)", ctypes.ErrHeightExceedsChainHead, height, base) + return 0, fmt.Errorf("%w (requested height: %d, base height: %d)", ctypes.ErrHeightNotAvailable, height, base) } return height, nil }