statesync: check all necessary heights when adding snapshot to pool (#5516) (#5518)

Fixes #5511.

CHANGELOG_PENDING.md

@@ -1,6 +1,6 @@
 # Unreleased Changes
 
-## v0.34.0-rc5
+## v0.34.0-rc6
 
 Special thanks to external contributors on this release:
 
@@ -15,7 +15,6 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi
 - P2P Protocol
 
 - Go API
-    - [evidence] [\#5499](https://github.com/tendermint/tendermint/pull/5449) `MaxNum` evidence consensus parameter has been changed to `MaxBytes` (@cmwaters)
 
 - Blockchain Protocol
 
@@ -23,11 +22,6 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi
 
 ### IMPROVEMENTS
 
-- [config] \#5433 `statesync.rpc_servers` is now properly set when writing the configuration file (@erikgrinaker)
-
-- [privval] \#5437 `NewSignerDialerEndpoint` can now be given `SignerServiceEndpointOption` (@erikgrinaker)
+- [statesync] \#5516 Check that all heights necessary to rebuild state for a snapshot exist before adding the snapshot to the pool. (@erikgrinaker)
 
 ### BUG FIXES
-
-- [privval] \#5441 Fix faulty ping message encoding causing nil message errors in logs (@erikgrinaker)
-- [rpc] \#5459 Register the interface of public keys for json encoding (@marbar3778)

statesync/stateprovider.go

@@ -94,6 +94,22 @@ func (s *lightClientStateProvider) AppHash(ctx context.Context, height uint64) (
 	if err != nil {
 		return nil, err
 	}
+	// We also try to fetch the blocks at height H and H+2, since we need these
+	// when building the state while restoring the snapshot. This avoids the race
+	// condition where we try to restore a snapshot before H+2 exists.
+	//
+	// FIXME This is a hack, since we can't add new methods to the interface without
+	// breaking it. We should instead have a Has(ctx, height) method which checks
+	// that the state provider has access to the necessary data for the height.
+	// We piggyback on AppHash() since it's called when adding snapshots to the pool.
+	_, err = s.lc.VerifyLightBlockAtHeight(ctx, int64(height+2), time.Now())
+	if err != nil {
+		return nil, err
+	}
+	_, err = s.lc.VerifyLightBlockAtHeight(ctx, int64(height), time.Now())
+	if err != nil {
+		return nil, err
+	}
 	return header.AppHash, nil
 }