e2e: automatically prune old app snapshots (#7034) (#7063)

This PR tackles the case of using the e2e application in a long lived testnet. The application continually saves snapshots (usually every 100 blocks) which after a while bloats the size of the application. This PR prunes older snapshots so that only the most recent 10 snapshots remain.

(cherry picked from commit 5703ae2fb3)

Co-authored-by: Callum Waters <cmwaters19@gmail.com>

test/e2e/app/app.go

@@ -184,6 +184,10 @@ func (app *Application) Commit() abci.ResponseCommit {
 			panic(err)
 		}
 		app.logger.Info("Created state sync snapshot", "height", snapshot.Height)
+		err = app.snapshots.Prune(maxSnapshotCount)
+		if err != nil {
+			app.logger.Error("Failed to prune snapshots", "err", err)
+		}
 	}
 	retainHeight := int64(0)
 	if app.cfg.RetainBlocks > 0 {

test/e2e/app/snapshots.go

@@ -16,6 +16,9 @@ import (
 
 const (
 	snapshotChunkSize = 1e6
+
+	// Keep only the most recent 10 snapshots. Older snapshots are pruned
+	maxSnapshotCount = 10
 )
 
 // SnapshotStore stores state sync snapshots. Snapshots are stored simply as
@@ -105,6 +108,27 @@ func (s *SnapshotStore) Create(state *State) (abci.Snapshot, error) {
 	return snapshot, nil
 }
 
+// Prune removes old snapshots ensuring only the most recent n snapshots remain
+func (s *SnapshotStore) Prune(n int) error {
+	s.Lock()
+	defer s.Unlock()
+	// snapshots are appended to the metadata struct, hence pruning removes from
+	// the front of the array
+	i := 0
+	for ; i < len(s.metadata)-n; i++ {
+		h := s.metadata[i].Height
+		if err := os.Remove(filepath.Join(s.dir, fmt.Sprintf("%v.json", h))); err != nil {
+			return err
+		}
+	}
+
+	// update metadata by removing the deleted snapshots
+	pruned := make([]abci.Snapshot, len(s.metadata[i:]))
+	copy(pruned, s.metadata[i:])
+	s.metadata = pruned
+	return nil
+}
+
 // List lists available snapshots.
 func (s *SnapshotStore) List() ([]*abci.Snapshot, error) {
 	s.RLock()

test/e2e/generator/generate.go

@@ -54,7 +54,7 @@ var (
 		e2e.StateSyncRPC:      45,
 	}
 	nodePersistIntervals  = uniformChoice{0, 1, 5}
-	nodeSnapshotIntervals = uniformChoice{0, 3}
+	nodeSnapshotIntervals = uniformChoice{0, 5}
 	nodeRetainBlocks      = uniformChoice{0, 2 * int(e2e.EvidenceAgeHeight), 4 * int(e2e.EvidenceAgeHeight)}
 	nodePerturbations     = probSetChoice{
 		"disconnect": 0.1,