Browse Source

e2e: automatically prune old app snapshots (#7034)

This PR tackles the case of using the e2e application in a long lived testnet. The application continually saves snapshots (usually every 100 blocks) which after a while bloats the size of the application. This PR prunes older snapshots so that only the most recent 10 snapshots remain.
pull/7062/head
Callum Waters 3 years ago
committed by GitHub
parent
commit
5703ae2fb3
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 29 additions and 1 deletions
  1. +4
    -0
      test/e2e/app/app.go
  2. +24
    -0
      test/e2e/app/snapshots.go
  3. +1
    -1
      test/e2e/generator/generate.go

+ 4
- 0
test/e2e/app/app.go View File

@ -184,6 +184,10 @@ func (app *Application) Commit() abci.ResponseCommit {
panic(err)
}
app.logger.Info("Created state sync snapshot", "height", snapshot.Height)
err = app.snapshots.Prune(maxSnapshotCount)
if err != nil {
app.logger.Error("Failed to prune snapshots", "err", err)
}
}
retainHeight := int64(0)
if app.cfg.RetainBlocks > 0 {


+ 24
- 0
test/e2e/app/snapshots.go View File

@ -16,6 +16,9 @@ import (
const (
snapshotChunkSize = 1e6
// Keep only the most recent 10 snapshots. Older snapshots are pruned
maxSnapshotCount = 10
)
// SnapshotStore stores state sync snapshots. Snapshots are stored simply as
@ -105,6 +108,27 @@ func (s *SnapshotStore) Create(state *State) (abci.Snapshot, error) {
return snapshot, nil
}
// Prune removes old snapshots ensuring only the most recent n snapshots remain
func (s *SnapshotStore) Prune(n int) error {
s.Lock()
defer s.Unlock()
// snapshots are appended to the metadata struct, hence pruning removes from
// the front of the array
i := 0
for ; i < len(s.metadata)-n; i++ {
h := s.metadata[i].Height
if err := os.Remove(filepath.Join(s.dir, fmt.Sprintf("%v.json", h))); err != nil {
return err
}
}
// update metadata by removing the deleted snapshots
pruned := make([]abci.Snapshot, len(s.metadata[i:]))
copy(pruned, s.metadata[i:])
s.metadata = pruned
return nil
}
// List lists available snapshots.
func (s *SnapshotStore) List() ([]*abci.Snapshot, error) {
s.RLock()


+ 1
- 1
test/e2e/generator/generate.go View File

@ -53,7 +53,7 @@ var (
e2e.StateSyncRPC: 45,
}
nodePersistIntervals = uniformChoice{0, 1, 5}
nodeSnapshotIntervals = uniformChoice{0, 3}
nodeSnapshotIntervals = uniformChoice{0, 5}
nodeRetainBlocks = uniformChoice{0, 2 * int(e2e.EvidenceAgeHeight), 4 * int(e2e.EvidenceAgeHeight)}
nodePerturbations = probSetChoice{
"disconnect": 0.1,


Loading…
Cancel
Save