You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

152 lines
3.7 KiB

  1. // nolint: gosec
  2. package app
  3. import (
  4. "encoding/json"
  5. "errors"
  6. "fmt"
  7. "io/ioutil"
  8. "math"
  9. "os"
  10. "path/filepath"
  11. "sync"
  12. abci "github.com/tendermint/tendermint/abci/types"
  13. )
  14. const (
  15. snapshotChunkSize = 1e6
  16. )
  17. // SnapshotStore stores state sync snapshots. Snapshots are stored simply as
  18. // JSON files, and chunks are generated on-the-fly by splitting the JSON data
  19. // into fixed-size chunks.
  20. type SnapshotStore struct {
  21. sync.RWMutex
  22. dir string
  23. metadata []abci.Snapshot
  24. }
  25. // NewSnapshotStore creates a new snapshot store.
  26. func NewSnapshotStore(dir string) (*SnapshotStore, error) {
  27. store := &SnapshotStore{dir: dir}
  28. if err := os.MkdirAll(dir, 0755); err != nil {
  29. return nil, err
  30. }
  31. if err := store.loadMetadata(); err != nil {
  32. return nil, err
  33. }
  34. return store, nil
  35. }
  36. // loadMetadata loads snapshot metadata. Does not take out locks, since it's
  37. // called internally on construction.
  38. func (s *SnapshotStore) loadMetadata() error {
  39. file := filepath.Join(s.dir, "metadata.json")
  40. metadata := []abci.Snapshot{}
  41. bz, err := ioutil.ReadFile(file)
  42. switch {
  43. case errors.Is(err, os.ErrNotExist):
  44. case err != nil:
  45. return fmt.Errorf("failed to load snapshot metadata from %q: %w", file, err)
  46. }
  47. if len(bz) != 0 {
  48. err = json.Unmarshal(bz, &metadata)
  49. if err != nil {
  50. return fmt.Errorf("invalid snapshot data in %q: %w", file, err)
  51. }
  52. }
  53. s.metadata = metadata
  54. return nil
  55. }
  56. // saveMetadata saves snapshot metadata. Does not take out locks, since it's
  57. // called internally from e.g. Create().
  58. func (s *SnapshotStore) saveMetadata() error {
  59. bz, err := json.Marshal(s.metadata)
  60. if err != nil {
  61. return err
  62. }
  63. // save the file to a new file and move it to make saving atomic.
  64. newFile := filepath.Join(s.dir, "metadata.json.new")
  65. file := filepath.Join(s.dir, "metadata.json")
  66. err = ioutil.WriteFile(newFile, bz, 0644) // nolint: gosec
  67. if err != nil {
  68. return err
  69. }
  70. return os.Rename(newFile, file)
  71. }
  72. // Create creates a snapshot of the given application state's key/value pairs.
  73. func (s *SnapshotStore) Create(state *State) (abci.Snapshot, error) {
  74. s.Lock()
  75. defer s.Unlock()
  76. bz, err := state.Export()
  77. if err != nil {
  78. return abci.Snapshot{}, err
  79. }
  80. snapshot := abci.Snapshot{
  81. Height: state.Height,
  82. Format: 1,
  83. Hash: hashItems(state.Values),
  84. Chunks: byteChunks(bz),
  85. }
  86. err = ioutil.WriteFile(filepath.Join(s.dir, fmt.Sprintf("%v.json", state.Height)), bz, 0644)
  87. if err != nil {
  88. return abci.Snapshot{}, err
  89. }
  90. s.metadata = append(s.metadata, snapshot)
  91. err = s.saveMetadata()
  92. if err != nil {
  93. return abci.Snapshot{}, err
  94. }
  95. return snapshot, nil
  96. }
  97. // List lists available snapshots.
  98. func (s *SnapshotStore) List() ([]*abci.Snapshot, error) {
  99. s.RLock()
  100. defer s.RUnlock()
  101. snapshots := make([]*abci.Snapshot, len(s.metadata))
  102. for idx := range s.metadata {
  103. snapshots[idx] = &s.metadata[idx]
  104. }
  105. return snapshots, nil
  106. }
  107. // LoadChunk loads a snapshot chunk.
  108. func (s *SnapshotStore) LoadChunk(height uint64, format uint32, chunk uint32) ([]byte, error) {
  109. s.RLock()
  110. defer s.RUnlock()
  111. for _, snapshot := range s.metadata {
  112. if snapshot.Height == height && snapshot.Format == format {
  113. bz, err := ioutil.ReadFile(filepath.Join(s.dir, fmt.Sprintf("%v.json", height)))
  114. if err != nil {
  115. return nil, err
  116. }
  117. return byteChunk(bz, chunk), nil
  118. }
  119. }
  120. return nil, nil
  121. }
  122. // byteChunk returns the chunk at a given index from the full byte slice.
  123. func byteChunk(bz []byte, index uint32) []byte {
  124. start := int(index * snapshotChunkSize)
  125. end := int((index + 1) * snapshotChunkSize)
  126. switch {
  127. case start >= len(bz):
  128. return nil
  129. case end >= len(bz):
  130. return bz[start:]
  131. default:
  132. return bz[start:end]
  133. }
  134. }
  135. // byteChunks calculates the number of chunks in the byte slice.
  136. func byteChunks(bz []byte) uint32 {
  137. return uint32(math.Ceil(float64(len(bz)) / snapshotChunkSize))
  138. }