diff --git a/snapshot/state.go b/snapshot/state.go new file mode 100644 index 00000000..cf481772 --- /dev/null +++ b/snapshot/state.go @@ -0,0 +1,93 @@ +package snapshot + +import ( + "fmt" + "os" + "path/filepath" + "sort" + + "github.com/hashicorp/raft" +) + +// RemoveAllTmpSnapshotData removes all temporary Snapshot data from the directory. +// This process is defined as follows: for every directory in dir, if the directory +// is a temporary directory, remove the directory. Then remove all other files +// that contain the name of a temporary directory, minus the temporary suffix, +// as prefix. +func RemoveAllTmpSnapshotData(dir string) error { + files, err := os.ReadDir(dir) + if err != nil { + return nil + } + for _, d := range files { + // If the directory is a temporary directory, remove it. + if d.IsDir() && isTmpName(d.Name()) { + files, err := filepath.Glob(filepath.Join(dir, nonTmpName(d.Name())) + "*") + if err != nil { + return err + } + + fullTmpDirPath := filepath.Join(dir, d.Name()) + for _, f := range files { + if f == fullTmpDirPath { + // Delete the directory last as a sign the deletion is complete. + continue + } + if err := os.Remove(f); err != nil { + return err + } + } + if err := os.RemoveAll(fullTmpDirPath); err != nil { + return err + } + } + } + return nil +} + +// LatestIndex returns the index of the latest snapshot in the given directory. +func LatestIndex(dir string) (uint64, error) { + meta, err := getSnapshots(dir) + if err != nil { + return 0, err + } + if len(meta) == 0 { + return 0, nil + } + return meta[len(meta)-1].Index, nil +} + +func getSnapshots(dir string) ([]*raft.SnapshotMeta, error) { + // Get the eligible snapshots + snapshots, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + + // Populate the metadata + var snapMeta []*raft.SnapshotMeta + for _, snap := range snapshots { + // Ignore any files + if !snap.IsDir() { + continue + } + + // Ignore any temporary snapshots + dirName := snap.Name() + if isTmpName(dirName) { + continue + } + + // Try to read the meta data + meta, err := readMeta(filepath.Join(dir, dirName)) + if err != nil { + return nil, fmt.Errorf("failed to read meta for snapshot %s: %s", dirName, err) + } + + // Append, but only return up to the retain count + snapMeta = append(snapMeta, meta) + } + + sort.Sort(snapMetaSlice(snapMeta)) + return snapMeta, nil +} diff --git a/snapshot/state_test.go b/snapshot/state_test.go new file mode 100644 index 00000000..613820b7 --- /dev/null +++ b/snapshot/state_test.go @@ -0,0 +1,111 @@ +package snapshot + +import ( + "io" + "os" + "testing" +) + +func Test_RemoveAllTmpSnapshotData(t *testing.T) { + dir := t.TempDir() + if err := RemoveAllTmpSnapshotData(dir); err != nil { + t.Fatalf("Failed to remove all tmp snapshot data: %v", err) + } + if !pathExists(dir) { + t.Fatalf("Expected dir to exist, but it does not") + } + directories, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("Failed to read dir: %v", err) + } + if len(directories) != 0 { + t.Fatalf("Expected dir to be empty, got %d files", len(directories)) + } + + mustTouchDir(t, dir+"/dir") + mustTouchFile(t, dir+"/file") + if err := RemoveAllTmpSnapshotData(dir); err != nil { + t.Fatalf("Failed to remove all tmp snapshot data: %v", err) + } + if !pathExists(dir + "/dir") { + t.Fatalf("Expected dir to exist, but it does not") + } + if !pathExists(dir + "/file") { + t.Fatalf("Expected file to exist, but it does not") + } + + mustTouchDir(t, dir+"/snapshot1234.tmp") + mustTouchFile(t, dir+"/snapshot1234.db") + mustTouchFile(t, dir+"/snapshot1234.db-wal") + mustTouchFile(t, dir+"/snapshot1234-5678") + if err := RemoveAllTmpSnapshotData(dir); err != nil { + t.Fatalf("Failed to remove all tmp snapshot data: %v", err) + } + if !pathExists(dir + "/dir") { + t.Fatalf("Expected dir to exist, but it does not") + } + if !pathExists(dir + "/file") { + t.Fatalf("Expected file to exist, but it does not") + } + if pathExists(dir + "/snapshot1234.tmp") { + t.Fatalf("Expected snapshot1234.tmp to not exist, but it does") + } + if pathExists(dir + "/snapshot1234.db") { + t.Fatalf("Expected snapshot1234.db to not exist, but it does") + } + if pathExists(dir + "/snapshot1234.db-wal") { + t.Fatalf("Expected snapshot1234.db-wal to not exist, but it does") + } + if pathExists(dir + "/snapshot1234-5678") { + t.Fatalf("Expected /snapshot1234-5678 to not exist, but it does") + } + + mustTouchFile(t, dir+"/snapshotABCD.tmp") + if err := RemoveAllTmpSnapshotData(dir); err != nil { + t.Fatalf("Failed to remove all tmp snapshot data: %v", err) + } + if !pathExists(dir + "/snapshotABCD.tmp") { + t.Fatalf("Expected /snapshotABCD.tmp to exist, but it does not") + } +} + +func Test_LatestIndex(t *testing.T) { + store := mustStore(t) + li, err := LatestIndex(store.dir) + if err != nil { + t.Fatalf("Failed to get latest index: %v", err) + } + if li != 0 { + t.Fatalf("Expected latest index to be 0, got %d", li) + } + + sink := NewSink(store, makeRaftMeta("snap-1234", 3, 2, 1)) + if sink == nil { + t.Fatalf("Failed to create new sink") + } + if err := sink.Open(); err != nil { + t.Fatalf("Failed to open sink: %v", err) + } + + sqliteFile := mustOpenFile(t, "testdata/db-and-wals/backup.db") + defer sqliteFile.Close() + n, err := io.Copy(sink, sqliteFile) + if err != nil { + t.Fatalf("Failed to copy SQLite file: %v", err) + } + sqliteFile.Close() // Reaping will fail on Windows if file is not closed. + if n != mustGetFileSize(t, "testdata/db-and-wals/backup.db") { + t.Fatalf("Unexpected number of bytes copied: %d", n) + } + if err := sink.Close(); err != nil { + t.Fatalf("Failed to close sink: %v", err) + } + + li, err = LatestIndex(store.dir) + if err != nil { + t.Fatalf("Failed to get latest index: %v", err) + } + if li != 3 { + t.Fatalf("Expected latest index to be 3, got %d", li) + } +} diff --git a/snapshot/store.go b/snapshot/store.go index b8f1707d..6fc39096 100644 --- a/snapshot/store.go +++ b/snapshot/store.go @@ -9,7 +9,6 @@ import ( "os" "path/filepath" "runtime" - "sort" "strings" "sync" "time" @@ -310,38 +309,7 @@ func (s *Store) check() (retError error) { // getSnapshots returns a list of all snapshots in the store, sorted // from oldest to newest. func (s *Store) getSnapshots() ([]*raft.SnapshotMeta, error) { - // Get the eligible snapshots - snapshots, err := os.ReadDir(s.dir) - if err != nil { - return nil, err - } - - // Populate the metadata - var snapMeta []*raft.SnapshotMeta - for _, snap := range snapshots { - // Ignore any files - if !snap.IsDir() { - continue - } - - // Ignore any temporary snapshots - dirName := snap.Name() - if isTmpName(dirName) { - continue - } - - // Try to read the meta data - meta, err := readMeta(filepath.Join(s.dir, dirName)) - if err != nil { - return nil, fmt.Errorf("failed to read meta for snapshot %s: %s", dirName, err) - } - - // Append, but only return up to the retain count - snapMeta = append(snapMeta, meta) - } - - sort.Sort(snapMetaSlice(snapMeta)) - return snapMeta, nil + return getSnapshots(s.dir) } // getDBPath returns the path to the database file for the most recent snapshot. @@ -365,42 +333,6 @@ func (s *Store) unsetFullNeeded() error { return nil } -// RemoveAllTmpSnapshotData removes all temporary Snapshot data from the directory. -// This process is defined as follows: for every directory in dir, if the directory -// is a temporary directory, remove the directory. Then remove all other files -// that contain the name of a temporary directory, minus the temporary suffix, -// as prefix. -func RemoveAllTmpSnapshotData(dir string) error { - files, err := os.ReadDir(dir) - if err != nil { - return nil - } - for _, d := range files { - // If the directory is a temporary directory, remove it. - if d.IsDir() && isTmpName(d.Name()) { - files, err := filepath.Glob(filepath.Join(dir, nonTmpName(d.Name())) + "*") - if err != nil { - return err - } - - fullTmpDirPath := filepath.Join(dir, d.Name()) - for _, f := range files { - if f == fullTmpDirPath { - // Delete the directory last as a sign the deletion is complete. - continue - } - if err := os.Remove(f); err != nil { - return err - } - } - if err := os.RemoveAll(fullTmpDirPath); err != nil { - return err - } - } - } - return nil -} - // snapshotName generates a name for the snapshot. func snapshotName(term, index uint64) string { now := time.Now() diff --git a/snapshot/store_test.go b/snapshot/store_test.go index 2bafa17c..356560e7 100644 --- a/snapshot/store_test.go +++ b/snapshot/store_test.go @@ -39,69 +39,6 @@ func Test_SnapshotMetaSort(t *testing.T) { } } -func Test_RemoveAllTmpSnapshotData(t *testing.T) { - dir := t.TempDir() - if err := RemoveAllTmpSnapshotData(dir); err != nil { - t.Fatalf("Failed to remove all tmp snapshot data: %v", err) - } - if !pathExists(dir) { - t.Fatalf("Expected dir to exist, but it does not") - } - directories, err := os.ReadDir(dir) - if err != nil { - t.Fatalf("Failed to read dir: %v", err) - } - if len(directories) != 0 { - t.Fatalf("Expected dir to be empty, got %d files", len(directories)) - } - - mustTouchDir(t, dir+"/dir") - mustTouchFile(t, dir+"/file") - if err := RemoveAllTmpSnapshotData(dir); err != nil { - t.Fatalf("Failed to remove all tmp snapshot data: %v", err) - } - if !pathExists(dir + "/dir") { - t.Fatalf("Expected dir to exist, but it does not") - } - if !pathExists(dir + "/file") { - t.Fatalf("Expected file to exist, but it does not") - } - - mustTouchDir(t, dir+"/snapshot1234.tmp") - mustTouchFile(t, dir+"/snapshot1234.db") - mustTouchFile(t, dir+"/snapshot1234.db-wal") - mustTouchFile(t, dir+"/snapshot1234-5678") - if err := RemoveAllTmpSnapshotData(dir); err != nil { - t.Fatalf("Failed to remove all tmp snapshot data: %v", err) - } - if !pathExists(dir + "/dir") { - t.Fatalf("Expected dir to exist, but it does not") - } - if !pathExists(dir + "/file") { - t.Fatalf("Expected file to exist, but it does not") - } - if pathExists(dir + "/snapshot1234.tmp") { - t.Fatalf("Expected snapshot1234.tmp to not exist, but it does") - } - if pathExists(dir + "/snapshot1234.db") { - t.Fatalf("Expected snapshot1234.db to not exist, but it does") - } - if pathExists(dir + "/snapshot1234.db-wal") { - t.Fatalf("Expected snapshot1234.db-wal to not exist, but it does") - } - if pathExists(dir + "/snapshot1234-5678") { - t.Fatalf("Expected /snapshot1234-5678 to not exist, but it does") - } - - mustTouchFile(t, dir+"/snapshotABCD.tmp") - if err := RemoveAllTmpSnapshotData(dir); err != nil { - t.Fatalf("Failed to remove all tmp snapshot data: %v", err) - } - if !pathExists(dir + "/snapshotABCD.tmp") { - t.Fatalf("Expected /snapshotABCD.tmp to exist, but it does not") - } -} - func Test_NewStore(t *testing.T) { dir := t.TempDir() store, err := NewStore(dir)