Better testing

db/db.go

@@ -174,6 +174,43 @@ func RemoveFiles(path string) error {
 	return nil
 }
 
+// ReplayWAL replays the given WAL files into the database at the given path,
+// in the order given by the slice. The supplied WAL files are assumed to be in the same
+// directory as the database file and are removed as a result of the replay operation.
+// The "real" WAL file is also removed. If deleteMode is true, the database file is also
+// put into DELETE mode.
+func ReplayWAL(path string, wals []string, deleteMode bool) error {
+	for _, wal := range wals {
+		if err := os.Rename(wal, path+"-wal"); err != nil {
+			return fmt.Errorf("rename WAL %s: %s", wal, err.Error())
+		}
+
+		db, err := Open(path, false, true)
+		if err != nil {
+			return err
+		}
+
+		if err := db.Checkpoint(defaultCheckpointTimeout); err != nil {
+			return fmt.Errorf("checkpoint WAL %s: %s", wal, err.Error())
+		}
+
+		if err := db.Close(); err != nil {
+			return err
+		}
+	}
+
+	if deleteMode {
+		db, err := Open(path, false, false)
+		if err != nil {
+			return err
+		}
+		if db.Close(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // Open opens a file-based database, creating it if it does not exist. After this
 // function returns, an actual SQLite file will always exist.
 func Open(dbPath string, fkEnabled, wal bool) (*DB, error) {
@@ -478,7 +515,7 @@ func (db *DB) Checkpoint(dur time.Duration) (err error) {
 	var nMoved int
 
 	f := func() error {
-		err := db.rwDB.QueryRow("PRAGMA wal_checkpoint(RESTART)").Scan(&ok, &nPages, &nMoved)
+		err := db.rwDB.QueryRow("PRAGMA wal_checkpoint(TRUNCATE)").Scan(&ok, &nPages, &nMoved)
 		if err != nil {
 			return err
 		}

db/db_ondisk_test.go

@@ -4,6 +4,7 @@ import (
 	"database/sql"
 	"fmt"
 	"os"
+	"path/filepath"
 	"testing"
 	"time"
 )
@@ -226,6 +227,119 @@ func Test_DELETEDatabaseCreatedOKFromWAL(t *testing.T) {
 	}
 }
 
+// Test_WALReplay tests that WAL files are replayed as expected.
+func Test_WALReplay(t *testing.T) {
+	testFunc := func(t *testing.T, replayIntoDelete bool) {
+		dbPath := mustTempFile()
+		defer os.Remove(dbPath)
+		db, err := Open(dbPath, false, true)
+		if err != nil {
+			t.Fatalf("failed to open database in WAL mode: %s", err.Error())
+		}
+		defer db.Close()
+
+		dbFile := filepath.Base(dbPath)
+		walPath := dbPath + "-wal"
+		walFile := filepath.Base(walPath)
+
+		replayDir := mustTempDir()
+		defer os.RemoveAll(replayDir)
+		replayDBPath := filepath.Join(replayDir, dbFile)
+
+		// Take over control of checkpointing
+		if err := db.DisableCheckpointing(); err != nil {
+			t.Fatalf("failed to disable checkpointing: %s", err.Error())
+		}
+
+		// Copy the SQLite file and WAL #1
+		if _, err := db.ExecuteStringStmt("CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"); err != nil {
+			t.Fatalf("failed to create table: %s", err.Error())
+		}
+		if !fileExists(walPath) {
+			t.Fatalf("WAL file at %s does not exist", walPath)
+		}
+		mustCopyFile(replayDBPath, dbPath)
+		mustCopyFile(filepath.Join(replayDir, walFile+"_001"), walPath)
+		if err := db.Checkpoint(5 * time.Second); err != nil {
+			t.Fatalf("failed to checkpoint database in WAL mode: %s", err.Error())
+		}
+
+		// Copy WAL #2
+		_, err = db.ExecuteStringStmt(`INSERT INTO foo(name) VALUES("fiona")`)
+		if err != nil {
+			t.Fatalf("error executing insertion into table: %s", err.Error())
+		}
+		if !fileExists(walPath) {
+			t.Fatalf("WAL file at %s does not exist", walPath)
+		}
+		mustCopyFile(filepath.Join(replayDir, walFile+"_002"), walPath)
+		if err := db.Checkpoint(5 * time.Second); err != nil {
+			t.Fatalf("failed to checkpoint database in WAL mode: %s", err.Error())
+		}
+
+		// Copy WAL #3
+		_, err = db.ExecuteStringStmt(`INSERT INTO foo(name) VALUES("declan")`)
+		if err != nil {
+			t.Fatalf("error executing insertion into table: %s", err.Error())
+		}
+		if !fileExists(walPath) {
+			t.Fatalf("WAL file at %s does not exist", walPath)
+		}
+		mustCopyFile(filepath.Join(replayDir, walFile+"_003"), walPath)
+
+		if err := db.Close(); err != nil {
+			t.Fatalf("failed to close database: %s", err.Error())
+		}
+
+		wals := []string{
+			filepath.Join(replayDir, walFile+"_001"),
+			filepath.Join(replayDir, walFile+"_002"),
+			filepath.Join(replayDir, walFile+"_003"),
+		}
+		if err := ReplayWAL(replayDBPath, wals, replayIntoDelete); err != nil {
+			t.Fatalf("failed to replay WAL files: %s", err.Error())
+		}
+
+		if replayIntoDelete {
+			if !IsDELETEModeEnabledSQLiteFile(replayDBPath) {
+				t.Fatal("replayed database not marked as DELETE mode")
+			}
+		} else {
+			if !IsWALModeEnabledSQLiteFile(replayDBPath) {
+				t.Fatal("replayed database not marked as WAL mode")
+			}
+		}
+
+		// Check that there are no files ending in -wal in the replay directory
+		walFiles, err := filepath.Glob(filepath.Join(replayDir, "*-wal"))
+		if err != nil {
+			t.Fatalf("failed to glob replay directory: %s", err.Error())
+		}
+		if len(walFiles) != 0 {
+			t.Fatalf("replay directory contains WAL files: %s", walFiles)
+		}
+
+		replayedDB, err := Open(replayDBPath, false, true)
+		if err != nil {
+			t.Fatalf("failed to open replayed database: %s", err.Error())
+		}
+		rows, err := replayedDB.QueryStringStmt("SELECT * FROM foo")
+		if err != nil {
+			t.Fatalf("failed to query WAL table: %s", err.Error())
+		}
+		if exp, got := `[{"columns":["id","name"],"types":["integer","text"],"values":[[1,"fiona"],[2,"declan"]]}]`, asJSON(rows); exp != got {
+			t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
+		}
+	}
+
+	t.Run("replayIntoWAL", func(t *testing.T) {
+		testFunc(t, false)
+	})
+	t.Run("replayIntoDELETE", func(t *testing.T) {
+		testFunc(t, true)
+	})
+}
+
 func test_FileCreationOnDisk(t *testing.T, db *DB) {
 	defer db.Close()
 	if db.InMemory() {

db/db_test.go

@@ -2,6 +2,7 @@ package db
 
 import (
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"sync"
@@ -289,6 +290,45 @@ func mustTempFile() string {
 	return tmpfile.Name()
 }
 
+func mustTempDir() string {
+	tmpdir, err := ioutil.TempDir("", "rqlite-db-test")
+	if err != nil {
+		panic(err.Error())
+	}
+	return tmpdir
+}
+
+func fileExists(path string) bool {
+	_, err := os.Stat(path)
+	return err == nil
+}
+
+// function which copies a src file to a dst file, panics if any error
+func mustCopyFile(dst, src string) {
+	srcFile, err := os.Open(src)
+	if err != nil {
+		panic(err)
+	}
+	defer srcFile.Close()
+
+	dstFile, err := os.Create(dst)
+	if err != nil {
+		panic(err)
+	}
+	defer dstFile.Close()
+
+	_, err = io.Copy(dstFile, srcFile)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func mustRenameFile(oldpath, newpath string) {
+	if err := os.Rename(oldpath, newpath); err != nil {
+		panic(err)
+	}
+}
+
 func mustCreateClosedFile(path string) {
 	f, err := os.Create(path)
 	if err != nil {