fluidB
/
fluidb-old
1
0
Fork 0
Code Pull Requests Activity

Remove in-memory support from DB layer

Browse Source
master
Philip O'Toole 1 year ago
parent 88ce311b37
commit 13fce2fac2
5 changed files with 665 additions and 1060 deletions
Show Stats Download Patch File Download Diff File

207
db/db.go
Unescape Escape View File

@ -79,7 +79,6 @@ func ResetStats() {
type DB struct {
path string // Path to database file, if running on-disk.
walPath string // Path to WAL file, if running on-disk and WAL is enabled.
memory bool // In-memory only.
fkEnabled bool // Foreign key constraints enabled
wal bool
@ -354,140 +353,6 @@ func Open(dbPath string, fkEnabled, wal bool) (*DB, error) {
}, nil
}
// OpenInMemory returns a new in-memory database.
func OpenInMemory(fkEnabled bool) (*DB, error) {
inMemPath := fmt.Sprintf("file:/%s", randomString())
rwOpts := []string{
"mode=rw",
"vfs=memdb",
"_txlock=immediate",
fmt.Sprintf("_fk=%s", strconv.FormatBool(fkEnabled)),
}
rwDSN := fmt.Sprintf("%s?%s", inMemPath, strings.Join(rwOpts, "&"))
rwDB, err := sql.Open("sqlite3", rwDSN)
if err != nil {
return nil, err
}
// Ensure there is only one connection and it never closes.
// If it closed, in-memory database could be lost.
rwDB.SetConnMaxIdleTime(0)
rwDB.SetConnMaxLifetime(0)
rwDB.SetMaxIdleConns(1)
rwDB.SetMaxOpenConns(1)
roOpts := []string{
"mode=ro",
"vfs=memdb",
"_txlock=deferred",
fmt.Sprintf("_fk=%s", strconv.FormatBool(fkEnabled)),
}
roDSN := fmt.Sprintf("%s?%s", inMemPath, strings.Join(roOpts, "&"))
roDB, err := sql.Open("sqlite3", roDSN)
if err != nil {
return nil, err
}
// Ensure database is basically healthy.
if err := rwDB.Ping(); err != nil {
return nil, fmt.Errorf("failed to ping in-memory database: %s", err.Error())
}
return &DB{
memory: true,
path: ":memory:",
fkEnabled: fkEnabled,
rwDB: rwDB,
roDB: roDB,
rwDSN: rwDSN,
roDSN: roDSN,
}, nil
}
// LoadIntoMemory loads an in-memory database with that at the path.
// Not safe to call while other operations are happening with the
// source database.
func LoadIntoMemory(dbPath string, fkEnabled, wal bool) (*DB, error) {
dstDB, err := OpenInMemory(fkEnabled)
if err != nil {
return nil, err
}
srcDB, err := Open(dbPath, fkEnabled, wal)
if err != nil {
return nil, err
}
if err := copyDatabase(dstDB, srcDB); err != nil {
return nil, err
}
if err := srcDB.Close(); err != nil {
return nil, err
}
return dstDB, nil
}
// DeserializeIntoMemory loads an in-memory database with that contained
// in the byte slide. The byte slice must not be changed or garbage-collected
// until after this function returns.
func DeserializeIntoMemory(b []byte, fkEnabled bool) (retDB *DB, retErr error) {
// Get a plain-ol' in-memory database.
tmpDB, err := sql.Open("sqlite3", ":memory:")
if err != nil {
return nil, fmt.Errorf("DeserializeIntoMemory: %s", err.Error())
}
defer tmpDB.Close()
tmpConn, err := tmpDB.Conn(context.Background())
if err != nil {
return nil, err
}
// tmpDB will still be using memory in Go space, so tmpDB needs to be explicitly
// copied to a new database, which we create now.
retDB, err = OpenInMemory(fkEnabled)
if err != nil {
return nil, fmt.Errorf("DeserializeIntoMemory: %s", err.Error())
}
defer func() {
// Don't leak a database if deserialization fails.
if retDB != nil && retErr != nil {
retDB.Close()
}
}()
if err := tmpConn.Raw(func(driverConn interface{}) error {
srcConn := driverConn.(*sqlite3.SQLiteConn)
err2 := srcConn.Deserialize(b, "")
if err2 != nil {
return fmt.Errorf("DeserializeIntoMemory: %s", err2.Error())
}
defer srcConn.Close()
// Now copy from tmp database to the database this function will return.
dbConn, err3 := retDB.rwDB.Conn(context.Background())
if err3 != nil {
return fmt.Errorf("DeserializeIntoMemory: %s", err3.Error())
}
defer dbConn.Close()
return dbConn.Raw(func(driverConn interface{}) error {
dstConn := driverConn.(*sqlite3.SQLiteConn)
return copyDatabaseConnection(dstConn, srcConn)
})
}); err != nil {
return nil, err
}
return retDB, nil
}
// Close closes the underlying database connection.
func (db *DB) Close() error {
if err := db.rwDB.Close(); err != nil {
@ -530,15 +395,13 @@ func (db *DB) Stats() (map[string]interface{}, error) {
}
stats["path"] = db.path
if !db.memory {
if stats["size"], err = db.FileSize(); err != nil {
if stats["size"], err = db.FileSize(); err != nil {
return nil, err
}
if db.wal {
if stats["wal_size"], err = db.WALSize(); err != nil {
return nil, err
}
if db.wal {
if stats["wal_size"], err = db.WALSize(); err != nil {
return nil, err
}
}
}
return stats, nil
}
@ -560,9 +423,6 @@ func (db *DB) Size() (int64, error) {
// FileSize returns the size of the SQLite file on disk. If running in
// on-memory mode, this function returns 0.
func (db *DB) FileSize() (int64, error) {
if db.memory {
return 0, nil
}
fi, err := os.Stat(db.path)
if err != nil {
return 0, err
@ -648,11 +508,6 @@ func (db *DB) EnableCheckpointing() error {
return err
}
// InMemory returns whether this database is in-memory.
func (db *DB) InMemory() bool {
return db.memory
}
// FKEnabled returns whether Foreign Key constraints are enabled.
func (db *DB) FKEnabled() bool {
return db.fkEnabled
@ -1118,55 +973,35 @@ func (db *DB) Copy(dstDB *DB) error {
// Serialize returns a byte slice representation of the SQLite database. For
// an ordinary on-disk database file, the serialization is just a copy of the
// disk file. For an in-memory database or a "TEMP" database, the serialization
// is the same sequence of bytes which would be written to disk if that database
// were backed up to disk. If the database is in WAL mode, a temporary on-disk
// disk file. If the database is in WAL mode, a temporary on-disk
// copy is made, and it is this copy that is serialized. This function must not
// be called while any writes are happening to the database.
func (db *DB) Serialize() ([]byte, error) {
if !db.memory {
if db.wal {
tmpFile, err := os.CreateTemp("", "rqlite-serialize")
if err != nil {
return nil, err
}
defer os.Remove(tmpFile.Name())
defer tmpFile.Close()
if db.wal {
tmpFile, err := os.CreateTemp("", "rqlite-serialize")
if err != nil {
return nil, err
}
defer os.Remove(tmpFile.Name())
defer tmpFile.Close()
if err := db.Backup(tmpFile.Name()); err != nil {
return nil, err
}
newDB, err := Open(tmpFile.Name(), db.fkEnabled, false)
if err != nil {
return nil, err
}
defer newDB.Close()
return newDB.Serialize()
if err := db.Backup(tmpFile.Name()); err != nil {
return nil, err
}
// Simply read and return the SQLite file.
b, err := os.ReadFile(db.path)
newDB, err := Open(tmpFile.Name(), db.fkEnabled, false)
if err != nil {
return nil, err
}
return b, nil
defer newDB.Close()
return newDB.Serialize()
}
conn, err := db.roDB.Conn(context.Background())
// Simply read and return the SQLite file.
b, err := os.ReadFile(db.path)
if err != nil {
return nil, err
}
defer conn.Close()
var b []byte
if err := conn.Raw(func(raw interface{}) error {
var err error
b, err = raw.(*sqlite3.SQLiteConn).Serialize("")
return err
}); err != nil {
return nil, fmt.Errorf("failed to serialize database: %s", err.Error())
}
return b, nil
}
// Dump writes a consistent snapshot of the database in SQL text format.

6
db/db_common_test.go
Unescape Escape View File

@ -1498,11 +1498,5 @@ func Test_DatabaseCommonOperations(t *testing.T) {
t.Run(tc.name+":wal", func(t *testing.T) {
tc.testFunc(t, db)
})
db = mustCreateInMemoryDatabase()
defer db.Close()
t.Run(tc.name+":memory", func(t *testing.T) {
tc.testFunc(t, db)
})
}
}

345
db/db_inmem_test.go
Unescape Escape View File

@ -1,346 +1 @@
package db
import (
"fmt"
"io/ioutil"
"os"
"sync"
"testing"
"time"
"github.com/rqlite/rqlite/command"
text "github.com/rqlite/rqlite/db/testdata"
)
// Test_TableCreationInMemory tests basic operation of an in-memory database,
// ensuring that using different connection objects (as the Execute and Query
// will do) works properly i.e. that the connections object work on the same
// in-memory database.
func Test_TableCreationInMemory(t *testing.T) {
db := mustCreateInMemoryDatabase()
defer db.Close()
if !db.InMemory() {
t.Fatal("in-memory database marked as not in-memory")
}
r, err := db.ExecuteStringStmt("CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)")
if err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
if exp, got := `[{}]`, asJSON(r); exp != got {
t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
}
q, err := db.QueryStringStmt("SELECT * FROM foo")
if err != nil {
t.Fatalf("failed to query empty table: %s", err.Error())
}
if exp, got := `[{"columns":["id","name"],"types":["integer","text"]}]`, asJSON(q); exp != got {
t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
}
// Confirm checkpoint works without error on an in-memory database. It should just be ignored.
if err := db.Checkpoint(5 * time.Second); err != nil {
t.Fatalf("failed to checkpoint in-memory database: %s", err.Error())
}
}
func Test_LoadIntoMemory(t *testing.T) {
db, path := mustCreateOnDiskDatabase()
defer db.Close()
defer os.Remove(path)
_, err := db.ExecuteStringStmt("CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)")
if err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
r, err := db.QueryStringStmt("SELECT * FROM foo")
if err != nil {
t.Fatalf("failed to query empty table: %s", err.Error())
}
if exp, got := `[{"columns":["id","name"],"types":["integer","text"]}]`, asJSON(r); exp != got {
t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
}
inmem, err := LoadIntoMemory(path, false, false)
if err != nil {
t.Fatalf("failed to create loaded in-memory database: %s", err.Error())
}
// Ensure it has been loaded correctly into the database
r, err = inmem.QueryStringStmt("SELECT * FROM foo")
if err != nil {
t.Fatalf("failed to query empty table: %s", err.Error())
}
if exp, got := `[{"columns":["id","name"],"types":["integer","text"]}]`, asJSON(r); exp != got {
t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
}
}
func Test_DeserializeIntoMemory(t *testing.T) {
db, path := mustCreateOnDiskDatabase()
defer db.Close()
defer os.Remove(path)
_, err := db.ExecuteStringStmt("CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)")
if err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
req := &command.Request{
Transaction: true,
Statements: []*command.Statement{
{
Sql: `INSERT INTO foo(id, name) VALUES(1, "fiona")`,
},
{
Sql: `INSERT INTO foo(id, name) VALUES(2, "fiona")`,
},
{
Sql: `INSERT INTO foo(id, name) VALUES(3, "fiona")`,
},
{
Sql: `INSERT INTO foo(id, name) VALUES(4, "fiona")`,
},
},
}
_, err = db.Execute(req, false)
if err != nil {
t.Fatalf("failed to insert records: %s", err.Error())
}
// Get byte representation of database on disk which, according to SQLite docs
// is the same as a serialized version.
b, err := ioutil.ReadFile(path)
if err != nil {
t.Fatalf("failed to read database on disk: %s", err.Error())
}
newDB, err := DeserializeIntoMemory(b, false)
if err != nil {
t.Fatalf("failed to deserialize database: %s", err.Error())
}
defer newDB.Close()
ro, err := newDB.QueryStringStmt(`SELECT * FROM foo`)
if err != nil {
t.Fatalf("failed to query table: %s", err.Error())
}
if exp, got := `[{"columns":["id","name"],"types":["integer","text"],"values":[[1,"fiona"],[2,"fiona"],[3,"fiona"],[4,"fiona"]]}]`, asJSON(ro); exp != got {
t.Fatalf("unexpected results for query\nexp: %s\ngot: %s", exp, got)
}
// Write a lot of records to the new database, to ensure it's fully functional.
req = &command.Request{
Statements: []*command.Statement{
{
Sql: `INSERT INTO foo(name) VALUES("fiona")`,
},
},
}
for i := 0; i < 5000; i++ {
_, err = newDB.Execute(req, false)
if err != nil {
t.Fatalf("failed to insert records: %s", err.Error())
}
}
ro, err = newDB.QueryStringStmt(`SELECT COUNT(*) FROM foo`)
if err != nil {
t.Fatalf("failed to query table: %s", err.Error())
}
if exp, got := `[{"columns":["COUNT(*)"],"types":["integer"],"values":[[5004]]}]`, asJSON(ro); exp != got {
t.Fatalf("unexpected results for query\nexp: %s\ngot: %s", exp, got)
}
}
// Test_ParallelOperationsInMemory runs multiple accesses concurrently, ensuring
// that correct results are returned in every goroutine. It's not 100% that this
// test would bring out a bug, but it's almost 100%.
//
// See https://github.com/mattn/go-sqlite3/issues/959#issuecomment-890283264
func Test_ParallelOperationsInMemory(t *testing.T) {
db := mustCreateInMemoryDatabase()
defer db.Close()
if _, err := db.ExecuteStringStmt("CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"); err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
if _, err := db.ExecuteStringStmt("CREATE TABLE bar (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"); err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
if _, err := db.ExecuteStringStmt("CREATE TABLE qux (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"); err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
// Confirm schema is as expected, when checked from same goroutine.
if rows, err := db.QueryStringStmt(`SELECT sql FROM sqlite_master`); err != nil {
t.Fatalf("failed to query for schema after creation: %s", err.Error())
} else {
if exp, got := `[{"columns":["sql"],"types":["text"],"values":[["CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"],["CREATE TABLE bar (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"],["CREATE TABLE qux (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"]]}]`, asJSON(rows); exp != got {
t.Fatalf("schema not as expected during after creation, exp %s, got %s", exp, got)
}
}
var exWg sync.WaitGroup
exWg.Add(3)
foo := make(chan time.Time)
bar := make(chan time.Time)
qux := make(chan time.Time)
done := make(chan bool)
ticker := time.NewTicker(1 * time.Millisecond)
go func() {
for {
select {
case t := <-ticker.C:
foo <- t
bar <- t
qux <- t
case <-done:
close(foo)
close(bar)
close(qux)
return
}
}
}()
go func() {
defer exWg.Done()
for range foo {
if _, err := db.ExecuteStringStmt(`INSERT INTO foo(id, name) VALUES(1, "fiona")`); err != nil {
t.Logf("failed to insert records into foo: %s", err.Error())
}
}
}()
go func() {
defer exWg.Done()
for range bar {
if _, err := db.ExecuteStringStmt(`INSERT INTO bar(id, name) VALUES(1, "fiona")`); err != nil {
t.Logf("failed to insert records into bar: %s", err.Error())
}
}
}()
go func() {
defer exWg.Done()
for range qux {
if _, err := db.ExecuteStringStmt(`INSERT INTO qux(id, name) VALUES(1, "fiona")`); err != nil {
t.Logf("failed to insert records into qux: %s", err.Error())
}
}
}()
var qWg sync.WaitGroup
qWg.Add(3)
for i := 0; i < 3; i++ {
go func(j int) {
defer qWg.Done()
var n int
for {
if rows, err := db.QueryStringStmt(`SELECT sql FROM sqlite_master`); err != nil {
t.Logf("failed to query for schema during goroutine %d execution: %s", j, err.Error())
} else {
n++
if exp, got := `[{"columns":["sql"],"types":["text"],"values":[["CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"],["CREATE TABLE bar (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"],["CREATE TABLE qux (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"]]}]`, asJSON(rows); exp != got {
t.Logf("schema not as expected during goroutine execution, exp %s, got %s, after %d queries", exp, got, n)
}
}
if n == 500000 {
break
}
}
}(i)
}
qWg.Wait()
close(done)
exWg.Wait()
}
// Test_TableCreationLoadRawInMemory tests for https://sqlite.org/forum/forumpost/d443fb0730
func Test_TableCreationLoadRawInMemory(t *testing.T) {
db := mustCreateInMemoryDatabase()
defer db.Close()
_, err := db.ExecuteStringStmt("CREATE TABLE logs (entry TEXT)")
if err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
done := make(chan struct{})
defer close(done)
// Insert some records continually, as fast as possible. Do it from a goroutine.
go func() {
for {
select {
case <-done:
return
default:
_, err := db.ExecuteStringStmt(`INSERT INTO logs(entry) VALUES("hello")`)
if err != nil {
return
}
}
}
}()
// Get the count over and over again.
for i := 0; i < 5000; i++ {
rows, err := db.QueryStringStmt(`SELECT COUNT(*) FROM logs`)
if err != nil {
t.Fatalf("failed to query for count: %s", err)
}
if rows[0].Error != "" {
t.Fatalf("rows had error after %d queries: %s", i, rows[0].Error)
}
}
}
// Test_1GiBInMemory tests that in-memory databases larger than 1GiB,
// but smaller than 2GiB, can be created without issue.
func Test_1GiBInMemory(t *testing.T) {
db := mustCreateInMemoryDatabase()
defer db.Close()
_, err := db.ExecuteStringStmt("CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, txt TEXT)")
if err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
stmt := fmt.Sprintf(`INSERT INTO foo(txt) VALUES("%s")`, text.Lorum)
for i := 0; i < 1715017; i++ {
r, err := db.ExecuteStringStmt(stmt)
if err != nil {
t.Fatalf("failed to Execute statement %s", err.Error())
}
if len(r) != 1 {
t.Fatalf("unexpected length for Execute results: %d", len(r))
}
if r[0].GetError() != "" {
t.Fatalf("failed to insert record: %s", r[0].GetError())
}
}
r, err := db.ExecuteStringStmt(stmt)
if err != nil {
t.Fatalf("failed to insert record %s", err.Error())
}
if exp, got := `[{"last_insert_id":1715018,"rows_affected":1}]`, asJSON(r); exp != got {
t.Fatalf("got incorrect response, exp: %s, got: %s", exp, got)
}
sz, err := db.Size()
if err != nil {
t.Fatalf("failed to get size: %s", err.Error())
}
if sz <= 1024*1024*1024 {
t.Fatalf("failed to create a database greater than 1 GiB in size: %d", sz)
}
}

501
db/db_ondisk_test.go
Unescape Escape View File

@ -1,502 +1 @@
package db
import (
"database/sql"
"fmt"
"os"
"path/filepath"
"testing"
"time"
)
func Test_IsValidSQLiteOnDisk(t *testing.T) {
path := mustTempFile()
defer os.Remove(path)
dsn := fmt.Sprintf("file:%s", path)
db, err := sql.Open("sqlite3", dsn)
if err != nil {
t.Fatalf("failed to create SQLite database: %s", err.Error())
}
_, err = db.Exec("CREATE TABLE foo (name TEXT)")
if err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
if err := db.Close(); err != nil {
t.Fatalf("failed to close database: %s", err.Error())
}
if !IsValidSQLiteFile(path) {
t.Fatalf("good SQLite file marked as invalid")
}
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("failed to read SQLite file: %s", err.Error())
}
if !IsValidSQLiteData(data) {
t.Fatalf("good SQLite data marked as invalid")
}
}
func Test_CheckIntegrityOnDisk(t *testing.T) {
path := mustTempFile()
defer os.Remove(path)
dsn := fmt.Sprintf("file:%s", path)
db, err := sql.Open("sqlite3", dsn)
if err != nil {
t.Fatalf("failed to create SQLite database: %s", err.Error())
}
_, err = db.Exec("CREATE TABLE foo (name TEXT)")
if err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
if err := db.Close(); err != nil {
t.Fatalf("failed to close database: %s", err.Error())
}
for _, check := range []bool{true, false} {
ok, err := CheckIntegrity(path, check)
if err != nil {
t.Fatalf("failed to check integrity of database (full=%t): %s", check, err.Error())
}
if !ok {
t.Fatalf("database failed integrity check (full=%t)", check)
}
}
sz := int(mustFileSize(path))
b, err := os.ReadFile(path)
if err != nil {
t.Fatalf("failed to read SQLite file: %s", err.Error())
}
// Chop the file in half, integrity check shouldn't even be error-free
if err := os.WriteFile(path, b[:len(b)-sz/2], 0644); err != nil {
t.Fatalf("failed to write SQLite file: %s", err.Error())
}
for _, check := range []bool{true, false} {
_, err := CheckIntegrity(path, check)
if err == nil {
t.Fatalf("succeeded checking integrity of database (full=%t): %s", check, err.Error())
}
}
// Unable to create a data set that actually fails integrity check.
}
func Test_IsWALModeEnabledOnDiskDELETE(t *testing.T) {
path := mustTempFile()
defer os.Remove(path)
dsn := fmt.Sprintf("file:%s", path)
db, err := sql.Open("sqlite3", dsn)
if err != nil {
t.Fatalf("failed to create SQLite database: %s", err.Error())
}
_, err = db.Exec("CREATE TABLE foo (name TEXT)")
if err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
if err := db.Close(); err != nil {
t.Fatalf("failed to close database: %s", err.Error())
}
if !IsDELETEModeEnabledSQLiteFile(path) {
t.Fatalf("DELETE file marked as non-DELETE")
}
if IsWALModeEnabledSQLiteFile(path) {
t.Fatalf("non WAL file marked as WAL")
}
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("failed to read SQLite file: %s", err.Error())
}
if IsWALModeEnabled(data) {
t.Fatalf("non WAL data marked as WAL")
}
if !IsDELETEModeEnabled(data) {
t.Fatalf("data marked as non-DELETE")
}
}
func Test_IsWALModeEnabledOnDiskWAL(t *testing.T) {
path := mustTempFile()
defer os.Remove(path)
dsn := fmt.Sprintf("file:%s", path)
db, err := sql.Open("sqlite3", dsn)
if err != nil {
t.Fatalf("failed to create SQLite database: %s", err.Error())
}
_, err = db.Exec("CREATE TABLE foo (name TEXT)")
if err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
_, err = db.Exec("PRAGMA journal_mode=WAL")
if err != nil {
t.Fatalf("failed to enable WAL mode: %s", err.Error())
}
_, err = db.Exec(`INSERT INTO foo(name) VALUES("fiona")`)
if err != nil {
t.Fatalf("error inserting record into table: %s", err.Error())
}
if !IsValidSQLiteWALFile(path + "-wal") {
t.Fatalf("WAL file marked not marked as valid")
}
if err := db.Close(); err != nil {
t.Fatalf("failed to close database: %s", err.Error())
}
if !IsWALModeEnabledSQLiteFile(path) {
t.Fatalf("WAL file marked as non-WAL")
}
if IsDELETEModeEnabledSQLiteFile(path) {
t.Fatalf("WAL file marked as DELETE")
}
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("failed to read SQLite file: %s", err.Error())
}
if !IsWALModeEnabled(data) {
t.Fatalf("WAL data marked as non-WAL")
}
if IsDELETEModeEnabled(data) {
t.Fatalf("WAL data marked as DELETE")
}
}
// Test_WALDatabaseCreatedOK tests that a WAL file is created, and that
// a checkpoint succeeds
func Test_WALDatabaseCreatedOK(t *testing.T) {
path := mustTempFile()
defer os.Remove(path)
db, err := Open(path, false, true)
if err != nil {
t.Fatalf("failed to open database in WAL mode: %s", err.Error())
}
defer db.Close()
if !db.WALEnabled() {
t.Fatalf("WAL mode not enabled")
}
if db.InMemory() {
t.Fatalf("on-disk WAL database marked as in-memory")
}
if _, err := db.ExecuteStringStmt("CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"); err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
if !IsWALModeEnabledSQLiteFile(path) {
t.Fatalf("SQLite file not marked as WAL")
}
walPath := path + "-wal"
if _, err := os.Stat(walPath); os.IsNotExist(err) {
t.Fatalf("WAL file does not exist")
}
if err := db.Checkpoint(5 * time.Second); err != nil {
t.Fatalf("failed to checkpoint database in WAL mode: %s", err.Error())
}
}
// Test_WALDatabaseCreatedOKFromDELETE tests that a WAL database is created properly,
// even when supplied with a DELETE-mode database.
func Test_WALDatabaseCreatedOKFromDELETE(t *testing.T) {
deletePath := mustTempFile()
defer os.Remove(deletePath)
deleteDB, err := Open(deletePath, false, false)
if err != nil {
t.Fatalf("failed to open database in WAL mode: %s", err.Error())
}
defer deleteDB.Close()
if _, err := deleteDB.ExecuteStringStmt("CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"); err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
_, err = deleteDB.ExecuteStringStmt(`INSERT INTO foo(name) VALUES("fiona")`)
if err != nil {
t.Fatalf("error executing insertion into table: %s", err.Error())
}
walDB, err := Open(deletePath, false, true)
if err != nil {
t.Fatalf("failed to open database in WAL mode: %s", err.Error())
}
defer walDB.Close()
if !IsWALModeEnabledSQLiteFile(deletePath) {
t.Fatalf("SQLite file not marked as WAL")
}
rows, err := walDB.QueryStringStmt("SELECT * FROM foo")
if err != nil {
t.Fatalf("failed to query WAL table: %s", err.Error())
}
if exp, got := `[{"columns":["id","name"],"types":["integer","text"],"values":[[1,"fiona"]]}]`, asJSON(rows); exp != got {
t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
}
}
// Test_DELETEDatabaseCreatedOKFromWAL tests that a DELETE database is created properly,
// even when supplied with a WAL-mode database.
func Test_DELETEDatabaseCreatedOKFromWAL(t *testing.T) {
walPath := mustTempFile()
defer os.Remove(walPath)
walDB, err := Open(walPath, false, true)
if err != nil {
t.Fatalf("failed to open database in WAL mode: %s", err.Error())
}
defer walDB.Close()
if _, err := walDB.ExecuteStringStmt("CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"); err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
_, err = walDB.ExecuteStringStmt(`INSERT INTO foo(name) VALUES("fiona")`)
if err != nil {
t.Fatalf("error executing insertion into table: %s", err.Error())
}
if err := walDB.Close(); err != nil {
// Closing the WAL database is required if it's to be opened in DELETE mode.
t.Fatalf("failed to close database: %s", err.Error())
}
deleteDB, err2 := Open(walPath, false, false)
if err2 != nil {
t.Fatalf("failed to open database in DELETE mode: %s", err2.Error())
}
defer deleteDB.Close()
if !IsDELETEModeEnabledSQLiteFile(walPath) {
t.Fatalf("SQLite file not marked as WAL")
}
rows, err := deleteDB.QueryStringStmt("SELECT * FROM foo")
if err != nil {
t.Fatalf("failed to query WAL table: %s", err.Error())
}
if exp, got := `[{"columns":["id","name"],"types":["integer","text"],"values":[[1,"fiona"]]}]`, asJSON(rows); exp != got {
t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
}
}
// Test_WALReplayOK tests that WAL files are replayed as expected.
func Test_WALReplayOK(t *testing.T) {
testFunc := func(t *testing.T, replayIntoDelete bool) {
dbPath := mustTempFile()
defer os.Remove(dbPath)
db, err := Open(dbPath, false, true)
if err != nil {
t.Fatalf("failed to open database in WAL mode: %s", err.Error())
}
defer db.Close()
dbFile := filepath.Base(dbPath)
walPath := dbPath + "-wal"
walFile := filepath.Base(walPath)
replayDir := mustTempDir()
defer os.RemoveAll(replayDir)
replayDBPath := filepath.Join(replayDir, dbFile)
// Take over control of checkpointing
if err := db.DisableCheckpointing(); err != nil {
t.Fatalf("failed to disable checkpointing: %s", err.Error())
}
// Copy the SQLite file and WAL #1
if _, err := db.ExecuteStringStmt("CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"); err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
if !fileExists(walPath) {
t.Fatalf("WAL file at %s does not exist", walPath)
}
mustCopyFile(replayDBPath, dbPath)
mustCopyFile(filepath.Join(replayDir, walFile+"_001"), walPath)
if err := db.Checkpoint(5 * time.Second); err != nil {
t.Fatalf("failed to checkpoint database in WAL mode: %s", err.Error())
}
// Copy WAL #2
_, err = db.ExecuteStringStmt(`INSERT INTO foo(name) VALUES("fiona")`)
if err != nil {
t.Fatalf("error executing insertion into table: %s", err.Error())
}
if !fileExists(walPath) {
t.Fatalf("WAL file at %s does not exist", walPath)
}
mustCopyFile(filepath.Join(replayDir, walFile+"_002"), walPath)
if err := db.Checkpoint(5 * time.Second); err != nil {
t.Fatalf("failed to checkpoint database in WAL mode: %s", err.Error())
}
// Copy WAL #3
_, err = db.ExecuteStringStmt(`INSERT INTO foo(name) VALUES("declan")`)
if err != nil {
t.Fatalf("error executing insertion into table: %s", err.Error())
}
if !fileExists(walPath) {
t.Fatalf("WAL file at %s does not exist", walPath)
}
mustCopyFile(filepath.Join(replayDir, walFile+"_003"), walPath)
if err := db.Close(); err != nil {
t.Fatalf("failed to close database: %s", err.Error())
}
wals := []string{
filepath.Join(replayDir, walFile+"_001"),
filepath.Join(replayDir, walFile+"_002"),
filepath.Join(replayDir, walFile+"_003"),
}
if err := ReplayWAL(replayDBPath, wals, replayIntoDelete); err != nil {
t.Fatalf("failed to replay WAL files: %s", err.Error())
}
if replayIntoDelete {
if !IsDELETEModeEnabledSQLiteFile(replayDBPath) {
t.Fatal("replayed database not marked as DELETE mode")
}
} else {
if !IsWALModeEnabledSQLiteFile(replayDBPath) {
t.Fatal("replayed database not marked as WAL mode")
}
}
// Check that there are no files related to WALs in the replay directory
// Both the copied WAL files should be gone, and there should be no
// "real" WAL file either.
walFiles, err := filepath.Glob(filepath.Join(replayDir, "*-wal*"))
if err != nil {
t.Fatalf("failed to glob replay directory: %s", err.Error())
}
if len(walFiles) != 0 {
t.Fatalf("replay directory contains WAL files: %s", walFiles)
}
replayedDB, err := Open(replayDBPath, false, true)
if err != nil {
t.Fatalf("failed to open replayed database: %s", err.Error())
}
rows, err := replayedDB.QueryStringStmt("SELECT * FROM foo")
if err != nil {
t.Fatalf("failed to query WAL table: %s", err.Error())
}
if exp, got := `[{"columns":["id","name"],"types":["integer","text"],"values":[[1,"fiona"],[2,"declan"]]}]`, asJSON(rows); exp != got {
t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
}
}
t.Run("replayIntoWAL", func(t *testing.T) {
testFunc(t, false)
})
t.Run("replayIntoDELETE", func(t *testing.T) {
testFunc(t, true)
})
}
func Test_WALReplayFailures(t *testing.T) {
dbDir := mustTempDir()
defer os.RemoveAll(dbDir)
walDir := mustTempDir()
defer os.RemoveAll(walDir)
err := ReplayWAL(filepath.Join(dbDir, "foo.db"), []string{filepath.Join(walDir, "foo.db-wal")}, false)
if err != ErrWALReplayDirectoryMismatch {
t.Fatalf("expected %s, got %s", ErrWALReplayDirectoryMismatch, err.Error())
}
}
func test_FileCreationOnDisk(t *testing.T, db *DB) {
defer db.Close()
if db.InMemory() {
t.Fatal("on-disk database marked as in-memory")
}
if db.FKEnabled() {
t.Fatal("FK constraints marked as enabled")
}
// Confirm checkpoint works on all types of on-disk databases. Worst case, this
// should be ignored.
if err := db.Checkpoint(5 * time.Second); err != nil {
t.Fatalf("failed to checkpoint database in DELETE mode: %s", err.Error())
}
}
// test_ConnectionIsolationOnDisk test that ISOLATION behavior of on-disk databases doesn't
// change unexpectedly.
func test_ConnectionIsolationOnDisk(t *testing.T, db *DB) {
r, err := db.ExecuteStringStmt("CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)")
if err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
if exp, got := `[{}]`, asJSON(r); exp != got {
t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
}
r, err = db.ExecuteStringStmt("BEGIN")
if err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
if exp, got := `[{}]`, asJSON(r); exp != got {
t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
}
r, err = db.ExecuteStringStmt(`INSERT INTO foo(name) VALUES("fiona")`)
if err != nil {
t.Fatalf("error executing insertion into table: %s", err.Error())
}
if exp, got := `[{"last_insert_id":1,"rows_affected":1}]`, asJSON(r); exp != got {
t.Fatalf("unexpected results for execute, expected %s, got %s", exp, got)
}
q, err := db.QueryStringStmt("SELECT * FROM foo")
if err != nil {
t.Fatalf("failed to query empty table: %s", err.Error())
}
if exp, got := `[{"columns":["id","name"],"types":["integer","text"]}]`, asJSON(q); exp != got {
t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
}
_, err = db.ExecuteStringStmt("COMMIT")
if err != nil {
t.Fatalf("error executing insertion into table: %s", err.Error())
}
q, err = db.QueryStringStmt("SELECT * FROM foo")
if err != nil {
t.Fatalf("failed to query empty table: %s", err.Error())
}
if exp, got := `[{"columns":["id","name"],"types":["integer","text"],"values":[[1,"fiona"]]}]`, asJSON(q); exp != got {
t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
}
}
func Test_DatabaseCommonOnDiskOperations(t *testing.T) {
testCases := []struct {
name string
testFunc func(*testing.T, *DB)
}{
{"FileCreationOnDisk", test_FileCreationOnDisk},
{"ConnectionIsolationOnDisk", test_ConnectionIsolationOnDisk},
}
for _, tc := range testCases {
db, path := mustCreateOnDiskDatabase()
defer db.Close()
defer os.Remove(path)
t.Run(tc.name+":disk", func(t *testing.T) {
tc.testFunc(t, db)
})
db, path = mustCreateOnDiskDatabaseWAL()
defer db.Close()
defer os.Remove(path)
t.Run(tc.name+":wal", func(t *testing.T) {
tc.testFunc(t, db)
})
}
}

666
db/db_test.go
Unescape Escape View File

@ -1,11 +1,13 @@
package db
import (
"database/sql"
"fmt"
"io"
"io/ioutil"
"math/rand"
"os"
"path/filepath"
"sync"
"testing"
"time"
@ -32,14 +34,46 @@ func Test_RemoveFiles(t *testing.T) {
}
}
// Test_TableCreationInMemoryFK ensures foreign key constraints work
func Test_TableCreationInMemoryFK(t *testing.T) {
// Test_TableCreation tests basic operation of an in-memory database,
// ensuring that using different connection objects (as the Execute and Query
// will do) works properly i.e. that the connections object work on the same
// in-memory database.
func Test_TableCreation(t *testing.T) {
db, path := mustCreateOnDiskDatabaseWAL()
defer db.Close()
defer os.Remove(path)
r, err := db.ExecuteStringStmt("CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)")
if err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
if exp, got := `[{}]`, asJSON(r); exp != got {
t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
}
q, err := db.QueryStringStmt("SELECT * FROM foo")
if err != nil {
t.Fatalf("failed to query empty table: %s", err.Error())
}
if exp, got := `[{"columns":["id","name"],"types":["integer","text"]}]`, asJSON(q); exp != got {
t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
}
// Confirm checkpoint works without error on an in-memory database. It should just be ignored.
if err := db.Checkpoint(5 * time.Second); err != nil {
t.Fatalf("failed to checkpoint in-memory database: %s", err.Error())
}
}
// Test_TableCreationFK ensures foreign key constraints work
func Test_TableCreationFK(t *testing.T) {
createTableFoo := "CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"
createTableBar := "CREATE TABLE bar (fooid INTEGER NOT NULL PRIMARY KEY, FOREIGN KEY(fooid) REFERENCES foo(id))"
insertIntoBar := "INSERT INTO bar(fooid) VALUES(1)"
db := mustCreateInMemoryDatabase()
db, path := mustCreateOnDiskDatabaseWAL()
defer db.Close()
defer os.Remove(path)
r, err := db.ExecuteStringStmt(createTableFoo)
if err != nil {
@ -66,8 +100,10 @@ func Test_TableCreationInMemoryFK(t *testing.T) {
}
// Now, do same testing with FK constraints enabled.
dbFK := mustCreateInMemoryDatabaseFK()
dbFK, path := mustCreateOnDiskDatabaseWALFK()
defer dbFK.Close()
defer os.Remove(path)
if !dbFK.FKEnabled() {
t.Fatal("FK constraints not marked as enabled")
}
@ -97,9 +133,10 @@ func Test_TableCreationInMemoryFK(t *testing.T) {
}
}
func Test_ConcurrentQueriesInMemory(t *testing.T) {
db := mustCreateInMemoryDatabase()
func Test_ConcurrentQueries(t *testing.T) {
db, path := mustCreateOnDiskDatabaseWAL()
defer db.Close()
defer os.Remove(path)
r, err := db.ExecuteStringStmt(`CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)`)
if err != nil {
@ -137,7 +174,7 @@ func Test_ConcurrentQueriesInMemory(t *testing.T) {
}
func Test_SimpleTransaction(t *testing.T) {
db, path := mustCreateOnDiskDatabase()
db, path := mustCreateOnDiskDatabaseWAL()
defer db.Close()
defer os.Remove(path)
@ -180,7 +217,7 @@ func Test_SimpleTransaction(t *testing.T) {
}
func Test_PartialFailTransaction(t *testing.T) {
db, path := mustCreateOnDiskDatabase()
db, path := mustCreateOnDiskDatabaseWAL()
defer db.Close()
defer os.Remove(path)
@ -222,6 +259,599 @@ func Test_PartialFailTransaction(t *testing.T) {
}
}
func Test_IsValidSQLiteOnDisk(t *testing.T) {
path := mustTempFile()
defer os.Remove(path)
dsn := fmt.Sprintf("file:%s", path)
db, err := sql.Open("sqlite3", dsn)
if err != nil {
t.Fatalf("failed to create SQLite database: %s", err.Error())
}
_, err = db.Exec("CREATE TABLE foo (name TEXT)")
if err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
if err := db.Close(); err != nil {
t.Fatalf("failed to close database: %s", err.Error())
}
if !IsValidSQLiteFile(path) {
t.Fatalf("good SQLite file marked as invalid")
}
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("failed to read SQLite file: %s", err.Error())
}
if !IsValidSQLiteData(data) {
t.Fatalf("good SQLite data marked as invalid")
}
}
func Test_CheckIntegrityOnDisk(t *testing.T) {
path := mustTempFile()
defer os.Remove(path)
dsn := fmt.Sprintf("file:%s", path)
db, err := sql.Open("sqlite3", dsn)
if err != nil {
t.Fatalf("failed to create SQLite database: %s", err.Error())
}
_, err = db.Exec("CREATE TABLE foo (name TEXT)")
if err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
if err := db.Close(); err != nil {
t.Fatalf("failed to close database: %s", err.Error())
}
for _, check := range []bool{true, false} {
ok, err := CheckIntegrity(path, check)
if err != nil {
t.Fatalf("failed to check integrity of database (full=%t): %s", check, err.Error())
}
if !ok {
t.Fatalf("database failed integrity check (full=%t)", check)
}
}
sz := int(mustFileSize(path))
b, err := os.ReadFile(path)
if err != nil {
t.Fatalf("failed to read SQLite file: %s", err.Error())
}
// Chop the file in half, integrity check shouldn't even be error-free
if err := os.WriteFile(path, b[:len(b)-sz/2], 0644); err != nil {
t.Fatalf("failed to write SQLite file: %s", err.Error())
}
for _, check := range []bool{true, false} {
_, err := CheckIntegrity(path, check)
if err == nil {
t.Fatalf("succeeded checking integrity of database (full=%t): %s", check, err.Error())
}
}
// Unable to create a data set that actually fails integrity check.
}
func Test_IsWALModeEnabledOnDiskDELETE(t *testing.T) {
path := mustTempFile()
defer os.Remove(path)
dsn := fmt.Sprintf("file:%s", path)
db, err := sql.Open("sqlite3", dsn)
if err != nil {
t.Fatalf("failed to create SQLite database: %s", err.Error())
}
_, err = db.Exec("CREATE TABLE foo (name TEXT)")
if err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
if err := db.Close(); err != nil {
t.Fatalf("failed to close database: %s", err.Error())
}
if !IsDELETEModeEnabledSQLiteFile(path) {
t.Fatalf("DELETE file marked as non-DELETE")
}
if IsWALModeEnabledSQLiteFile(path) {
t.Fatalf("non WAL file marked as WAL")
}
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("failed to read SQLite file: %s", err.Error())
}
if IsWALModeEnabled(data) {
t.Fatalf("non WAL data marked as WAL")
}
if !IsDELETEModeEnabled(data) {
t.Fatalf("data marked as non-DELETE")
}
}
func Test_IsWALModeEnabledOnDiskWAL(t *testing.T) {
path := mustTempFile()
defer os.Remove(path)
dsn := fmt.Sprintf("file:%s", path)
db, err := sql.Open("sqlite3", dsn)
if err != nil {
t.Fatalf("failed to create SQLite database: %s", err.Error())
}
_, err = db.Exec("CREATE TABLE foo (name TEXT)")
if err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
_, err = db.Exec("PRAGMA journal_mode=WAL")
if err != nil {
t.Fatalf("failed to enable WAL mode: %s", err.Error())
}
_, err = db.Exec(`INSERT INTO foo(name) VALUES("fiona")`)
if err != nil {
t.Fatalf("error inserting record into table: %s", err.Error())
}
if !IsValidSQLiteWALFile(path + "-wal") {
t.Fatalf("WAL file marked not marked as valid")
}
if err := db.Close(); err != nil {
t.Fatalf("failed to close database: %s", err.Error())
}
if !IsWALModeEnabledSQLiteFile(path) {
t.Fatalf("WAL file marked as non-WAL")
}
if IsDELETEModeEnabledSQLiteFile(path) {
t.Fatalf("WAL file marked as DELETE")
}
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("failed to read SQLite file: %s", err.Error())
}
if !IsWALModeEnabled(data) {
t.Fatalf("WAL data marked as non-WAL")
}
if IsDELETEModeEnabled(data) {
t.Fatalf("WAL data marked as DELETE")
}
}
// Test_WALDatabaseCreatedOK tests that a WAL file is created, and that
// a checkpoint succeeds
func Test_WALDatabaseCreatedOK(t *testing.T) {
path := mustTempFile()
defer os.Remove(path)
db, err := Open(path, false, true)
if err != nil {
t.Fatalf("failed to open database in WAL mode: %s", err.Error())
}
defer db.Close()
if !db.WALEnabled() {
t.Fatalf("WAL mode not enabled")
}
if _, err := db.ExecuteStringStmt("CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"); err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
if !IsWALModeEnabledSQLiteFile(path) {
t.Fatalf("SQLite file not marked as WAL")
}
walPath := path + "-wal"
if _, err := os.Stat(walPath); os.IsNotExist(err) {
t.Fatalf("WAL file does not exist")
}
if err := db.Checkpoint(5 * time.Second); err != nil {
t.Fatalf("failed to checkpoint database in WAL mode: %s", err.Error())
}
}
// Test_WALDatabaseCreatedOKFromDELETE tests that a WAL database is created properly,
// even when supplied with a DELETE-mode database.
func Test_WALDatabaseCreatedOKFromDELETE(t *testing.T) {
deletePath := mustTempFile()
defer os.Remove(deletePath)
deleteDB, err := Open(deletePath, false, false)
if err != nil {
t.Fatalf("failed to open database in WAL mode: %s", err.Error())
}
defer deleteDB.Close()
if _, err := deleteDB.ExecuteStringStmt("CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"); err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
_, err = deleteDB.ExecuteStringStmt(`INSERT INTO foo(name) VALUES("fiona")`)
if err != nil {
t.Fatalf("error executing insertion into table: %s", err.Error())
}
walDB, err := Open(deletePath, false, true)
if err != nil {
t.Fatalf("failed to open database in WAL mode: %s", err.Error())
}
defer walDB.Close()
if !IsWALModeEnabledSQLiteFile(deletePath) {
t.Fatalf("SQLite file not marked as WAL")
}
rows, err := walDB.QueryStringStmt("SELECT * FROM foo")
if err != nil {
t.Fatalf("failed to query WAL table: %s", err.Error())
}
if exp, got := `[{"columns":["id","name"],"types":["integer","text"],"values":[[1,"fiona"]]}]`, asJSON(rows); exp != got {
t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
}
}
// Test_DELETEDatabaseCreatedOKFromWAL tests that a DELETE database is created properly,
// even when supplied with a WAL-mode database.
func Test_DELETEDatabaseCreatedOKFromWAL(t *testing.T) {
walPath := mustTempFile()
defer os.Remove(walPath)
walDB, err := Open(walPath, false, true)
if err != nil {
t.Fatalf("failed to open database in WAL mode: %s", err.Error())
}
defer walDB.Close()
if _, err := walDB.ExecuteStringStmt("CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"); err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
_, err = walDB.ExecuteStringStmt(`INSERT INTO foo(name) VALUES("fiona")`)
if err != nil {
t.Fatalf("error executing insertion into table: %s", err.Error())
}
if err := walDB.Close(); err != nil {
// Closing the WAL database is required if it's to be opened in DELETE mode.
t.Fatalf("failed to close database: %s", err.Error())
}
deleteDB, err2 := Open(walPath, false, false)
if err2 != nil {
t.Fatalf("failed to open database in DELETE mode: %s", err2.Error())
}
defer deleteDB.Close()
if !IsDELETEModeEnabledSQLiteFile(walPath) {
t.Fatalf("SQLite file not marked as WAL")
}
rows, err := deleteDB.QueryStringStmt("SELECT * FROM foo")
if err != nil {
t.Fatalf("failed to query WAL table: %s", err.Error())
}
if exp, got := `[{"columns":["id","name"],"types":["integer","text"],"values":[[1,"fiona"]]}]`, asJSON(rows); exp != got {
t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
}
}
// Test_WALReplayOK tests that WAL files are replayed as expected.
func Test_WALReplayOK(t *testing.T) {
testFunc := func(t *testing.T, replayIntoDelete bool) {
dbPath := mustTempFile()
defer os.Remove(dbPath)
db, err := Open(dbPath, false, true)
if err != nil {
t.Fatalf("failed to open database in WAL mode: %s", err.Error())
}
defer db.Close()
dbFile := filepath.Base(dbPath)
walPath := dbPath + "-wal"
walFile := filepath.Base(walPath)
replayDir := mustTempDir()
defer os.RemoveAll(replayDir)
replayDBPath := filepath.Join(replayDir, dbFile)
// Take over control of checkpointing
if err := db.DisableCheckpointing(); err != nil {
t.Fatalf("failed to disable checkpointing: %s", err.Error())
}
// Copy the SQLite file and WAL #1
if _, err := db.ExecuteStringStmt("CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"); err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
if !fileExists(walPath) {
t.Fatalf("WAL file at %s does not exist", walPath)
}
mustCopyFile(replayDBPath, dbPath)
mustCopyFile(filepath.Join(replayDir, walFile+"_001"), walPath)
if err := db.Checkpoint(5 * time.Second); err != nil {
t.Fatalf("failed to checkpoint database in WAL mode: %s", err.Error())
}
// Copy WAL #2
_, err = db.ExecuteStringStmt(`INSERT INTO foo(name) VALUES("fiona")`)
if err != nil {
t.Fatalf("error executing insertion into table: %s", err.Error())
}
if !fileExists(walPath) {
t.Fatalf("WAL file at %s does not exist", walPath)
}
mustCopyFile(filepath.Join(replayDir, walFile+"_002"), walPath)
if err := db.Checkpoint(5 * time.Second); err != nil {
t.Fatalf("failed to checkpoint database in WAL mode: %s", err.Error())
}
// Copy WAL #3
_, err = db.ExecuteStringStmt(`INSERT INTO foo(name) VALUES("declan")`)
if err != nil {
t.Fatalf("error executing insertion into table: %s", err.Error())
}
if !fileExists(walPath) {
t.Fatalf("WAL file at %s does not exist", walPath)
}
mustCopyFile(filepath.Join(replayDir, walFile+"_003"), walPath)
if err := db.Close(); err != nil {
t.Fatalf("failed to close database: %s", err.Error())
}
wals := []string{
filepath.Join(replayDir, walFile+"_001"),
filepath.Join(replayDir, walFile+"_002"),
filepath.Join(replayDir, walFile+"_003"),
}
if err := ReplayWAL(replayDBPath, wals, replayIntoDelete); err != nil {
t.Fatalf("failed to replay WAL files: %s", err.Error())
}
if replayIntoDelete {
if !IsDELETEModeEnabledSQLiteFile(replayDBPath) {
t.Fatal("replayed database not marked as DELETE mode")
}
} else {
if !IsWALModeEnabledSQLiteFile(replayDBPath) {
t.Fatal("replayed database not marked as WAL mode")
}
}
// Check that there are no files related to WALs in the replay directory
// Both the copied WAL files should be gone, and there should be no
// "real" WAL file either.
walFiles, err := filepath.Glob(filepath.Join(replayDir, "*-wal*"))
if err != nil {
t.Fatalf("failed to glob replay directory: %s", err.Error())
}
if len(walFiles) != 0 {
t.Fatalf("replay directory contains WAL files: %s", walFiles)
}
replayedDB, err := Open(replayDBPath, false, true)
if err != nil {
t.Fatalf("failed to open replayed database: %s", err.Error())
}
rows, err := replayedDB.QueryStringStmt("SELECT * FROM foo")
if err != nil {
t.Fatalf("failed to query WAL table: %s", err.Error())
}
if exp, got := `[{"columns":["id","name"],"types":["integer","text"],"values":[[1,"fiona"],[2,"declan"]]}]`, asJSON(rows); exp != got {
t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
}
}
t.Run("replayIntoWAL", func(t *testing.T) {
testFunc(t, false)
})
t.Run("replayIntoDELETE", func(t *testing.T) {
testFunc(t, true)
})
}
func Test_WALReplayFailures(t *testing.T) {
dbDir := mustTempDir()
defer os.RemoveAll(dbDir)
walDir := mustTempDir()
defer os.RemoveAll(walDir)
err := ReplayWAL(filepath.Join(dbDir, "foo.db"), []string{filepath.Join(walDir, "foo.db-wal")}, false)
if err != ErrWALReplayDirectoryMismatch {
t.Fatalf("expected %s, got %s", ErrWALReplayDirectoryMismatch, err.Error())
}
}
func test_FileCreationOnDisk(t *testing.T, db *DB) {
defer db.Close()
if db.FKEnabled() {
t.Fatal("FK constraints marked as enabled")
}
// Confirm checkpoint works on all types of on-disk databases. Worst case, this
// should be ignored.
if err := db.Checkpoint(5 * time.Second); err != nil {
t.Fatalf("failed to checkpoint database in DELETE mode: %s", err.Error())
}
}
// test_ConnectionIsolationOnDisk test that ISOLATION behavior of on-disk databases doesn't
// change unexpectedly.
func test_ConnectionIsolationOnDisk(t *testing.T, db *DB) {
r, err := db.ExecuteStringStmt("CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)")
if err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
if exp, got := `[{}]`, asJSON(r); exp != got {
t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
}
r, err = db.ExecuteStringStmt("BEGIN")
if err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
if exp, got := `[{}]`, asJSON(r); exp != got {
t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
}
r, err = db.ExecuteStringStmt(`INSERT INTO foo(name) VALUES("fiona")`)
if err != nil {
t.Fatalf("error executing insertion into table: %s", err.Error())
}
if exp, got := `[{"last_insert_id":1,"rows_affected":1}]`, asJSON(r); exp != got {
t.Fatalf("unexpected results for execute, expected %s, got %s", exp, got)
}
q, err := db.QueryStringStmt("SELECT * FROM foo")
if err != nil {
t.Fatalf("failed to query empty table: %s", err.Error())
}
if exp, got := `[{"columns":["id","name"],"types":["integer","text"]}]`, asJSON(q); exp != got {
t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
}
_, err = db.ExecuteStringStmt("COMMIT")
if err != nil {
t.Fatalf("error executing insertion into table: %s", err.Error())
}
q, err = db.QueryStringStmt("SELECT * FROM foo")
if err != nil {
t.Fatalf("failed to query empty table: %s", err.Error())
}
if exp, got := `[{"columns":["id","name"],"types":["integer","text"],"values":[[1,"fiona"]]}]`, asJSON(q); exp != got {
t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
}
}
func Test_DatabaseCommonOnDiskOperations(t *testing.T) {
testCases := []struct {
name string
testFunc func(*testing.T, *DB)
}{
{"FileCreationOnDisk", test_FileCreationOnDisk},
{"ConnectionIsolationOnDisk", test_ConnectionIsolationOnDisk},
}
for _, tc := range testCases {
db, path := mustCreateOnDiskDatabase()
defer db.Close()
defer os.Remove(path)
t.Run(tc.name+":disk", func(t *testing.T) {
tc.testFunc(t, db)
})
db, path = mustCreateOnDiskDatabaseWAL()
defer db.Close()
defer os.Remove(path)
t.Run(tc.name+":wal", func(t *testing.T) {
tc.testFunc(t, db)
})
}
}
// Test_ParallelOperationsInMemory runs multiple accesses concurrently, ensuring
// that correct results are returned in every goroutine. It's not 100% that this
// test would bring out a bug, but it's almost 100%.
//
// See https://github.com/mattn/go-sqlite3/issues/959#issuecomment-890283264
func Test_ParallelOperationsInMemory(t *testing.T) {
db, path := mustCreateOnDiskDatabaseWAL()
defer db.Close()
defer os.Remove(path)
if _, err := db.ExecuteStringStmt("CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"); err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
if _, err := db.ExecuteStringStmt("CREATE TABLE bar (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"); err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
if _, err := db.ExecuteStringStmt("CREATE TABLE qux (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"); err != nil {
t.Fatalf("failed to create table: %s", err.Error())
}
// Confirm schema is as expected, when checked from same goroutine.
if rows, err := db.QueryStringStmt(`SELECT sql FROM sqlite_master`); err != nil {
t.Fatalf("failed to query for schema after creation: %s", err.Error())
} else {
if exp, got := `[{"columns":["sql"],"types":["text"],"values":[["CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"],["CREATE TABLE bar (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"],["CREATE TABLE qux (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"]]}]`, asJSON(rows); exp != got {
t.Fatalf("schema not as expected during after creation, exp %s, got %s", exp, got)
}
}
var exWg sync.WaitGroup
exWg.Add(3)
foo := make(chan time.Time)
bar := make(chan time.Time)
qux := make(chan time.Time)
done := make(chan bool)
ticker := time.NewTicker(1 * time.Millisecond)
go func() {
for {
select {
case t := <-ticker.C:
foo <- t
bar <- t
qux <- t
case <-done:
close(foo)
close(bar)
close(qux)
return
}
}
}()
go func() {
defer exWg.Done()
for range foo {
if _, err := db.ExecuteStringStmt(`INSERT INTO foo(id, name) VALUES(1, "fiona")`); err != nil {
t.Logf("failed to insert records into foo: %s", err.Error())
}
}
}()
go func() {
defer exWg.Done()
for range bar {
if _, err := db.ExecuteStringStmt(`INSERT INTO bar(id, name) VALUES(1, "fiona")`); err != nil {
t.Logf("failed to insert records into bar: %s", err.Error())
}
}
}()
go func() {
defer exWg.Done()
for range qux {
if _, err := db.ExecuteStringStmt(`INSERT INTO qux(id, name) VALUES(1, "fiona")`); err != nil {
t.Logf("failed to insert records into qux: %s", err.Error())
}
}
}()
var qWg sync.WaitGroup
qWg.Add(3)
for i := 0; i < 3; i++ {
go func(j int) {
defer qWg.Done()
var n int
for {
if rows, err := db.QueryStringStmt(`SELECT sql FROM sqlite_master`); err != nil {
t.Logf("failed to query for schema during goroutine %d execution: %s", j, err.Error())
} else {
n++
if exp, got := `[{"columns":["sql"],"types":["text"],"values":[["CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"],["CREATE TABLE bar (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"],["CREATE TABLE qux (id INTEGER NOT NULL PRIMARY KEY, name TEXT)"]]}]`, asJSON(rows); exp != got {
t.Logf("schema not as expected during goroutine execution, exp %s, got %s, after %d queries", exp, got, n)
}
}
if n == 500000 {
break
}
}
}(i)
}
qWg.Wait()
close(done)
exWg.Wait()
}
func mustCreateOnDiskDatabase() (*DB, string) {
var err error
f := mustTempFile()
@ -229,7 +859,6 @@ func mustCreateOnDiskDatabase() (*DB, string) {
if err != nil {
panic("failed to open database in DELETE mode")
}
return db, f
}
@ -240,24 +869,17 @@ func mustCreateOnDiskDatabaseWAL() (*DB, string) {
if err != nil {
panic("failed to open database in WAL mode")
}
return db, f
}
func mustCreateInMemoryDatabase() *DB {
db, err := OpenInMemory(false)
if err != nil {
panic("failed to open in-memory database")
}
return db
}
func mustCreateInMemoryDatabaseFK() *DB {
db, err := OpenInMemory(true)
func mustCreateOnDiskDatabaseWALFK() (*DB, string) {
var err error
f := mustTempFile()
db, err := Open(f, true, true)
if err != nil {
panic("failed to open in-memory database with foreign key constraints")
panic("failed to open database in WAL mode")
}
return db
return db, f
}
// mustExecute executes a statement, and panics on failure. Used for statements

Write Preview
Loading…
Cancel
Save