Merge pull request #1296 from rqlite/common-db-tests

Common DB-layer tests

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 7.20.2 (unreleased)
+### Implementation changes and bug fixes
+- [PR #1296](https://github.com/rqlite/rqlite/pull/1296): Use correct connection when checking a SQL statement for "read-only" status, otherwise "database locked" could result. Also refactors much of the DB-level unit tests.
+
 ## 7.20.1 (June 1st 2023)
 ### Implementation changes and bug fixes
 - [PR #1291](https://github.com/rqlite/rqlite/pull/1291): Allow bootstrap-join even with preexisting state. Fixes [issue #1290](https://github.com/rqlite/rqlite/issues/1290) 

db/db.go

@@ -582,7 +582,7 @@ func (db *DB) queryWithConn(req *command.Request, xTime bool, conn *sql.Conn) ([
 		var rows *command.QueryRows
 		var err error
 
-		readOnly, err := db.StmtReadOnly(sql)
+		readOnly, err := db.StmtReadOnlyWithConn(sql, conn)
 		if err != nil {
 			stats.Add(numQueryErrors, 1)
 			rows = &command.QueryRows{
@@ -738,7 +738,7 @@ func (db *DB) Request(req *command.Request, xTime bool) ([]*command.ExecuteQuery
 			continue
 		}
 
-		ro, err := db.StmtReadOnly(ss)
+		ro, err := db.StmtReadOnlyWithConn(ss, conn)
 		if err != nil {
 			eqResponse = append(eqResponse, &command.ExecuteQueryResponse{
 				Result: &command.ExecuteQueryResponse_Error{
@@ -927,6 +927,17 @@ func (db *DB) Dump(w io.Writer) error {
 // As per https://www.sqlite.org/c3ref/stmt_readonly.html, this function
 // may not return 100% correct results, but should cover most scenarios.
 func (db *DB) StmtReadOnly(sql string) (bool, error) {
+	conn, err := db.roDB.Conn(context.Background())
+	if err != nil {
+		return false, err
+	}
+	defer conn.Close()
+	return db.StmtReadOnlyWithConn(sql, conn)
+}
+
+// StmtReadOnlyWithConn returns whether the given SQL statement is read-only, using
+// the given connection.
+func (db *DB) StmtReadOnlyWithConn(sql string, conn *sql.Conn) (bool, error) {
 	var readOnly bool
 	f := func(driverConn interface{}) error {
 		c := driverConn.(*sqlite3.SQLiteConn)
@@ -940,11 +951,6 @@ func (db *DB) StmtReadOnly(sql string) (bool, error) {
 		return nil
 	}
 
-	conn, err := db.roDB.Conn(context.Background())
-	if err != nil {
-		return false, err
-	}
-	defer conn.Close()
 	if err := conn.Raw(f); err != nil {
 		return false, err
 	}

db/db_inmem_test.go

@@ -1,6 +1,7 @@
 package db
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"sync"
@@ -8,6 +9,7 @@ import (
 	"time"
 
 	"github.com/rqlite/rqlite/command"
+	text "github.com/rqlite/rqlite/db/testdata"
 )
 
 // Test_TableCreationInMemory tests basic operation of an in-memory database,
@@ -295,3 +297,45 @@ func Test_TableCreationLoadRawInMemory(t *testing.T) {
 		}
 	}
 }
+
+// Test_1GiBInMemory tests that in-memory databases larger than 1GiB,
+// but smaller than 2GiB, can be created without issue.
+func Test_1GiBInMemory(t *testing.T) {
+	db := mustCreateInMemoryDatabase()
+	defer db.Close()
+
+	_, err := db.ExecuteStringStmt("CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, txt TEXT)")
+	if err != nil {
+		t.Fatalf("failed to create table: %s", err.Error())
+	}
+
+	stmt := fmt.Sprintf(`INSERT INTO foo(txt) VALUES("%s")`, text.Lorum)
+	for i := 0; i < 1715017; i++ {
+		r, err := db.ExecuteStringStmt(stmt)
+		if err != nil {
+			t.Fatalf("failed to Execute statement %s", err.Error())
+		}
+		if len(r) != 1 {
+			t.Fatalf("unexpected length for Execute results: %d", len(r))
+		}
+		if r[0].GetError() != "" {
+			t.Fatalf("failed to insert record: %s", r[0].GetError())
+		}
+	}
+
+	r, err := db.ExecuteStringStmt(stmt)
+	if err != nil {
+		t.Fatalf("failed to insert record %s", err.Error())
+	}
+	if exp, got := `[{"last_insert_id":1715018,"rows_affected":1}]`, asJSON(r); exp != got {
+		t.Fatalf("got incorrect response, exp: %s, got: %s", exp, got)
+	}
+
+	sz, err := db.Size()
+	if err != nil {
+		t.Fatalf("failed to get size: %s", err.Error())
+	}
+	if sz <= 1024*1024*1024 {
+		t.Fatalf("failed to create a database greater than 1 GiB in size: %d", sz)
+	}
+}

db/db_ondisk_test.go

@@ -49,6 +49,7 @@ func Test_FileCreationOnDisk(t *testing.T) {
 	if db == nil {
 		t.Fatal("database is nil")
 	}
+	defer db.Close()
 	if db.InMemory() {
 		t.Fatal("on-disk database marked as in-memory")
 	}
@@ -67,3 +68,61 @@ func Test_FileCreationOnDisk(t *testing.T) {
 		t.Fatalf("failed to close database: %s", err.Error())
 	}
 }
+
+// Test_ConnectionIsolationOnDisk test that ISOLATION behavior of on-disk databases doesn't
+// change unexpectedly.
+func Test_ConnectionIsolationOnDisk(t *testing.T) {
+	dir := t.TempDir()
+	dbPath := path.Join(dir, "test_db")
+
+	db, err := Open(dbPath, false)
+	if err != nil {
+		t.Fatalf("failed to open new database: %s", err.Error())
+	}
+	defer db.Close()
+
+	r, err := db.ExecuteStringStmt("CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)")
+	if err != nil {
+		t.Fatalf("failed to create table: %s", err.Error())
+	}
+	if exp, got := `[{}]`, asJSON(r); exp != got {
+		t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
+	}
+
+	r, err = db.ExecuteStringStmt("BEGIN")
+	if err != nil {
+		t.Fatalf("failed to create table: %s", err.Error())
+	}
+	if exp, got := `[{}]`, asJSON(r); exp != got {
+		t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
+	}
+
+	r, err = db.ExecuteStringStmt(`INSERT INTO foo(name) VALUES("fiona")`)
+	if err != nil {
+		t.Fatalf("error executing insertion into table: %s", err.Error())
+	}
+	if exp, got := `[{"last_insert_id":1,"rows_affected":1}]`, asJSON(r); exp != got {
+		t.Fatalf("unexpected results for execute, expected %s, got %s", exp, got)
+	}
+
+	q, err := db.QueryStringStmt("SELECT * FROM foo")
+	if err != nil {
+		t.Fatalf("failed to query empty table: %s", err.Error())
+	}
+	if exp, got := `[{"columns":["id","name"],"types":["integer","text"]}]`, asJSON(q); exp != got {
+		t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
+	}
+
+	_, err = db.ExecuteStringStmt("COMMIT")
+	if err != nil {
+		t.Fatalf("error executing insertion into table: %s", err.Error())
+	}
+
+	q, err = db.QueryStringStmt("SELECT * FROM foo")
+	if err != nil {
+		t.Fatalf("failed to query empty table: %s", err.Error())
+	}
+	if exp, got := `[{"columns":["id","name"],"types":["integer","text"],"values":[[1,"fiona"]]}]`, asJSON(q); exp != got {
+		t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
+	}
+}