Merge pull request #591 from rqlite/store_sql_backups

Store layer supports generating SQL format backups

CHANGELOG.md

@@ -1,4 +1,5 @@
 ## 4.6.0 (unreleased)
+- [PR #591](https://github.com/rqlite/rqlite/pull/591): Store layer supports generating SQL format backups
 - [PR #590](https://github.com/rqlite/rqlite/pull/590): DB layer supports generating SQL format backups
 - [PR #589](https://github.com/rqlite/rqlite/pull/589): Add restore command to CLI. Thanks @eariassoto.
 - [PR #588](https://github.com/rqlite/rqlite/pull/588): Abort transaction if load operation fails.

db/db.go

@@ -102,26 +102,10 @@ func LoadInMemoryWithDSN(dbPath, dsn string) (*DB, error) {
 		return nil, err
 	}
 
-	bk, err := db.sqlite3conn.Backup("main", srcDB.sqlite3conn, "main")
-	if err != nil {
+	if err := copyDatabase(db.sqlite3conn, srcDB.sqlite3conn); err != nil {
 		return nil, err
 	}
 
-	for {
-		done, err := bk.Step(-1)
-		if err != nil {
-			bk.Finish()
-			return nil, err
-		}
-		if done {
-			break
-		}
-		time.Sleep(bkDelay * time.Millisecond)
-	}
-
-	if err := bk.Finish(); err != nil {
-		return nil, err
-	}
 	if err := srcDB.Close(); err != nil {
 		return nil, err
 	}
@@ -383,24 +367,7 @@ func (db *DB) Backup(path string) error {
 		return err
 	}
 
-	bk, err := dstDB.sqlite3conn.Backup("main", db.sqlite3conn, "main")
-	if err != nil {
-		return err
-	}
-
-	for {
-		done, err := bk.Step(-1)
-		if err != nil {
-			bk.Finish()
-			return err
-		}
-		if done {
-			break
-		}
-		time.Sleep(bkDelay * time.Millisecond)
-	}
-
-	if err := bk.Finish(); err != nil {
+	if err := copyDatabase(dstDB.sqlite3conn, db.sqlite3conn); err != nil {
 		return err
 	}
 
@@ -414,15 +381,25 @@ func (db *DB) Dump(w io.Writer) error {
 	}
 
 	// Get a new connection, so the dump creation is isolated from other activity.
-	conn, err := open(fqdsn(db.path, db.dsn))
+	dstDB, err := OpenInMemory()
 	if err != nil {
-		return nil
+		return err
+	}
+	defer func(db *DB, err *error) {
+		cerr := db.Close()
+		if *err == nil {
+			*err = cerr
+		}
+	}(dstDB, &err)
+
+	if err := copyDatabase(dstDB.sqlite3conn, db.sqlite3conn); err != nil {
+		return err
 	}
 
 	// Get the schema.
 	query := `SELECT "name", "type", "sql" FROM "sqlite_master"
               WHERE "sql" NOT NULL AND "type" == 'table' ORDER BY "name"`
-	rows, err := conn.Query([]string{query}, false, false)
+	rows, err := dstDB.Query([]string{query}, false, false)
 	if err != nil {
 		return err
 	}
@@ -446,7 +423,7 @@ func (db *DB) Dump(w io.Writer) error {
 		}
 
 		tableIndent := strings.Replace(table, `"`, `""`, -1)
-		r, err := conn.Query([]string{fmt.Sprintf(`PRAGMA table_info("%s")`, tableIndent)}, false, false)
+		r, err := dstDB.Query([]string{fmt.Sprintf(`PRAGMA table_info("%s")`, tableIndent)}, false, false)
 		if err != nil {
 			return err
 		}
@@ -459,7 +436,7 @@ func (db *DB) Dump(w io.Writer) error {
 			tableIndent,
 			strings.Join(columnNames, ","),
 			tableIndent)
-		r, err = conn.Query([]string{query}, false, false)
+		r, err = dstDB.Query([]string{query}, false, false)
 		if err != nil {
 			return err
 		}
@@ -474,7 +451,7 @@ func (db *DB) Dump(w io.Writer) error {
 	// Do indexes, triggers, and views.
 	query = `SELECT "name", "type", "sql" FROM "sqlite_master"
 			  WHERE "sql" NOT NULL AND "type" IN ('index', 'trigger', 'view')`
-	rows, err = conn.Query([]string{query}, false, false)
+	rows, err = db.Query([]string{query}, false, false)
 	if err != nil {
 		return err
 	}
@@ -492,6 +469,31 @@ func (db *DB) Dump(w io.Writer) error {
 	return nil
 }
 
+func copyDatabase(dst *sqlite3.SQLiteConn, src *sqlite3.SQLiteConn) error {
+	bk, err := dst.Backup("main", src, "main")
+	if err != nil {
+		return err
+	}
+
+	for {
+		done, err := bk.Step(-1)
+		if err != nil {
+			bk.Finish()
+			return err
+		}
+		if done {
+			break
+		}
+		time.Sleep(bkDelay * time.Millisecond)
+	}
+
+	if err := bk.Finish(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // normalizeRowValues performs some normalization of values in the returned rows.
 // Text values come over (from sqlite-go) as []byte instead of strings
 // for some reason, so we have explicitly convert (but only when type

db/db_test.go

@@ -749,6 +749,33 @@ func Test_Dump(t *testing.T) {
 	}
 }
 
+func Test_DumpMemory(t *testing.T) {
+	t.Parallel()
+
+	db, path := mustCreateDatabase()
+	defer db.Close()
+	defer os.Remove(path)
+
+	inmem, err := LoadInMemoryWithDSN(path, "")
+	if err != nil {
+		t.Fatalf("failed to create loaded in-memory database: %s", err.Error())
+	}
+
+	_, err = inmem.Execute([]string{chinook.DB}, false, false)
+	if err != nil {
+		t.Fatalf("failed to load chinook dump: %s", err.Error())
+	}
+
+	var b strings.Builder
+	if err := inmem.Dump(&b); err != nil {
+		t.Fatalf("failed to dump database: %s", err.Error())
+	}
+
+	if b.String() != chinook.DB {
+		t.Fatal("dumped database does not equal entered database")
+	}
+}
+
 func mustCreateDatabase() (*DB, string) {
 	var err error
 	f, err := ioutil.TempFile("", "rqlilte-test-")

http/service.go

@@ -59,7 +59,7 @@ type Store interface {
 	Stats() (map[string]interface{}, error)
 
 	// Backup returns a byte slice representing a backup of the node state.
-	Backup(leader bool) ([]byte, error)
+	Backup(leader bool, f store.BackupFormat) ([]byte, error)
 }
 
 // CredentialStore is the interface credential stores must support.
@@ -380,8 +380,6 @@ func (s *Service) handleRemove(w http.ResponseWriter, r *http.Request) {
 
 // handleBackup returns the consistent database snapshot.
 func (s *Service) handleBackup(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Content-Type", "application/octet-stream")
-
 	if !s.CheckRequestPerm(r, PermBackup) {
 		w.WriteHeader(http.StatusUnauthorized)
 		return
@@ -398,9 +396,15 @@ func (s *Service) handleBackup(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	b, err := s.store.Backup(!noLeader)
+	bf, err := backupFormat(w, r)
 	if err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	b, err := s.store.Backup(!noLeader, bf)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
 
@@ -844,6 +848,12 @@ func stmtParam(req *http.Request) (string, error) {
 	return stmt, nil
 }
 
+// fmtParam returns the value for URL param 'fmt', if present.
+func fmtParam(req *http.Request) (string, error) {
+	q := req.URL.Query()
+	return strings.TrimSpace(q.Get("fmt")), nil
+}
+
 // isPretty returns whether the HTTP response body should be pretty-printed.
 func isPretty(req *http.Request) (bool, error) {
 	return queryParam(req, "pretty")
@@ -881,6 +891,21 @@ func level(req *http.Request) (store.ConsistencyLevel, error) {
 	}
 }
 
+// backupFormat returns the request backup format, setting the response header
+// accordingly.
+func backupFormat(w http.ResponseWriter, r *http.Request) (store.BackupFormat, error) {
+	fmt, err := fmtParam(r)
+	if err != nil {
+		return store.BackupBinary, err
+	}
+	if fmt == "sql" {
+		w.Header().Set("Content-Type", "application/sql")
+		return store.BackupSQL, nil
+	}
+	w.Header().Set("Content-Type", "application/octet-stream")
+	return store.BackupBinary, nil
+}
+
 func prettyEnabled(e bool) string {
 	if e {
 		return "enabled"

http/service_test.go

@@ -511,7 +511,7 @@ func (m *MockStore) Stats() (map[string]interface{}, error) {
 	return nil, nil
 }
 
-func (m *MockStore) Backup(leader bool) ([]byte, error) {
+func (m *MockStore) Backup(leader bool, f store.BackupFormat) ([]byte, error) {
 	return nil, nil
 }
 

store/store.go

@@ -32,6 +32,10 @@ var (
 	// ErrOpenTimeout is returned when the Store does not apply its initial
 	// logs within the specified time.
 	ErrOpenTimeout = errors.New("timeout waiting for initial logs application")
+
+	// ErrInvalidBackupFormat is returned when the requested backup format
+	// is not valid.
+	ErrInvalidBackupFormat = errors.New("invalid backup format")
 )
 
 const (
@@ -49,6 +53,13 @@ const (
 	numRestores = "num_restores"
 )
 
+type BackupFormat int
+
+const (
+	BackupSQL BackupFormat = iota
+	BackupBinary
+)
+
 // stats captures stats for the Store.
 var stats *expvar.Map
 
@@ -540,26 +551,38 @@ func (s *Store) execute(ex *ExecuteRequest) ([]*sql.Result, error) {
 // If leader is true, this operation is performed with a read consistency
 // level equivalent to "weak". Otherwise no guarantees are made about the
 // read consistency level.
-func (s *Store) Backup(leader bool) ([]byte, error) {
+func (s *Store) Backup(leader bool, fmt BackupFormat) ([]byte, error) {
 	if leader && s.raft.State() != raft.Leader {
 		return nil, ErrNotLeader
 	}
 
-	f, err := ioutil.TempFile("", "rqlite-bak-")
-	if err != nil {
-		return nil, err
-	}
-	f.Close()
-	defer os.Remove(f.Name())
+	var b []byte
+	if fmt == BackupBinary {
+		f, err := ioutil.TempFile("", "rqlite-bak-")
+		if err != nil {
+			return nil, err
+		}
+		f.Close()
+		defer os.Remove(f.Name())
 
-	if err := s.db.Backup(f.Name()); err != nil {
-		return nil, err
-	}
+		if err := s.db.Backup(f.Name()); err != nil {
+			return nil, err
+		}
 
-	b, err := ioutil.ReadFile(f.Name())
-	if err != nil {
-		return nil, err
+		b, err = ioutil.ReadFile(f.Name())
+		if err != nil {
+			return nil, err
+		}
+	} else if fmt == BackupSQL {
+		buf := bytes.NewBuffer(nil)
+		if err := s.db.Dump(buf); err != nil {
+			return nil, err
+		}
+		b = buf.Bytes()
+	} else {
+		return nil, ErrInvalidBackupFormat
 	}
+
 	stats.Add(numBackups, 1)
 	return b, nil
 }