From 0e69be3949abec2d2652d0c44c456d11f18e9868 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Fri, 29 Nov 2019 14:53:47 -0500 Subject: [PATCH] Store layer supports generating SQL format backups Port PR453. --- CHANGELOG.md | 1 + db/db.go | 84 +++++++++++++++++++++++--------------------- db/db_test.go | 27 ++++++++++++++ http/service.go | 35 +++++++++++++++--- http/service_test.go | 2 +- store/store.go | 49 +++++++++++++++++++------- 6 files changed, 138 insertions(+), 60 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2772ad74..57c532de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## 4.6.0 (unreleased) +- [PR #591](https://github.com/rqlite/rqlite/pull/591): Store layer supports generating SQL format backups - [PR #590](https://github.com/rqlite/rqlite/pull/590): DB layer supports generating SQL format backups - [PR #589](https://github.com/rqlite/rqlite/pull/589): Add restore command to CLI. Thanks @eariassoto. - [PR #588](https://github.com/rqlite/rqlite/pull/588): Abort transaction if load operation fails. diff --git a/db/db.go b/db/db.go index 286e5a74..fc87cfdc 100644 --- a/db/db.go +++ b/db/db.go @@ -102,26 +102,10 @@ func LoadInMemoryWithDSN(dbPath, dsn string) (*DB, error) { return nil, err } - bk, err := db.sqlite3conn.Backup("main", srcDB.sqlite3conn, "main") - if err != nil { + if err := copyDatabase(db.sqlite3conn, srcDB.sqlite3conn); err != nil { return nil, err } - for { - done, err := bk.Step(-1) - if err != nil { - bk.Finish() - return nil, err - } - if done { - break - } - time.Sleep(bkDelay * time.Millisecond) - } - - if err := bk.Finish(); err != nil { - return nil, err - } if err := srcDB.Close(); err != nil { return nil, err } @@ -383,24 +367,7 @@ func (db *DB) Backup(path string) error { return err } - bk, err := dstDB.sqlite3conn.Backup("main", db.sqlite3conn, "main") - if err != nil { - return err - } - - for { - done, err := bk.Step(-1) - if err != nil { - bk.Finish() - return err - } - if done { - break - } - time.Sleep(bkDelay * time.Millisecond) - } - - if err := bk.Finish(); err != nil { + if err := copyDatabase(dstDB.sqlite3conn, db.sqlite3conn); err != nil { return err } @@ -414,15 +381,25 @@ func (db *DB) Dump(w io.Writer) error { } // Get a new connection, so the dump creation is isolated from other activity. - conn, err := open(fqdsn(db.path, db.dsn)) + dstDB, err := OpenInMemory() if err != nil { - return nil + return err + } + defer func(db *DB, err *error) { + cerr := db.Close() + if *err == nil { + *err = cerr + } + }(dstDB, &err) + + if err := copyDatabase(dstDB.sqlite3conn, db.sqlite3conn); err != nil { + return err } // Get the schema. query := `SELECT "name", "type", "sql" FROM "sqlite_master" WHERE "sql" NOT NULL AND "type" == 'table' ORDER BY "name"` - rows, err := conn.Query([]string{query}, false, false) + rows, err := dstDB.Query([]string{query}, false, false) if err != nil { return err } @@ -446,7 +423,7 @@ func (db *DB) Dump(w io.Writer) error { } tableIndent := strings.Replace(table, `"`, `""`, -1) - r, err := conn.Query([]string{fmt.Sprintf(`PRAGMA table_info("%s")`, tableIndent)}, false, false) + r, err := dstDB.Query([]string{fmt.Sprintf(`PRAGMA table_info("%s")`, tableIndent)}, false, false) if err != nil { return err } @@ -459,7 +436,7 @@ func (db *DB) Dump(w io.Writer) error { tableIndent, strings.Join(columnNames, ","), tableIndent) - r, err = conn.Query([]string{query}, false, false) + r, err = dstDB.Query([]string{query}, false, false) if err != nil { return err } @@ -474,7 +451,7 @@ func (db *DB) Dump(w io.Writer) error { // Do indexes, triggers, and views. query = `SELECT "name", "type", "sql" FROM "sqlite_master" WHERE "sql" NOT NULL AND "type" IN ('index', 'trigger', 'view')` - rows, err = conn.Query([]string{query}, false, false) + rows, err = db.Query([]string{query}, false, false) if err != nil { return err } @@ -492,6 +469,31 @@ func (db *DB) Dump(w io.Writer) error { return nil } +func copyDatabase(dst *sqlite3.SQLiteConn, src *sqlite3.SQLiteConn) error { + bk, err := dst.Backup("main", src, "main") + if err != nil { + return err + } + + for { + done, err := bk.Step(-1) + if err != nil { + bk.Finish() + return err + } + if done { + break + } + time.Sleep(bkDelay * time.Millisecond) + } + + if err := bk.Finish(); err != nil { + return err + } + + return nil +} + // normalizeRowValues performs some normalization of values in the returned rows. // Text values come over (from sqlite-go) as []byte instead of strings // for some reason, so we have explicitly convert (but only when type diff --git a/db/db_test.go b/db/db_test.go index cb1d97e9..af6cf336 100644 --- a/db/db_test.go +++ b/db/db_test.go @@ -749,6 +749,33 @@ func Test_Dump(t *testing.T) { } } +func Test_DumpMemory(t *testing.T) { + t.Parallel() + + db, path := mustCreateDatabase() + defer db.Close() + defer os.Remove(path) + + inmem, err := LoadInMemoryWithDSN(path, "") + if err != nil { + t.Fatalf("failed to create loaded in-memory database: %s", err.Error()) + } + + _, err = inmem.Execute([]string{chinook.DB}, false, false) + if err != nil { + t.Fatalf("failed to load chinook dump: %s", err.Error()) + } + + var b strings.Builder + if err := inmem.Dump(&b); err != nil { + t.Fatalf("failed to dump database: %s", err.Error()) + } + + if b.String() != chinook.DB { + t.Fatal("dumped database does not equal entered database") + } +} + func mustCreateDatabase() (*DB, string) { var err error f, err := ioutil.TempFile("", "rqlilte-test-") diff --git a/http/service.go b/http/service.go index e459ccde..70c0d8d4 100644 --- a/http/service.go +++ b/http/service.go @@ -59,7 +59,7 @@ type Store interface { Stats() (map[string]interface{}, error) // Backup returns a byte slice representing a backup of the node state. - Backup(leader bool) ([]byte, error) + Backup(leader bool, f store.BackupFormat) ([]byte, error) } // CredentialStore is the interface credential stores must support. @@ -380,8 +380,6 @@ func (s *Service) handleRemove(w http.ResponseWriter, r *http.Request) { // handleBackup returns the consistent database snapshot. func (s *Service) handleBackup(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/octet-stream") - if !s.CheckRequestPerm(r, PermBackup) { w.WriteHeader(http.StatusUnauthorized) return @@ -398,9 +396,15 @@ func (s *Service) handleBackup(w http.ResponseWriter, r *http.Request) { return } - b, err := s.store.Backup(!noLeader) + bf, err := backupFormat(w, r) if err != nil { - w.WriteHeader(http.StatusInternalServerError) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + b, err := s.store.Backup(!noLeader, bf) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -844,6 +848,12 @@ func stmtParam(req *http.Request) (string, error) { return stmt, nil } +// fmtParam returns the value for URL param 'fmt', if present. +func fmtParam(req *http.Request) (string, error) { + q := req.URL.Query() + return strings.TrimSpace(q.Get("fmt")), nil +} + // isPretty returns whether the HTTP response body should be pretty-printed. func isPretty(req *http.Request) (bool, error) { return queryParam(req, "pretty") @@ -881,6 +891,21 @@ func level(req *http.Request) (store.ConsistencyLevel, error) { } } +// backupFormat returns the request backup format, setting the response header +// accordingly. +func backupFormat(w http.ResponseWriter, r *http.Request) (store.BackupFormat, error) { + fmt, err := fmtParam(r) + if err != nil { + return store.BackupBinary, err + } + if fmt == "sql" { + w.Header().Set("Content-Type", "application/sql") + return store.BackupSQL, nil + } + w.Header().Set("Content-Type", "application/octet-stream") + return store.BackupBinary, nil +} + func prettyEnabled(e bool) string { if e { return "enabled" diff --git a/http/service_test.go b/http/service_test.go index add663ce..b59c048c 100644 --- a/http/service_test.go +++ b/http/service_test.go @@ -511,7 +511,7 @@ func (m *MockStore) Stats() (map[string]interface{}, error) { return nil, nil } -func (m *MockStore) Backup(leader bool) ([]byte, error) { +func (m *MockStore) Backup(leader bool, f store.BackupFormat) ([]byte, error) { return nil, nil } diff --git a/store/store.go b/store/store.go index c5fa00e4..98460f35 100644 --- a/store/store.go +++ b/store/store.go @@ -32,6 +32,10 @@ var ( // ErrOpenTimeout is returned when the Store does not apply its initial // logs within the specified time. ErrOpenTimeout = errors.New("timeout waiting for initial logs application") + + // ErrInvalidBackupFormat is returned when the requested backup format + // is not valid. + ErrInvalidBackupFormat = errors.New("invalid backup format") ) const ( @@ -49,6 +53,13 @@ const ( numRestores = "num_restores" ) +type BackupFormat int + +const ( + BackupSQL BackupFormat = iota + BackupBinary +) + // stats captures stats for the Store. var stats *expvar.Map @@ -540,26 +551,38 @@ func (s *Store) execute(ex *ExecuteRequest) ([]*sql.Result, error) { // If leader is true, this operation is performed with a read consistency // level equivalent to "weak". Otherwise no guarantees are made about the // read consistency level. -func (s *Store) Backup(leader bool) ([]byte, error) { +func (s *Store) Backup(leader bool, fmt BackupFormat) ([]byte, error) { if leader && s.raft.State() != raft.Leader { return nil, ErrNotLeader } - f, err := ioutil.TempFile("", "rqlite-bak-") - if err != nil { - return nil, err - } - f.Close() - defer os.Remove(f.Name()) + var b []byte + if fmt == BackupBinary { + f, err := ioutil.TempFile("", "rqlite-bak-") + if err != nil { + return nil, err + } + f.Close() + defer os.Remove(f.Name()) - if err := s.db.Backup(f.Name()); err != nil { - return nil, err - } + if err := s.db.Backup(f.Name()); err != nil { + return nil, err + } - b, err := ioutil.ReadFile(f.Name()) - if err != nil { - return nil, err + b, err = ioutil.ReadFile(f.Name()) + if err != nil { + return nil, err + } + } else if fmt == BackupSQL { + buf := bytes.NewBuffer(nil) + if err := s.db.Dump(buf); err != nil { + return nil, err + } + b = buf.Bytes() + } else { + return nil, ErrInvalidBackupFormat } + stats.Add(numBackups, 1) return b, nil }