1
0
Fork 0

Merge pull request #247 from rqlite/simple_dump_load

Use single load for dump processing
master
Philip O'Toole 8 years ago committed by GitHub
commit bdd56cb538

@ -8,7 +8,6 @@ import (
"encoding/json"
"expvar"
"fmt"
"io"
"io/ioutil"
"log"
"net"
@ -54,9 +53,6 @@ type Store interface {
// Backup returns a byte slice representing a backup of the node state.
Backup(leader bool) ([]byte, error)
// Load loads a SQLite .dump state from a reader
Load(r io.Reader) (int, error)
}
// CredentialStore is the interface credential stores must support.
@ -101,6 +97,8 @@ const (
PermStatus = "status"
// PermBackup means user can backup node.
PermBackup = "backup"
// PermLoad means user can load a SQLite dump into a node.
PermLoad = "load"
)
func init() {
@ -372,7 +370,7 @@ func (s *Service) handleBackup(w http.ResponseWriter, r *http.Request) {
// handleLoad loads the state contained in a .dump output.
func (s *Service) handleLoad(w http.ResponseWriter, r *http.Request) {
if !s.CheckRequestPerm(r, PermBackup) {
if !s.CheckRequestPerm(r, PermLoad) {
w.WriteHeader(http.StatusUnauthorized)
return
}
@ -382,12 +380,41 @@ func (s *Service) handleLoad(w http.ResponseWriter, r *http.Request) {
return
}
n, err := s.store.Load(r.Body)
resp := NewResponse()
timings, err := timings(r)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
s.logger.Printf(`.dump data with %d commands loaded OK`, n)
b, err := ioutil.ReadAll(r.Body)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
r.Body.Close()
queries := []string{string(b)}
results, err := s.store.Execute(queries, timings, false)
if err != nil {
if err == store.ErrNotLeader {
leader := s.store.Peer(s.store.Leader())
if leader == "" {
http.Error(w, err.Error(), http.StatusServiceUnavailable)
return
}
redirect := s.FormRedirect(r, leader)
http.Redirect(w, r, redirect, http.StatusMovedPermanently)
return
}
resp.Error = err.Error()
} else {
resp.Results = results
}
resp.end = time.Now()
writeResponse(w, r, resp)
}
// handleStatus returns status on the system.

@ -2,7 +2,6 @@ package http
import (
"fmt"
"io"
"net/http"
"testing"
@ -264,6 +263,7 @@ func Test_401Routes_NoBasicAuth(t *testing.T) {
"/db/execute",
"/db/query",
"/db/backup",
"/db/load",
"/join",
"/delete",
"/status",
@ -295,6 +295,7 @@ func Test_401Routes_BasicAuthBadPassword(t *testing.T) {
"/db/execute",
"/db/query",
"/db/backup",
"/db/load",
"/join",
"/status",
} {
@ -331,6 +332,7 @@ func Test_401Routes_BasicAuthBadPerm(t *testing.T) {
"/db/execute",
"/db/query",
"/db/backup",
"/db/load",
"/join",
"/status",
} {
@ -393,10 +395,6 @@ func (m *MockStore) Backup(leader bool) ([]byte, error) {
return nil, nil
}
func (m *MockStore) Load(r io.Reader) (int, error) {
return 0, nil
}
type mockCredentialStore struct {
CheckOK bool
HasPermOK bool

@ -1,112 +0,0 @@
package sql
import (
"bufio"
"bytes"
"io"
"strings"
)
// stack represents a stack.
type stack struct {
c []rune
}
// newStack returns an instance of a stack.
func newStack() *stack {
return &stack{
c: make([]rune, 0),
}
}
// push pushes a rune onto the stack.
func (s *stack) push(r rune) {
s.c = append(s.c, r)
}
// pop pops a rune off the stack.
func (s *stack) pop() rune {
if len(s.c) == 0 {
return rune(0)
}
c := s.c[len(s.c)-1]
s.c = s.c[:len(s.c)-1]
return c
}
// peek returns what is on the stack, without changing the stack.
func (s *stack) peek() rune {
if len(s.c) == 0 {
return rune(0)
}
c := s.c[len(s.c)-1]
return c
}
// empty returns whether the stack is empty.
func (s *stack) empty() bool {
return len(s.c) == 0
}
// Scanner represents a SQL statement scanner.
type Scanner struct {
r *bufio.Reader
c *stack
}
// NewScanner returns a new instance of Scanner.
func NewScanner(r io.Reader) *Scanner {
return &Scanner{
r: bufio.NewReader(r),
c: newStack(),
}
}
// read reads the next rune from the bufferred reader.
// Returns the rune(0) if an error occurs (or io.EOF is returned).
func (s *Scanner) read() rune {
ch, _, err := s.r.ReadRune()
if err != nil {
return eof
}
return ch
}
// Scan returns the next SQL statement.
func (s *Scanner) Scan() (string, error) {
var buf bytes.Buffer
seekSemi := true
for {
ch := s.read()
if ch == eof {
return "", io.EOF
}
// Store the character.
_, _ = buf.WriteRune(ch)
if ch == '\'' || ch == '"' {
if s.c.empty() {
s.c.push(ch)
seekSemi = false
} else if s.c.peek() != ch {
s.c.push(ch)
seekSemi = false
} else {
s.c.pop()
if s.c.empty() {
seekSemi = true
}
}
} else if ch == ';' && seekSemi {
break
}
}
return strings.Trim(strings.TrimRight(buf.String(), ";"), "\n"), nil
}
// eof represents a marker rune for the end of the reader.
var eof = rune(0)

@ -1,216 +0,0 @@
package sql
import (
"bytes"
"io"
"strings"
"testing"
)
func Test_stackEmpty(t *testing.T) {
s := newStack()
if !s.empty() {
t.Fatal("new stack is not empty")
}
if s.peek() != rune(0) {
t.Fatal("peek of empty stack does not return correct value")
}
}
func Test_stackSingle(t *testing.T) {
s := newStack()
s.push('x')
if s.empty() {
t.Fatal("non-empty stack marked as empty")
}
if s.peek() != 'x' {
t.Fatal("peek of stack with single entry does not return correct value")
}
if s.pop() != 'x' {
t.Fatal("pop of stack with single entry does not return correct value")
}
if !s.empty() {
t.Fatal("popped stack is not empty")
}
}
func Test_stackMulti(t *testing.T) {
s := newStack()
s.push('x')
s.push('y')
s.push('z')
if s.pop() != 'z' {
t.Fatal("pop of 1st multi stack does not return correct value")
}
if s.pop() != 'y' {
t.Fatal("pop of 2nd multi stack does not return correct value")
}
if s.pop() != 'x' {
t.Fatal("pop of 3rd multi stack does not return correct value")
}
if !s.empty() {
t.Fatal("popped mstack is not empty")
}
}
func Test_ScannerNew(t *testing.T) {
s := NewScanner(nil)
if s == nil {
t.Fatalf("failed to create basic Scanner")
}
}
func Test_ScannerEmpty(t *testing.T) {
r := bytes.NewBufferString("")
s := NewScanner(r)
_, err := s.Scan()
if err != io.EOF {
t.Fatal("Scan of empty string did not return EOF")
}
}
func Test_ScannerSemi(t *testing.T) {
r := bytes.NewBufferString(";")
s := NewScanner(r)
l, err := s.Scan()
if err != nil {
t.Fatal("Scan of single semicolon failed")
}
if l != "" {
t.Fatal("Scan of single semicolon returned incorrect value")
}
_, err = s.Scan()
if err != io.EOF {
t.Fatal("Scan of empty string after semicolon did not return EOF")
}
}
func Test_ScannerSingleStatement(t *testing.T) {
r := bytes.NewBufferString("SELECT * FROM foo;")
s := NewScanner(r)
l, err := s.Scan()
if err != nil {
t.Fatal("Scan of single statement failed")
}
if l != "SELECT * FROM foo" {
t.Fatal("Scan of single statement returned incorrect value")
}
_, err = s.Scan()
if err != io.EOF {
t.Fatal("Scan of empty string after statement did not return EOF")
}
}
func Test_ScannerSingleStatementQuotes(t *testing.T) {
r := bytes.NewBufferString(`SELECT * FROM "foo";`)
s := NewScanner(r)
l, err := s.Scan()
if err != nil {
t.Fatal("Scan of single statement failed")
}
if l != `SELECT * FROM "foo"` {
t.Fatal("Scan of single statement returned incorrect value")
}
_, err = s.Scan()
if err != io.EOF {
t.Fatal("Scan of empty string after statement did not return EOF")
}
}
func Test_ScannerSingleStatementQuotesEmbedded(t *testing.T) {
r := bytes.NewBufferString(`SELECT * FROM ";SELECT * FROM '"foo"'";`)
s := NewScanner(r)
l, err := s.Scan()
if err != nil {
t.Fatal("Scan of single statement failed")
}
if l != `SELECT * FROM ";SELECT * FROM '"foo"'"` {
t.Fatal("Scan of single statement returned incorrect value")
}
_, err = s.Scan()
if err != io.EOF {
t.Fatal("Scan of empty string after statement did not return EOF")
}
}
func Test_ScannerMultiStatement(t *testing.T) {
e := []string{`SELECT * FROM foo;`, `SELECT * FROM bar;`}
r := bytes.NewBufferString(strings.Join(e, ""))
s := NewScanner(r)
for i := range e {
l, err := s.Scan()
if err != nil {
t.Fatal("Scan of multi statement failed")
}
if l != strings.Trim(e[i], "\n;") {
t.Fatalf("Scan of multi statement returned incorrect value, exp %s, got %s", e[i], l)
}
}
}
func Test_ScannerMultiStatementQuotesEmbedded(t *testing.T) {
e := []string{`SELECT * FROM "foo;barx";`, `SELECT * FROM bar;`}
r := bytes.NewBufferString(strings.Join(e, ""))
s := NewScanner(r)
for i := range e {
l, err := s.Scan()
if err != nil {
t.Fatal("Scan of multi statement failed")
}
if l != strings.Trim(e[i], "\n;") {
t.Fatalf("Scan of multi statement returned incorrect value, exp %s, got %s", e[i], l)
}
}
}
// XX I am missing this case: '"' ????
func Test_ScannerMultiLine(t *testing.T) {
stmt := `CREATE TABLE [Customer]
(
[CustomerId] INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
[FirstName] NVARCHAR(40) NOT NULL,
[LastName] NVARCHAR(20) NOT NULL,
[Company] NVARCHAR(80),
[Address] NVARCHAR(70),
[City] NVARCHAR(40),
[State] NVARCHAR(40),
[Country] NVARCHAR(40),
[PostalCode] NVARCHAR(10),
[Phone] NVARCHAR(24),
[Fax] NVARCHAR(24),
[Email] NVARCHAR(60) NOT NULL,
[SupportRepId] INTEGER,
FOREIGN KEY ([SupportRepId]) REFERENCES [Employee] ([EmployeeId])
ON DELETE NO ACTION ON UPDATE NO ACTION
);`
r := bytes.NewBufferString(stmt)
s := NewScanner(r)
l, err := s.Scan()
if err != nil {
t.Fatal("Scan of multiline statement failed")
}
if l != strings.Trim(stmt, "\n;") {
t.Fatal("Scan of multiline statement returned incorrect value")
}
_, err = s.Scan()
if err != io.EOF {
t.Fatal("Scan of empty string after statement did not return EOF")
}
}

@ -21,7 +21,6 @@ import (
"github.com/hashicorp/raft"
"github.com/hashicorp/raft-boltdb"
sql "github.com/rqlite/rqlite/db"
parser "github.com/rqlite/rqlite/sql"
)
var (
@ -451,40 +450,6 @@ func (s *Store) Execute(queries []string, timings, tx bool) ([]*sql.Result, erro
return r.results, r.error
}
// Load loads a SQLite .dump state from a reader.
func (s *Store) Load(r io.Reader) (int, error) {
if s.raft.State() != raft.Leader {
return 0, ErrNotLeader
}
// Read the dump, executing the commands.
var queries []string
scanner := parser.NewScanner(r)
for {
cmd, err := scanner.Scan()
if err != nil && err != io.EOF {
return len(queries), err
}
if cmd == "" {
if err == io.EOF {
break
}
continue
}
queries = append(queries, cmd)
}
if len(queries) > 0 {
_, err := s.Execute(queries, false, false)
if err != nil {
return len(queries), err
}
}
return len(queries), nil
}
// Backup return a snapshot of the underlying database.
//
// If leader is true, this operation is performed with a read consistency

@ -1,7 +1,6 @@
package store
import (
"bytes"
"encoding/json"
"io/ioutil"
"net"
@ -222,13 +221,9 @@ CREATE TABLE foo (id integer not null primary key, name text);
INSERT INTO "foo" VALUES(1,'fiona');
COMMIT;
`
buf := bytes.NewBufferString(dump)
n, err := s.Load(buf)
_, err := s.Execute([]string{dump}, false, false)
if err != nil {
t.Fatalf("failed to load dump: %s", err.Error())
}
if n != 5 {
t.Fatal("wrong number of statements loaded")
t.Fatalf("failed to load simple dump: %s", err.Error())
}
// Check that data were loaded correctly.
@ -244,7 +239,7 @@ COMMIT;
}
}
func Test_SingleNodeLoadBatchLargeBlank(t *testing.T) {
func Test_SingleNodeSingleCommandTrigger(t *testing.T) {
s := mustNewStore(true)
defer os.RemoveAll(s.Path())
@ -256,31 +251,30 @@ func Test_SingleNodeLoadBatchLargeBlank(t *testing.T) {
dump := `PRAGMA foreign_keys=OFF;
BEGIN TRANSACTION;
CREATE TABLE foo (id integer not null primary key, name text);
INSERT INTO "foo" VALUES(1,'fiona');
CREATE TABLE foo (id integer primary key asc, name text);
INSERT INTO "foo" VALUES(1,'bob');
INSERT INTO "foo" VALUES(2,'alice');
INSERT INTO "foo" VALUES(3,'eve');
CREATE TABLE bar (nameid integer, age integer);
INSERT INTO "bar" VALUES(1,44);
INSERT INTO "bar" VALUES(2,46);
INSERT INTO "bar" VALUES(3,8);
CREATE VIEW foobar as select name as Person, Age as age from foo inner join bar on foo.id == bar.nameid;
CREATE TRIGGER new_foobar instead of insert on foobar begin insert into foo (name) values (new.Person); insert into bar (nameid, age) values ((select id from foo where name == new.Person), new.Age); end;
COMMIT;
`
buf := bytes.NewBufferString(dump)
n, err := s.Load(buf)
_, err := s.Execute([]string{dump}, false, false)
if err != nil {
t.Fatalf("failed to load dump: %s", err.Error())
}
if n != 5 {
t.Fatal("wrong number of statements loaded, exp: 2, got: ", n)
t.Fatalf("failed to load dump with trigger: %s", err.Error())
}
// Check that data were loaded correctly.
r, err := s.Query([]string{`SELECT * FROM foo`}, false, true, Strong)
// Check that the VIEW and TRIGGER are OK by using both.
r, err := s.Execute([]string{`INSERT INTO foobar VALUES('jason', 16)`}, false, true)
if err != nil {
t.Fatalf("failed to query single node: %s", err.Error())
}
if exp, got := `["id","name"]`, asJSON(r[0].Columns); exp != got {
t.Fatalf("unexpected results for query\nexp: %s\ngot: %s", exp, got)
t.Fatalf("failed to insert into view on single node: %s", err.Error())
}
if exp, got := `[[1,"fiona"]]`, asJSON(r[0].Values); exp != got {
t.Fatalf("unexpected results for query\nexp: %s\ngot: %s", exp, got)
if exp, got := int64(3), r[0].LastInsertID; exp != got {
t.Fatalf("unexpected results for query\nexp: %d\ngot: %d", exp, got)
}
}
@ -298,13 +292,9 @@ func Test_SingleNodeLoadNoStatements(t *testing.T) {
BEGIN TRANSACTION;
COMMIT;
`
buf := bytes.NewBufferString(dump)
n, err := s.Load(buf)
_, err := s.Execute([]string{dump}, false, false)
if err != nil {
t.Fatalf("failed to load dump: %s", err.Error())
}
if n != 3 {
t.Fatal("wrong number of statements loaded, exp: 1, got: ", n)
t.Fatalf("failed to load dump with no commands: %s", err.Error())
}
}
@ -318,13 +308,10 @@ func Test_SingleNodeLoadEmpty(t *testing.T) {
defer s.Close(true)
s.WaitForLeader(10 * time.Second)
buf := bytes.NewBufferString("")
n, err := s.Load(buf)
dump := ``
_, err := s.Execute([]string{dump}, false, false)
if err != nil {
t.Fatalf("failed to load dump: %s", err.Error())
}
if n != 0 {
t.Fatal("wrong number of statements loaded")
t.Fatalf("failed to load empty dump: %s", err.Error())
}
}
@ -338,10 +325,9 @@ func Test_SingleNodeLoadChinook(t *testing.T) {
defer s.Close(true)
s.WaitForLeader(10 * time.Second)
buf := bytes.NewBufferString(chinook.DB)
_, err := s.Load(buf)
_, err := s.Execute([]string{chinook.DB}, false, false)
if err != nil {
t.Fatalf("failed to load dump: %s", err.Error())
t.Fatalf("failed to load chinook dump: %s", err.Error())
}
// Check that data were loaded correctly.

Loading…
Cancel
Save