From 6ee9cdbbdf827a93b43f3993575c3ed8a4e80870 Mon Sep 17 00:00:00 2001 From: Philip O Toole Date: Thu, 10 Nov 2016 12:25:08 -0800 Subject: [PATCH 1/6] Start integration of SQL line parser --- store/store.go | 24 ++++++++---------------- store/store_test.go | 2 +- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/store/store.go b/store/store.go index 4e9dfc09..bd0973d6 100644 --- a/store/store.go +++ b/store/store.go @@ -4,7 +4,6 @@ package store import ( - "bufio" "bytes" "encoding/binary" "encoding/json" @@ -16,13 +15,13 @@ import ( "net" "os" "path/filepath" - "strings" "sync" "time" "github.com/hashicorp/raft" "github.com/hashicorp/raft-boltdb" sql "github.com/rqlite/rqlite/db" + parser "github.com/rqlite/rqlite/sql" ) var ( @@ -460,21 +459,14 @@ func (s *Store) Load(r io.Reader, sz int) (int64, error) { // Read the dump, executing the commands. var queries []string var n int64 - buf := bufio.NewReader(r) + scanner := parser.NewScanner(r) for { - cmd, err := buf.ReadString('\n') - if err != nil && err != io.EOF { - return 0, err - } - cmd = strings.TrimRight(cmd, "\n;") - if cmd == "PRAGMA foreign_keys=OFF" || - cmd == "BEGIN TRANSACTION" || - cmd == "COMMIT" { - continue - } - - if cmd == "" && err == io.EOF { - break + cmd, err := scanner.Scan() + if err != nil { + if cmd == "" && err == io.EOF { + break + } + return n, err } queries = append(queries, cmd) diff --git a/store/store_test.go b/store/store_test.go index b2fb9348..17b14458 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -264,7 +264,7 @@ COMMIT; t.Fatalf("failed to load dump: %s", err.Error()) } if n != 2 { - t.Fatal("wrong number of statements loaded") + t.Fatalf("wrong number of statements loaded, exp %d, got %d", 2, n) } // Check that data were loaded correctly. From b87de293cec53d258c7b297a03dcc1d48b8aef32 Mon Sep 17 00:00:00 2001 From: Philip O Toole Date: Thu, 10 Nov 2016 12:33:52 -0800 Subject: [PATCH 2/6] Move \n ; trimming to SQL line parser --- sql/parser.go | 3 ++- sql/parser_test.go | 14 +++++++------- store/store.go | 14 ++++++++++---- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/sql/parser.go b/sql/parser.go index 00c86203..c5d053c2 100644 --- a/sql/parser.go +++ b/sql/parser.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "io" + "strings" ) // stack represents a stack. @@ -104,7 +105,7 @@ func (s *Scanner) Scan() (string, error) { } } - return buf.String(), nil + return strings.TrimRight(buf.String(), "\n;"), nil } // eof represents a marker rune for the end of the reader. diff --git a/sql/parser_test.go b/sql/parser_test.go index 7fb05a47..2a9c6a07 100644 --- a/sql/parser_test.go +++ b/sql/parser_test.go @@ -84,7 +84,7 @@ func Test_ScannerSemi(t *testing.T) { if err != nil { t.Fatal("Scan of single semicolon failed") } - if l != ";" { + if l != "" { t.Fatal("Scan of single semicolon returned incorrect value") } _, err = s.Scan() @@ -101,7 +101,7 @@ func Test_ScannerSingleStatement(t *testing.T) { if err != nil { t.Fatal("Scan of single statement failed") } - if l != "SELECT * FROM foo;" { + if l != "SELECT * FROM foo" { t.Fatal("Scan of single statement returned incorrect value") } _, err = s.Scan() @@ -118,7 +118,7 @@ func Test_ScannerSingleStatementQuotes(t *testing.T) { if err != nil { t.Fatal("Scan of single statement failed") } - if l != `SELECT * FROM "foo";` { + if l != `SELECT * FROM "foo"` { t.Fatal("Scan of single statement returned incorrect value") } _, err = s.Scan() @@ -135,7 +135,7 @@ func Test_ScannerSingleStatementQuotesEmbedded(t *testing.T) { if err != nil { t.Fatal("Scan of single statement failed") } - if l != `SELECT * FROM ";SELECT * FROM '"foo"'";` { + if l != `SELECT * FROM ";SELECT * FROM '"foo"'"` { t.Fatal("Scan of single statement returned incorrect value") } _, err = s.Scan() @@ -155,7 +155,7 @@ func Test_ScannerMultiStatement(t *testing.T) { t.Fatal("Scan of multi statement failed") } - if l != e[i] { + if l != strings.Trim(e[i], "\n;") { t.Fatalf("Scan of multi statement returned incorrect value, exp %s, got %s", e[i], l) } } @@ -172,7 +172,7 @@ func Test_ScannerMultiStatementQuotesEmbedded(t *testing.T) { t.Fatal("Scan of multi statement failed") } - if l != e[i] { + if l != strings.Trim(e[i], "\n;") { t.Fatalf("Scan of multi statement returned incorrect value, exp %s, got %s", e[i], l) } } @@ -206,7 +206,7 @@ func Test_ScannerMultiLine(t *testing.T) { if err != nil { t.Fatal("Scan of multiline statement failed") } - if l != stmt { + if l != strings.Trim(stmt, "\n;") { t.Fatal("Scan of multiline statement returned incorrect value") } _, err = s.Scan() diff --git a/store/store.go b/store/store.go index bd0973d6..c06c855d 100644 --- a/store/store.go +++ b/store/store.go @@ -462,12 +462,18 @@ func (s *Store) Load(r io.Reader, sz int) (int64, error) { scanner := parser.NewScanner(r) for { cmd, err := scanner.Scan() - if err != nil { - if cmd == "" && err == io.EOF { - break - } + if err != nil && err != io.EOF { return n, err } + cmd = strings.TrimRight(cmd, "\n;") + if cmd == "PRAGMA foreign_keys=OFF" || + cmd == "BEGIN TRANSACTION" || + cmd == "COMMIT" { + continue + } + if cmd == "" && err == io.EOF { + break + } queries = append(queries, cmd) if len(queries) == sz { From 84f3be9c10f2ec147cd0cc839b1cacf23ac1b830 Mon Sep 17 00:00:00 2001 From: Philip O Toole Date: Thu, 10 Nov 2016 12:38:24 -0800 Subject: [PATCH 3/6] Add missing package --- store/store.go | 1 + 1 file changed, 1 insertion(+) diff --git a/store/store.go b/store/store.go index c06c855d..7d523318 100644 --- a/store/store.go +++ b/store/store.go @@ -15,6 +15,7 @@ import ( "net" "os" "path/filepath" + "strings" "sync" "time" From 1f5cfc65d8e4d8e039ea8fa95adbdd7063572c00 Mon Sep 17 00:00:00 2001 From: Philip O Toole Date: Mon, 14 Nov 2016 11:22:40 -0800 Subject: [PATCH 4/6] Trim leading and trailing newlines --- store/store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/store.go b/store/store.go index 7d523318..6b638630 100644 --- a/store/store.go +++ b/store/store.go @@ -466,7 +466,7 @@ func (s *Store) Load(r io.Reader, sz int) (int64, error) { if err != nil && err != io.EOF { return n, err } - cmd = strings.TrimRight(cmd, "\n;") + cmd = strings.Trim(strings.TrimRight(cmd, ";"), "\n") if cmd == "PRAGMA foreign_keys=OFF" || cmd == "BEGIN TRANSACTION" || cmd == "COMMIT" { From 7f114fd1068636ce118f011b3629e6fad78f2125 Mon Sep 17 00:00:00 2001 From: Philip O Toole Date: Mon, 14 Nov 2016 11:25:13 -0800 Subject: [PATCH 5/6] Trim leading and trailing whitespace --- sql/parser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/parser.go b/sql/parser.go index c5d053c2..5cace8b4 100644 --- a/sql/parser.go +++ b/sql/parser.go @@ -105,7 +105,7 @@ func (s *Scanner) Scan() (string, error) { } } - return strings.TrimRight(buf.String(), "\n;"), nil + return strings.Trim(strings.TrimRight(buf.String(), ";"), "\n"), nil } // eof represents a marker rune for the end of the reader. From 9c180d8f437fbec8d9c7d21850f03b997e58c664 Mon Sep 17 00:00:00 2001 From: Philip O Toole Date: Mon, 14 Nov 2016 11:26:55 -0800 Subject: [PATCH 6/6] SQL parser now strips newlines --- store/store.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/store/store.go b/store/store.go index 6b638630..bfc271a4 100644 --- a/store/store.go +++ b/store/store.go @@ -15,7 +15,6 @@ import ( "net" "os" "path/filepath" - "strings" "sync" "time" @@ -466,7 +465,6 @@ func (s *Store) Load(r io.Reader, sz int) (int64, error) { if err != nil && err != io.EOF { return n, err } - cmd = strings.Trim(strings.TrimRight(cmd, ";"), "\n") if cmd == "PRAGMA foreign_keys=OFF" || cmd == "BEGIN TRANSACTION" || cmd == "COMMIT" {