From 488a9554eae30fc070809fb11f3277d8a88974bd Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Thu, 28 Jan 2021 22:00:34 -0500 Subject: [PATCH 1/5] Switch to rqlite fork of go-sqlite3 --- CONTRIBUTING.md | 2 +- db/db.go | 2 +- go.mod | 5 +++-- go.sum | 3 +++ 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7d5b3db0..b99c5ca7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -58,7 +58,7 @@ protoc -I=$SRC_DIR --go_out=$DEST_DIR $SRC_DIR/command.proto It can be rather slow to rebuild rqlite, due to the repeated compilation of the SQLite source code. You can compile and install the SQLite libary once, so subsequent builds are much faster. To do so, execute the following commands: ```bash cd $GOPATH -go install github.com/mattn/go-sqlite3 +go install github.com/rqlite/go-sqlite3 ``` ## Cloning a fork diff --git a/db/db.go b/db/db.go index f85ae8a3..ea00f84f 100644 --- a/db/db.go +++ b/db/db.go @@ -11,7 +11,7 @@ import ( "strings" "time" - "github.com/mattn/go-sqlite3" + "github.com/rqlite/go-sqlite3" "github.com/rqlite/rqlite/command" ) diff --git a/go.mod b/go.mod index f0e0698a..b11fe6ef 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.13 require ( github.com/Bowery/prompt v0.0.0-20190916142128-fa8279994f75 - github.com/armon/go-metrics v0.3.5 // indirect + github.com/armon/go-metrics v0.3.6 // indirect github.com/fatih/color v1.10.0 // indirect github.com/golang/protobuf v1.4.3 github.com/hashicorp/go-hclog v0.15.0 // indirect @@ -17,9 +17,10 @@ require ( github.com/mattn/go-sqlite3 v2.0.3+incompatible github.com/mkideal/cli v0.2.3 github.com/mkideal/pkg v0.1.2 + github.com/rqlite/go-sqlite3 v1.15.0 golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad golang.org/x/net v0.0.0-20200707034311-ab3426394381 - golang.org/x/sys v0.0.0-20201223074533-0d417f636930 // indirect + golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf // indirect google.golang.org/protobuf v1.25.0 ) diff --git a/go.sum b/go.sum index 16028595..f4f7c859 100644 --- a/go.sum +++ b/go.sum @@ -19,6 +19,7 @@ github.com/armon/go-metrics v0.3.4 h1:Xqf+7f2Vhl9tsqDYmXhnXInUdcrtgpRNpIA15/uldS github.com/armon/go-metrics v0.3.4/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-metrics v0.3.5 h1:uq4txK6NAUvLQ60rotN+K+JuTnf3XP4TdQmcs9ma5mk= github.com/armon/go-metrics v0.3.5/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= +github.com/armon/go-metrics v0.3.6/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= @@ -192,6 +193,7 @@ github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rqlite/go-sqlite3 v1.15.0/go.mod h1:ml55MVv28UP7V8zrxILd2EsrI6Wfsz76YSskpg08Ut4= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -268,6 +270,7 @@ golang.org/x/sys v0.0.0-20201116194326-cc9327a14d48/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201223074533-0d417f636930 h1:vRgIt+nup/B/BwIS0g2oC0haq0iqbV3ZA+u6+0TlNCo= golang.org/x/sys v0.0.0-20201223074533-0d417f636930/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201113234701-d7a72108b828 h1:htWEtQEuEVJ4tU/Ngx7Cd/4Q7e3A5Up1owgyBtVsTwk= golang.org/x/term v0.0.0-20201113234701-d7a72108b828/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 h1:/ZHdbVpdR/jk3g30/d4yUL0JU9kksj8+F/bnQUVLGDM= From 93a78f09a0ce8da3ccad7c0d1a827b34eb96aebd Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Thu, 28 Jan 2021 22:45:26 -0500 Subject: [PATCH 2/5] Add call to Serialize function --- db/db.go | 13 +++++++++++++ go.sum | 3 +++ store/store.go | 2 +- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/db/db.go b/db/db.go index ea00f84f..b2052225 100644 --- a/db/db.go +++ b/db/db.go @@ -454,6 +454,19 @@ func (db *DB) Backup(path string) error { return err } +// Serialize returns a byte slice representation of the SQLite database. For +// an ordinary on-disk database file, the serialization is just a copy of the +// disk file. For an in-memory database or a "TEMP" database, the serialization +// is the same sequence of bytes which would be written to disk if that database +// were backed up to disk. +func (db *DB) Serialize() ([]byte, error) { + b := db.sqlite3conn.Serialize("") + if b == nil { + return nil, fmt.Errorf("failed to serialize database") + } + return b, nil +} + // Dump writes a consistent snapshot of the database in SQL text format. func (db *DB) Dump(w io.Writer) error { if _, err := w.Write([]byte("PRAGMA foreign_keys=OFF;\nBEGIN TRANSACTION;\n")); err != nil { diff --git a/go.sum b/go.sum index f4f7c859..5aa442c2 100644 --- a/go.sum +++ b/go.sum @@ -19,6 +19,7 @@ github.com/armon/go-metrics v0.3.4 h1:Xqf+7f2Vhl9tsqDYmXhnXInUdcrtgpRNpIA15/uldS github.com/armon/go-metrics v0.3.4/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-metrics v0.3.5 h1:uq4txK6NAUvLQ60rotN+K+JuTnf3XP4TdQmcs9ma5mk= github.com/armon/go-metrics v0.3.5/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= +github.com/armon/go-metrics v0.3.6 h1:x/tmtOF9cDBoXH7XoAGOz2qqm1DknFD1590XmD/DUJ8= github.com/armon/go-metrics v0.3.6/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -193,6 +194,7 @@ github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rqlite/go-sqlite3 v1.15.0 h1:O0EjJwzTgNSwSAhZqMJGQr0GL9F4SN67AufMydMUdZk= github.com/rqlite/go-sqlite3 v1.15.0/go.mod h1:ml55MVv28UP7V8zrxILd2EsrI6Wfsz76YSskpg08Ut4= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -270,6 +272,7 @@ golang.org/x/sys v0.0.0-20201116194326-cc9327a14d48/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201223074533-0d417f636930 h1:vRgIt+nup/B/BwIS0g2oC0haq0iqbV3ZA+u6+0TlNCo= golang.org/x/sys v0.0.0-20201223074533-0d417f636930/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201113234701-d7a72108b828 h1:htWEtQEuEVJ4tU/Ngx7Cd/4Q7e3A5Up1owgyBtVsTwk= golang.org/x/term v0.0.0-20201113234701-d7a72108b828/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= diff --git a/store/store.go b/store/store.go index 5e3860dc..cbf92251 100644 --- a/store/store.go +++ b/store/store.go @@ -936,7 +936,7 @@ func (s *Store) Database(leader bool) ([]byte, error) { s.mu.Lock() defer s.mu.Unlock() - f, err := ioutil.TempFile("", "rqlilte-snap-") + f, err := ioutil.TempFile("", "rqlite-snap-") if err != nil { return nil, err } From 6fad9a6a80316b5095ddc141ed4194875d42ba8f Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Thu, 28 Jan 2021 22:55:30 -0500 Subject: [PATCH 3/5] Finally get go-sqlite3 fork working --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index b11fe6ef..3d19cb78 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/mattn/go-sqlite3 v2.0.3+incompatible github.com/mkideal/cli v0.2.3 github.com/mkideal/pkg v0.1.2 - github.com/rqlite/go-sqlite3 v1.15.0 + github.com/rqlite/go-sqlite3 v1.18.0 golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad golang.org/x/net v0.0.0-20200707034311-ab3426394381 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect diff --git a/go.sum b/go.sum index 5aa442c2..a5fbf9ec 100644 --- a/go.sum +++ b/go.sum @@ -196,6 +196,8 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rqlite/go-sqlite3 v1.15.0 h1:O0EjJwzTgNSwSAhZqMJGQr0GL9F4SN67AufMydMUdZk= github.com/rqlite/go-sqlite3 v1.15.0/go.mod h1:ml55MVv28UP7V8zrxILd2EsrI6Wfsz76YSskpg08Ut4= +github.com/rqlite/go-sqlite3 v1.18.0 h1:nRIqbrChAY87Re6XwbqjJyzzeijnY5Y1x/SJn0qUk6Q= +github.com/rqlite/go-sqlite3 v1.18.0/go.mod h1:ml55MVv28UP7V8zrxILd2EsrI6Wfsz76YSskpg08Ut4= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= From 92102974139c374752676d1a7fcf22e828b76485 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Thu, 28 Jan 2021 23:00:55 -0500 Subject: [PATCH 4/5] Unit test DB serialize --- db/db_test.go | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/db/db_test.go b/db/db_test.go index a15e6240..63717584 100644 --- a/db/db_test.go +++ b/db/db_test.go @@ -961,6 +961,70 @@ func Test_Backup(t *testing.T) { } } +func Test_Serialize(t *testing.T) { + db, path := mustCreateDatabase() + defer db.Close() + defer os.Remove(path) + + _, err := db.ExecuteStringStmt("CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)") + if err != nil { + t.Fatalf("failed to create table: %s", err.Error()) + } + + req := &command.Request{ + Transaction: true, + Statements: []*command.Statement{ + { + Sql: `INSERT INTO foo(id, name) VALUES(1, "fiona")`, + }, + { + Sql: `INSERT INTO foo(id, name) VALUES(2, "fiona")`, + }, + { + Sql: `INSERT INTO foo(id, name) VALUES(3, "fiona")`, + }, + { + Sql: `INSERT INTO foo(id, name) VALUES(4, "fiona")`, + }, + }, + } + _, err = db.Execute(req, false) + if err != nil { + t.Fatalf("failed to insert records: %s", err.Error()) + } + + dstDB, err := ioutil.TempFile("", "rqlite-bak-") + if err != nil { + t.Fatalf("failed to create temp file: %s", err.Error()) + } + dstDB.Close() + defer os.Remove(dstDB.Name()) + + // Get the bytes, and write to a temp file. + b, err := db.Serialize() + if err != nil { + t.Fatalf("failed to serialize database: %s", err.Error()) + } + err = ioutil.WriteFile(dstDB.Name(), b, 0644) + if err != nil { + t.Fatalf("failed to write serialized database to file: %s", err.Error()) + } + + newDB, err := Open(dstDB.Name()) + if err != nil { + t.Fatalf("failed to open backup database: %s", err.Error()) + } + defer newDB.Close() + defer os.Remove(dstDB.Name()) + ro, err := newDB.QueryStringStmt(`SELECT * FROM foo`) + if err != nil { + t.Fatalf("failed to query table: %s", err.Error()) + } + if exp, got := `[{"columns":["id","name"],"types":["integer","text"],"values":[[1,"fiona"],[2,"fiona"],[3,"fiona"],[4,"fiona"]]}]`, asJSON(ro); exp != got { + t.Fatalf("unexpected results for query\nexp: %s\ngot: %s", exp, got) + } +} + func Test_Dump(t *testing.T) { t.Parallel() @@ -1012,7 +1076,7 @@ func Test_DumpMemory(t *testing.T) { func mustCreateDatabase() (*DB, string) { var err error - f, err := ioutil.TempFile("", "rqlilte-test-") + f, err := ioutil.TempFile("", "rqlite-test-") if err != nil { panic("failed to create temp file") } From db64f4c9235ec0b35d2e59ff2f8683880f7e859a Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Thu, 28 Jan 2021 23:03:02 -0500 Subject: [PATCH 5/5] Don't use temp file during database snapshot Snapshotting the database does not now need to write to disk, and copy the SQLite database straight to RAM, for handoff to the Raft subsystem. --- store/store.go | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/store/store.go b/store/store.go index cbf92251..ada86514 100644 --- a/store/store.go +++ b/store/store.go @@ -935,19 +935,7 @@ func (s *Store) Database(leader bool) ([]byte, error) { // Ensure only one snapshot can take place at once, and block all queries. s.mu.Lock() defer s.mu.Unlock() - - f, err := ioutil.TempFile("", "rqlite-snap-") - if err != nil { - return nil, err - } - f.Close() - defer os.Remove(f.Name()) - - if err := s.db.Backup(f.Name()); err != nil { - return nil, err - } - - return ioutil.ReadFile(f.Name()) + return s.db.Serialize() } // Snapshot returns a snapshot of the database. The caller must ensure that