Integrate SQL rewrite with rqlite for RANDOM (#1046)
Philip O'Toole
2 years ago
committed by
GitHub
parent
66b3c024dd
commit
95dfead226
@ -0,0 +1,62 @@
|
||||
# rqlite and Non-deterministic Functions
|
||||
|
||||
## Contents
|
||||
* [Understanding the problem](#understanding-the-problem)
|
||||
* [How rqlite solves this problem](#how-rqlite-solves-this-problem)
|
||||
* [What does rqlite rewrite?](#what-does-rqlite-rewrite)
|
||||
* [RANDOM()](#random)
|
||||
* [Date and time functions](#date-and-time-functions)
|
||||
* [Credits](#credits)
|
||||
|
||||
## Understanding the problem
|
||||
rqlite peforms _statement-based replication_. This means that every SQL statement is stored in the Raft log exactly in the form it was received. Each rqlite node then reads the Raft log and applies the SQL statements it finds there to its own local copy of SQLite.
|
||||
|
||||
But if a SQL statement contains a [non-deterministic function](https://www.sqlite.org/deterministic.html), this type of replication can result in different SQLite data under each node -- which is not meant to happen. For example, the following statement could result in a different SQLite database under each node:
|
||||
```
|
||||
INSERT INTO foo (n) VALUES(random());
|
||||
```
|
||||
This is because `RANDOM()` is evaluated by each node independently, and `RANDOM()` will almost certainly return a different value on each node.
|
||||
|
||||
## How rqlite solves this problem
|
||||
An rqlite node addresses this issue by _rewriting_ received SQL statements that contain certain non-deterministic functions, before sending the statement to any other node.
|
||||
|
||||
## What does rqlite rewrite?
|
||||
|
||||
### `RANDOM()`
|
||||
> :warning: **This functionality was introduced in version 7.7.0. It does not exist in earlier releases.**
|
||||
|
||||
Any SQL statement containing `RANDOM()` is rewritten under any of the following circumstances:
|
||||
- the statement is part of a write-request i.e. the request is sent to the `/db/execute` HTTP API.
|
||||
- the statement is part of a read-request i.e. the request is sent to the `/db/execute` HTTP API **and** the read-request is made with _strong_ read consistency.
|
||||
- `RANDOM()` is not used as an `ORDER BY` qualifier.
|
||||
|
||||
`RANDOM()` is replaced with a random integer between -9223372036854775808 and +9223372036854775807.
|
||||
|
||||
#### Examples
|
||||
```bash
|
||||
# Will be rewritten
|
||||
curl -XPOST 'localhost:4001/db/execute' -H "Content-Type: application/json" -d '[
|
||||
"INSERT INTO foo(id, age) VALUES(1234, RANDOM())"
|
||||
]'
|
||||
|
||||
# RANDOM() rewriting explicitly disabled at request-time
|
||||
curl -XPOST 'localhost:4001/db/execute?norwrandom' -H "Content-Type: application/json" -d '[
|
||||
"INSERT INTO foo(id, age) VALUES(1234, RANDOM())"
|
||||
]'
|
||||
|
||||
# Not rewritten
|
||||
curl -G 'localhost:4001/db/query' --data-urlencode 'q=SELECT * FROM foo WHERE id = RANDOM()'
|
||||
|
||||
# Rewritten
|
||||
curl -G 'localhost:4001/db/query?level=strong' --data-urlencode 'q=SELECT * FROM foo WHERE id = RANDOM()'
|
||||
```
|
||||
|
||||
### Date and time functions
|
||||
rqlite does not yet rewrite [SQLite date and time functions](https://www.sqlite.org/lang_datefunc.html) that are non-deterministic in nature. A example of such a function is
|
||||
|
||||
`INSERT INTO datetime_text (d1, d2) VALUES(datetime('now'),datetime('now', 'localtime'))`
|
||||
|
||||
Using such functions will result in undefined behavior. Date and time functions that use absolute values will work without issue.
|
||||
|
||||
## Credits
|
||||
Many thanks to [Ben Johnson](https://github.com/benbjohnson) who wrote the SQLite parser used by rqlite.
|
||||
@ -0,0 +1,37 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/rqlite/sql"
|
||||
)
|
||||
|
||||
// Rewrite rewrites the statements such that RANDOM is rewritten,
|
||||
// if r is true.
|
||||
func Rewrite(stmts []*Statement, r bool) error {
|
||||
if !r {
|
||||
return nil
|
||||
}
|
||||
|
||||
rw := &sql.Rewriter{
|
||||
RewriteRand: r,
|
||||
}
|
||||
|
||||
for i := range stmts {
|
||||
// Only replace the incoming statement with a rewritten version if
|
||||
// there was no error, or if the rewriter did anything. If the statement
|
||||
// is bad SQLite syntax, let SQLite deal with it -- and let its error
|
||||
// be returned. Those errors will probably be clearer.
|
||||
s, err := sql.NewParser(strings.NewReader(stmts[i].Sql)).ParseStatement()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
s, f, err := rw.Do(s)
|
||||
if err != nil || !f {
|
||||
continue
|
||||
}
|
||||
|
||||
stmts[i].Sql = s.String()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -0,0 +1,81 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func Test_NoRewrites(t *testing.T) {
|
||||
for _, str := range []string{
|
||||
`INSERT INTO "names" VALUES (1, 'bob', '123-45-678')`,
|
||||
`INSERT INTO "names" VALUES (RANDOM(), 'bob', '123-45-678')`,
|
||||
`SELECT title FROM albums ORDER BY RANDOM()`,
|
||||
`INSERT INTO foo(name, age) VALUES(?, ?)`,
|
||||
} {
|
||||
|
||||
stmts := []*Statement{
|
||||
{
|
||||
Sql: str,
|
||||
},
|
||||
}
|
||||
if err := Rewrite(stmts, false); err != nil {
|
||||
t.Fatalf("failed to not rewrite: %s", err)
|
||||
}
|
||||
if stmts[0].Sql != str {
|
||||
t.Fatalf("SQL is modified: %s", stmts[0].Sql)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func Test_NoRewritesMulti(t *testing.T) {
|
||||
stmts := []*Statement{
|
||||
{
|
||||
Sql: `INSERT INTO "names" VALUES (1, 'bob', '123-45-678')`,
|
||||
},
|
||||
{
|
||||
Sql: `INSERT INTO "names" VALUES (RANDOM(), 'bob', '123-45-678')`,
|
||||
},
|
||||
{
|
||||
Sql: `SELECT title FROM albums ORDER BY RANDOM()`,
|
||||
},
|
||||
}
|
||||
if err := Rewrite(stmts, false); err != nil {
|
||||
t.Fatalf("failed to not rewrite: %s", err)
|
||||
}
|
||||
if len(stmts) != 3 {
|
||||
t.Fatalf("returned stmts is wrong length: %d", len(stmts))
|
||||
}
|
||||
if stmts[0].Sql != `INSERT INTO "names" VALUES (1, 'bob', '123-45-678')` {
|
||||
t.Fatalf("SQL is modified: %s", stmts[0].Sql)
|
||||
}
|
||||
if stmts[1].Sql != `INSERT INTO "names" VALUES (RANDOM(), 'bob', '123-45-678')` {
|
||||
t.Fatalf("SQL is modified: %s", stmts[0].Sql)
|
||||
}
|
||||
if stmts[2].Sql != `SELECT title FROM albums ORDER BY RANDOM()` {
|
||||
t.Fatalf("SQL is modified: %s", stmts[0].Sql)
|
||||
}
|
||||
}
|
||||
|
||||
func Test_Rewrites(t *testing.T) {
|
||||
testSQLs := []string{
|
||||
`INSERT INTO "names" VALUES (1, 'bob', '123-45-678')`, `INSERT INTO "names" VALUES \(1, 'bob', '123-45-678'\)`,
|
||||
`INSERT INTO "names" VALUES (RANDOM(), 'bob', '123-45-678')`, `INSERT INTO "names" VALUES \(-?[0-9]+, 'bob', '123-45-678'\)`,
|
||||
`SELECT title FROM albums ORDER BY RANDOM()`, `SELECT title FROM albums ORDER BY RANDOM\(\)`,
|
||||
`SELECT RANDOM()`, `SELECT -?[0-9]+`,
|
||||
}
|
||||
for i := 0; i < len(testSQLs)-1; i += 2 {
|
||||
stmts := []*Statement{
|
||||
{
|
||||
Sql: testSQLs[i],
|
||||
},
|
||||
}
|
||||
if err := Rewrite(stmts, true); err != nil {
|
||||
t.Fatalf("failed to not rewrite: %s", err)
|
||||
}
|
||||
|
||||
match := regexp.MustCompile(testSQLs[i+1])
|
||||
if !match.MatchString(stmts[0].Sql) {
|
||||
t.Fatalf("test %d failed, %s (rewritten as %s) does not regex-match with %s", i, testSQLs[i], stmts[0].Sql, testSQLs[i+1])
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue