@ -13,7 +13,6 @@ import (
"time"
"github.com/rqlite/rqlite/command"
"github.com/rqlite/rqlite/command/chunking"
"github.com/rqlite/rqlite/command/encoding"
"github.com/rqlite/rqlite/db"
"github.com/rqlite/rqlite/random"
@ -1249,436 +1248,6 @@ COMMIT;
}
}
// Test_SingleNodeLoadChunkBinary tests that a Store correctly loads data in SQLite
// binary format from a file using chunked loading.
func Test_SingleNodeLoadChunkBinary ( t * testing . T ) {
s , ln := mustNewStore ( t )
defer ln . Close ( )
if err := s . Open ( ) ; err != nil {
t . Fatalf ( "failed to open single-node store: %s" , err . Error ( ) )
}
if err := s . Bootstrap ( NewServer ( s . ID ( ) , s . Addr ( ) , true ) ) ; err != nil {
t . Fatalf ( "failed to bootstrap single-node store: %s" , err . Error ( ) )
}
defer s . Close ( true )
if _ , err := s . WaitForLeader ( 10 * time . Second ) ; err != nil {
t . Fatalf ( "Error waiting for leader: %s" , err )
}
// Load a dataset, to check it's erased by the SQLite file load.
dump := ` PRAGMA foreign_keys = OFF ;
BEGIN TRANSACTION ;
CREATE TABLE bar ( id integer not null primary key , name text ) ;
INSERT INTO "bar" VALUES ( 1 , ' declan ' ) ;
COMMIT ;
`
_ , err := s . Execute ( executeRequestFromString ( dump , false , false ) )
if err != nil {
t . Fatalf ( "failed to load simple dump: %s" , err . Error ( ) )
}
// Check that data were loaded correctly.
qr := queryRequestFromString ( "SELECT * FROM bar" , false , true )
qr . Level = command . QueryRequest_QUERY_REQUEST_LEVEL_STRONG
r , err := s . Query ( qr )
if err != nil {
t . Fatalf ( "failed to query single node: %s" , err . Error ( ) )
}
if exp , got := ` ["id","name"] ` , asJSON ( r [ 0 ] . Columns ) ; exp != got {
t . Fatalf ( "unexpected results for query\nexp: %s\ngot: %s" , exp , got )
}
if exp , got := ` [[1,"declan"]] ` , asJSON ( r [ 0 ] . Values ) ; exp != got {
t . Fatalf ( "unexpected results for query\nexp: %s\ngot: %s" , exp , got )
}
// Open the SQLite file for loading.
f , err := os . Open ( filepath . Join ( "testdata" , "load.sqlite" ) )
if err != nil {
t . Fatalf ( "failed to open SQLite file: %s" , err . Error ( ) )
}
defer f . Close ( )
numSnapshots := s . numSnapshots
chunker := chunking . NewChunker ( f , 2048 )
for {
chunk , err := chunker . Next ( )
if err != nil {
t . Fatalf ( "failed to read next chunk: %s" , err . Error ( ) )
}
err = s . LoadChunk ( chunk )
if err != nil {
t . Fatalf ( "failed to load chunk: %s" , err . Error ( ) )
}
if chunk . IsLast {
break
}
}
// Chunked loading should trigger a snapshot, so check that the snapshot
// exists. Check that numSnapshots is 1
testPoll ( t , func ( ) bool {
s . numSnapshotsMu . Lock ( )
defer s . numSnapshotsMu . Unlock ( )
return s . numSnapshots == numSnapshots + 1
} , 100 * time . Millisecond , 3 * time . Second )
// Check that data were loaded correctly.
qr = queryRequestFromString ( "SELECT * FROM foo WHERE id=2" , false , true )
qr . Level = command . QueryRequest_QUERY_REQUEST_LEVEL_STRONG
r , err = s . Query ( qr )
if err != nil {
t . Fatalf ( "failed to query single node: %s" , err . Error ( ) )
}
if exp , got := ` ["id","name"] ` , asJSON ( r [ 0 ] . Columns ) ; exp != got {
t . Fatalf ( "unexpected results for query\nexp: %s\ngot: %s" , exp , got )
}
if exp , got := ` [[2,"fiona"]] ` , asJSON ( r [ 0 ] . Values ) ; exp != got {
t . Fatalf ( "unexpected results for query\nexp: %s\ngot: %s" , exp , got )
}
qr = queryRequestFromString ( "SELECT count(*) FROM foo" , false , true )
qr . Level = command . QueryRequest_QUERY_REQUEST_LEVEL_STRONG
r , err = s . Query ( qr )
if err != nil {
t . Fatalf ( "failed to query single node: %s" , err . Error ( ) )
}
if exp , got := ` ["count(*)"] ` , asJSON ( r [ 0 ] . Columns ) ; exp != got {
t . Fatalf ( "unexpected results for query\nexp: %s\ngot: %s" , exp , got )
}
if exp , got := ` [[3]] ` , asJSON ( r [ 0 ] . Values ) ; exp != got {
t . Fatalf ( "unexpected results for query\nexp: %s\ngot: %s" , exp , got )
}
// Check pre-existing data is gone.
qr = queryRequestFromString ( "SELECT * FROM bar" , false , true )
qr . Level = command . QueryRequest_QUERY_REQUEST_LEVEL_STRONG
r , err = s . Query ( qr )
if err != nil {
t . Fatalf ( "failed to query single node: %s" , err . Error ( ) )
}
if exp , got := ` { "error":"no such table: bar"} ` , asJSON ( r [ 0 ] ) ; exp != got {
t . Fatalf ( "unexpected results for query\nexp: %s\ngot: %s" , exp , got )
}
}
// Test_SingleNodeLoadChunkBinary tests that a Store reopens with the correct
// state after loading data in SQLite binary format from a file using chunked loading.
// This is important because chunked loading involves forcing the creation of a new
// snapshot.
func Test_SingleNodeLoadChunkBinaryReopen ( t * testing . T ) {
s , ln := mustNewStore ( t )
defer ln . Close ( )
if err := s . Open ( ) ; err != nil {
t . Fatalf ( "failed to open single-node store: %s" , err . Error ( ) )
}
if err := s . Bootstrap ( NewServer ( s . ID ( ) , s . Addr ( ) , true ) ) ; err != nil {
t . Fatalf ( "failed to bootstrap single-node store: %s" , err . Error ( ) )
}
defer s . Close ( true )
if _ , err := s . WaitForLeader ( 10 * time . Second ) ; err != nil {
t . Fatalf ( "Error waiting for leader: %s" , err )
}
// Open the file for loading.
f , err := os . Open ( filepath . Join ( "testdata" , "load.sqlite" ) )
if err != nil {
t . Fatalf ( "failed to open SQLite file: %s" , err . Error ( ) )
}
defer f . Close ( )
numSnapshots := s . numSnapshots
chunker := chunking . NewChunker ( f , 2048 )
for {
chunk , err := chunker . Next ( )
if err != nil {
t . Fatalf ( "failed to read next chunk: %s" , err . Error ( ) )
}
err = s . LoadChunk ( chunk )
if err != nil {
t . Fatalf ( "failed to load chunk: %s" , err . Error ( ) )
}
if chunk . IsLast {
break
}
}
// Chunked loading should trigger a snapshot, so check that the snapshot
// exists. Check that numSnapshots is 1
testPoll ( t , func ( ) bool {
s . numSnapshotsMu . Lock ( )
defer s . numSnapshotsMu . Unlock ( )
return s . numSnapshots == numSnapshots + 1
} , 100 * time . Millisecond , 3 * time . Second )
// Close and re-open the store.
if err := s . Close ( true ) ; err != nil {
t . Fatalf ( "failed to close store: %s" , err . Error ( ) )
}
if err := s . Open ( ) ; err != nil {
t . Fatalf ( "failed to re-open store: %s" , err . Error ( ) )
}
if _ , err := s . WaitForLeader ( 10 * time . Second ) ; err != nil {
t . Fatalf ( "Error waiting for leader: %s" , err )
}
// Check that data were loaded correctly.
qr := queryRequestFromString ( "SELECT COUNT(*) FROM foo" , false , true )
qr . Level = command . QueryRequest_QUERY_REQUEST_LEVEL_STRONG
r , err := s . Query ( qr )
if err != nil {
t . Fatalf ( "failed to query single node: %s" , err . Error ( ) )
}
if exp , got := ` ["COUNT(*)"] ` , asJSON ( r [ 0 ] . Columns ) ; exp != got {
t . Fatalf ( "unexpected results for query\nexp: %s\ngot: %s" , exp , got )
}
if exp , got := ` [[3]] ` , asJSON ( r [ 0 ] . Values ) ; exp != got {
t . Fatalf ( "unexpected results for query\nexp: %s\ngot: %s" , exp , got )
}
// Insert one more row to be sure everything is still working.
er := executeRequestFromString ( "INSERT INTO foo VALUES(4, 'bob')" , false , true )
_ , err = s . Execute ( er )
if err != nil {
t . Fatalf ( "failed to insert into foo: %s" , err . Error ( ) )
}
qr = queryRequestFromString ( "SELECT COUNT(*) FROM foo" , false , true )
qr . Level = command . QueryRequest_QUERY_REQUEST_LEVEL_STRONG
r , err = s . Query ( qr )
if err != nil {
t . Fatalf ( "failed to query single node: %s" , err . Error ( ) )
}
if exp , got := ` ["COUNT(*)"] ` , asJSON ( r [ 0 ] . Columns ) ; exp != got {
t . Fatalf ( "unexpected results for query\nexp: %s\ngot: %s" , exp , got )
}
if exp , got := ` [[4]] ` , asJSON ( r [ 0 ] . Values ) ; exp != got {
t . Fatalf ( "unexpected results for query\nexp: %s\ngot: %s" , exp , got )
}
// Close and re-open the store, and check one last time.
if err := s . Close ( true ) ; err != nil {
t . Fatalf ( "failed to close store: %s" , err . Error ( ) )
}
if err := s . Open ( ) ; err != nil {
t . Fatalf ( "failed to re-open store: %s" , err . Error ( ) )
}
if _ , err := s . WaitForLeader ( 10 * time . Second ) ; err != nil {
t . Fatalf ( "Error waiting for leader: %s" , err )
}
qr = queryRequestFromString ( "SELECT COUNT(*) FROM foo" , false , true )
qr . Level = command . QueryRequest_QUERY_REQUEST_LEVEL_STRONG
r , err = s . Query ( qr )
if err != nil {
t . Fatalf ( "failed to query single node: %s" , err . Error ( ) )
}
if exp , got := ` ["COUNT(*)"] ` , asJSON ( r [ 0 ] . Columns ) ; exp != got {
t . Fatalf ( "unexpected results for query\nexp: %s\ngot: %s" , exp , got )
}
if exp , got := ` [[4]] ` , asJSON ( r [ 0 ] . Values ) ; exp != got {
t . Fatalf ( "unexpected results for query\nexp: %s\ngot: %s" , exp , got )
}
}
// Test_SingleNodeLoadChunk_SnapshotBlock tests that a Store correctly loads data in SQLite
// binary format from a file using chunked loading, and that snapshotting is blocked
// during the load.
func Test_SingleNodeLoadChunk_SnapshotBlock ( t * testing . T ) {
s , ln := mustNewStore ( t )
defer ln . Close ( )
if err := s . Open ( ) ; err != nil {
t . Fatalf ( "failed to open single-node store: %s" , err . Error ( ) )
}
if err := s . Bootstrap ( NewServer ( s . ID ( ) , s . Addr ( ) , true ) ) ; err != nil {
t . Fatalf ( "failed to bootstrap single-node store: %s" , err . Error ( ) )
}
defer s . Close ( true )
if _ , err := s . WaitForLeader ( 10 * time . Second ) ; err != nil {
t . Fatalf ( "Error waiting for leader: %s" , err )
}
getChunker := func ( ) ( * os . File , * chunking . Chunker ) {
// Open the SQLite file for loading.
f , err := os . Open ( filepath . Join ( "testdata" , "load.sqlite" ) )
if err != nil {
t . Fatalf ( "failed to open SQLite file: %s" , err . Error ( ) )
}
return f , chunking . NewChunker ( f , 2048 )
}
// Load the first chunk, which should block snapshotting, complete
// the load, and then check that snapshotting is no longer blocked.
fd , chunker := getChunker ( )
defer fd . Close ( )
chunk , err := chunker . Next ( )
if err != nil {
t . Fatalf ( "failed to read first chunk: %s" , err . Error ( ) )
}
if err := s . LoadChunk ( chunk ) ; err != nil {
t . Fatalf ( "failed to load first chunk: %s" , err . Error ( ) )
}
if s . NumLoadsInProgress ( ) != 1 {
t . Fatalf ( "expected 1 load in progress, got %d" , s . NumLoadsInProgress ( ) )
}
if err := s . Snapshot ( ) ; err != ErrLoadInProgress {
t . Fatalf ( "snapshot should have been blocked" )
}
for {
chunk , err := chunker . Next ( )
if err != nil {
t . Fatalf ( "failed to read next chunk: %s" , err . Error ( ) )
}
err = s . LoadChunk ( chunk )
if err != nil {
t . Fatalf ( "failed to load chunk: %s" , err . Error ( ) )
}
if chunk . IsLast {
break
}
}
if err := s . Snapshot ( ) ; err != nil {
t . Fatalf ( "snapshot should not have been blocked" )
}
if s . NumLoadsInProgress ( ) != 0 {
t . Fatalf ( "expected 0 load in progress, got %d" , s . NumLoadsInProgress ( ) )
}
// Load the first chunk, which should block snapshotting, send an
// Abort chunk, then check that snapshotting is no longer blocked.
fd , chunker = getChunker ( )
defer fd . Close ( )
chunk , err = chunker . Next ( )
if err != nil {
t . Fatalf ( "failed to read first chunk: %s" , err . Error ( ) )
}
if err := s . LoadChunk ( chunk ) ; err != nil {
t . Fatalf ( "failed to load first chunk: %s" , err . Error ( ) )
}
if err := s . Snapshot ( ) ; err != ErrLoadInProgress {
t . Fatalf ( "snapshot should have been blocked" )
}
if err := s . LoadChunk ( chunker . Abort ( ) ) ; err != nil {
t . Fatalf ( "failed to load abort chunk: %s" , err . Error ( ) )
}
if err := s . Snapshot ( ) ; err != nil {
t . Fatalf ( "snapshot should not have been blocked" )
}
// Load the first chunk, which should block snapshotting, then send
// execute, then check that snapshotting is no longer blocked.
fd , chunker = getChunker ( )
defer fd . Close ( )
chunk , err = chunker . Next ( )
if err != nil {
t . Fatalf ( "failed to read first chunk: %s" , err . Error ( ) )
}
if err := s . LoadChunk ( chunk ) ; err != nil {
t . Fatalf ( "failed to load first chunk: %s" , err . Error ( ) )
}
if err := s . Snapshot ( ) ; err != ErrLoadInProgress {
t . Fatalf ( "snapshot should have been blocked" )
}
er := executeRequestFromString ( "anything at all, doesn't matter" , false , false )
_ , err = s . Execute ( er )
if err != nil {
t . Fatalf ( "failed to execute: %s" , err . Error ( ) )
}
if err := s . Snapshot ( ) ; err != nil {
t . Fatalf ( "snapshot should not have been blocked, err: %s" , err . Error ( ) )
}
}
// Test_SingleNodeLoadBinaryFromReader tests that a Store correctly loads data in SQLite
// binary format from a reader.
func Test_SingleNodeLoadBinaryFromReader ( t * testing . T ) {
s , ln := mustNewStore ( t )
defer ln . Close ( )
if err := s . Open ( ) ; err != nil {
t . Fatalf ( "failed to open single-node store: %s" , err . Error ( ) )
}
if err := s . Bootstrap ( NewServer ( s . ID ( ) , s . Addr ( ) , true ) ) ; err != nil {
t . Fatalf ( "failed to bootstrap single-node store: %s" , err . Error ( ) )
}
defer s . Close ( true )
if _ , err := s . WaitForLeader ( 10 * time . Second ) ; err != nil {
t . Fatalf ( "Error waiting for leader: %s" , err )
}
// Load a dataset, to check it's erased by the SQLite file load.
dump := ` PRAGMA foreign_keys = OFF ;
BEGIN TRANSACTION ;
CREATE TABLE bar ( id integer not null primary key , name text ) ;
INSERT INTO "bar" VALUES ( 1 , ' declan ' ) ;
COMMIT ;
`
_ , err := s . Execute ( executeRequestFromString ( dump , false , false ) )
if err != nil {
t . Fatalf ( "failed to load simple dump: %s" , err . Error ( ) )
}
// Check that data were loaded correctly.
qr := queryRequestFromString ( "SELECT * FROM bar" , false , true )
qr . Level = command . QueryRequest_QUERY_REQUEST_LEVEL_STRONG
r , err := s . Query ( qr )
if err != nil {
t . Fatalf ( "failed to query single node: %s" , err . Error ( ) )
}
if exp , got := ` ["id","name"] ` , asJSON ( r [ 0 ] . Columns ) ; exp != got {
t . Fatalf ( "unexpected results for query\nexp: %s\ngot: %s" , exp , got )
}
if exp , got := ` [[1,"declan"]] ` , asJSON ( r [ 0 ] . Values ) ; exp != got {
t . Fatalf ( "unexpected results for query\nexp: %s\ngot: %s" , exp , got )
}
f , err := os . Open ( filepath . Join ( "testdata" , "load.sqlite" ) )
if err != nil {
t . Fatalf ( "failed to open SQLite file: %s" , err . Error ( ) )
}
defer f . Close ( )
err = s . LoadFromReader ( f , 1024 * 1024 )
if err != nil {
t . Fatalf ( "failed to load SQLite file via Reader: %s" , err . Error ( ) )
}
// Check that data were loaded correctly.
qr = queryRequestFromString ( "SELECT * FROM foo WHERE id=2" , false , true )
qr . Level = command . QueryRequest_QUERY_REQUEST_LEVEL_STRONG
r , err = s . Query ( qr )
if err != nil {
t . Fatalf ( "failed to query single node: %s" , err . Error ( ) )
}
if exp , got := ` ["id","name"] ` , asJSON ( r [ 0 ] . Columns ) ; exp != got {
t . Fatalf ( "unexpected results for query\nexp: %s\ngot: %s" , exp , got )
}
if exp , got := ` [[2,"fiona"]] ` , asJSON ( r [ 0 ] . Values ) ; exp != got {
t . Fatalf ( "unexpected results for query\nexp: %s\ngot: %s" , exp , got )
}
qr = queryRequestFromString ( "SELECT count(*) FROM foo" , false , true )
qr . Level = command . QueryRequest_QUERY_REQUEST_LEVEL_STRONG
r , err = s . Query ( qr )
if err != nil {
t . Fatalf ( "failed to query single node: %s" , err . Error ( ) )
}
if exp , got := ` ["count(*)"] ` , asJSON ( r [ 0 ] . Columns ) ; exp != got {
t . Fatalf ( "unexpected results for query\nexp: %s\ngot: %s" , exp , got )
}
if exp , got := ` [[3]] ` , asJSON ( r [ 0 ] . Values ) ; exp != got {
t . Fatalf ( "unexpected results for query\nexp: %s\ngot: %s" , exp , got )
}
// Check pre-existing data is gone.
qr = queryRequestFromString ( "SELECT * FROM bar" , false , true )
qr . Level = command . QueryRequest_QUERY_REQUEST_LEVEL_STRONG
r , err = s . Query ( qr )
if err != nil {
t . Fatalf ( "failed to query single node: %s" , err . Error ( ) )
}
if exp , got := ` { "error":"no such table: bar"} ` , asJSON ( r [ 0 ] ) ; exp != got {
t . Fatalf ( "unexpected results for query\nexp: %s\ngot: %s" , exp , got )
}
}
func Test_SingleNodeAutoRestore ( t * testing . T ) {
s , ln := mustNewStore ( t )
defer ln . Close ( )
Philip O'Toole
9 months ago