|
|
|
package store
|
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
|
|
|
"path/filepath"
|
|
|
|
"testing"
|
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/rqlite/rqlite/v8/command/proto"
|
|
|
|
)
|
|
|
|
|
|
|
|
func Test_IsStaleRead(t *testing.T) {
|
|
|
|
tests := []struct {
|
|
|
|
Name string
|
|
|
|
LeaderLastContact time.Time
|
|
|
|
LastFSMUpdateTime time.Time
|
|
|
|
LastAppendedAtTime time.Time
|
|
|
|
FSMIndex uint64
|
|
|
|
CommitIndex uint64
|
|
|
|
Freshness time.Duration
|
|
|
|
MaxStale time.Duration
|
|
|
|
Exp bool
|
|
|
|
}{
|
|
|
|
{
|
|
|
|
Name: "no freshness set",
|
|
|
|
Freshness: 0,
|
|
|
|
Exp: false,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
Name: "no freshness set, but clearly unfresh connection",
|
|
|
|
LeaderLastContact: time.Now().Add(-1000 * time.Hour),
|
|
|
|
Freshness: 0,
|
|
|
|
Exp: false,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
Name: "freshness set, but not exceeded",
|
|
|
|
LeaderLastContact: time.Now().Add(10 * time.Second),
|
|
|
|
Freshness: time.Minute,
|
|
|
|
Exp: false,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
Name: "freshness set and exceeded",
|
|
|
|
LeaderLastContact: time.Now().Add(-10 * time.Second),
|
|
|
|
Freshness: time.Second,
|
|
|
|
Exp: true,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
Name: "freshness set, is ok, max stale exceeded, but applied index is up-to-date",
|
|
|
|
LeaderLastContact: time.Now().Add(-1 * time.Second),
|
|
|
|
LastFSMUpdateTime: time.Now().Add(-1 * time.Second),
|
|
|
|
LastAppendedAtTime: time.Now().Add(-10 * time.Second),
|
|
|
|
FSMIndex: 10,
|
|
|
|
CommitIndex: 10,
|
|
|
|
Freshness: time.Minute,
|
|
|
|
MaxStale: time.Second,
|
|
|
|
Exp: false,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
Name: "freshness set, is ok, max stale exceeded, and applied index is behind",
|
|
|
|
LeaderLastContact: time.Now().Add(-1 * time.Second),
|
|
|
|
LastFSMUpdateTime: time.Now().Add(-1 * time.Second),
|
|
|
|
LastAppendedAtTime: time.Now().Add(-10 * time.Second),
|
|
|
|
FSMIndex: 9,
|
|
|
|
CommitIndex: 10,
|
|
|
|
Freshness: time.Minute,
|
|
|
|
MaxStale: time.Second,
|
|
|
|
Exp: true,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
Name: "freshness set, is ok, max stale not exceeded, but applied index is behind",
|
|
|
|
LeaderLastContact: time.Now().Add(-1 * time.Second),
|
|
|
|
LastFSMUpdateTime: time.Now().Add(-1 * time.Second),
|
|
|
|
LastAppendedAtTime: time.Now().Add(-10 * time.Second),
|
|
|
|
FSMIndex: 9,
|
|
|
|
CommitIndex: 10,
|
|
|
|
Freshness: time.Minute,
|
|
|
|
MaxStale: time.Minute,
|
|
|
|
Exp: false,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
Name: "freshness set, is ok, applied index is behind, but max stale not set",
|
|
|
|
LeaderLastContact: time.Now().Add(-1 * time.Second),
|
|
|
|
LastFSMUpdateTime: time.Now().Add(-1 * time.Second),
|
|
|
|
LastAppendedAtTime: time.Now().Add(-10 * time.Second),
|
|
|
|
FSMIndex: 9,
|
|
|
|
CommitIndex: 10,
|
|
|
|
Freshness: time.Minute,
|
|
|
|
MaxStale: 0,
|
|
|
|
Exp: false,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, tt := range tests {
|
|
|
|
if got, exp := IsStaleRead(
|
|
|
|
tt.LeaderLastContact,
|
|
|
|
tt.LastFSMUpdateTime,
|
|
|
|
tt.LastAppendedAtTime,
|
|
|
|
tt.FSMIndex,
|
|
|
|
tt.CommitIndex,
|
|
|
|
tt.Freshness.Nanoseconds(),
|
|
|
|
tt.MaxStale.Nanoseconds()), tt.Exp; got != exp {
|
|
|
|
t.Fatalf("unexpected result for IsStaleRead test %s\nexp: %v\ngot: %v", tt.Name, exp, got)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func Test_Store_IsNewNode(t *testing.T) {
|
|
|
|
s, ln := mustNewStore(t)
|
|
|
|
defer ln.Close()
|
|
|
|
|
|
|
|
if !IsNewNode(s.raftDir) {
|
|
|
|
t.Fatalf("new store is not new")
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := s.Open(); err != nil {
|
|
|
|
t.Fatalf("failed to open single-node store: %s", err.Error())
|
|
|
|
}
|
|
|
|
defer s.Close(true)
|
|
|
|
if err := s.Bootstrap(NewServer(s.ID(), s.Addr(), true)); err != nil {
|
|
|
|
t.Fatalf("failed to bootstrap single-node store: %s", err.Error())
|
|
|
|
}
|
|
|
|
if _, err := s.WaitForLeader(10 * time.Second); err != nil {
|
|
|
|
t.Fatalf("Error waiting for leader: %s", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if IsNewNode(s.raftDir) {
|
|
|
|
t.Fatalf("new store is new")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func Test_Store_HasData(t *testing.T) {
|
|
|
|
s, ln := mustNewStore(t)
|
|
|
|
defer ln.Close()
|
|
|
|
|
|
|
|
h, err := HasData(s.raftDir)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("failed to check for data: %s", err.Error())
|
|
|
|
}
|
|
|
|
if h {
|
|
|
|
t.Fatalf("new store has data")
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := s.Open(); err != nil {
|
|
|
|
t.Fatalf("failed to open single-node store: %s", err.Error())
|
|
|
|
}
|
|
|
|
if err := s.Bootstrap(NewServer(s.ID(), s.Addr(), true)); err != nil {
|
|
|
|
t.Fatalf("failed to bootstrap single-node store: %s", err.Error())
|
|
|
|
}
|
|
|
|
if _, err := s.WaitForLeader(10 * time.Second); err != nil {
|
|
|
|
t.Fatalf("Error waiting for leader: %s", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Write some data.
|
|
|
|
er := executeRequestFromStrings([]string{
|
|
|
|
`CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)`,
|
|
|
|
}, false, false)
|
|
|
|
_, err = s.Execute(er)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("failed to execute on single node: %s", err.Error())
|
|
|
|
}
|
|
|
|
|
|
|
|
// Close the store to unblock the Bolt database.
|
|
|
|
s.Close(true)
|
|
|
|
|
|
|
|
h, err = HasData(s.raftDir)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("failed to check for data: %s", err.Error())
|
|
|
|
}
|
|
|
|
if !h {
|
|
|
|
t.Fatalf("store does not have data")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Test_SingleNodeRecoverNoChange tests a node recovery that doesn't
|
|
|
|
// actually change anything.
|
|
|
|
func Test_SingleNodeRecoverNoChange(t *testing.T) {
|
|
|
|
s, ln := mustNewStore(t)
|
|
|
|
defer ln.Close()
|
|
|
|
if err := s.Open(); err != nil {
|
|
|
|
t.Fatalf("failed to open single-node store: %s", err.Error())
|
|
|
|
}
|
|
|
|
if err := s.Bootstrap(NewServer(s.ID(), s.Addr(), true)); err != nil {
|
|
|
|
t.Fatalf("failed to bootstrap single-node store: %s", err.Error())
|
|
|
|
}
|
|
|
|
if _, err := s.WaitForLeader(10 * time.Second); err != nil {
|
|
|
|
t.Fatalf("Error waiting for leader: %s", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
queryTest := func() {
|
|
|
|
t.Helper()
|
|
|
|
qr := queryRequestFromString("SELECT * FROM foo", false, false)
|
|
|
|
qr.Level = proto.QueryRequest_QUERY_REQUEST_LEVEL_NONE
|
|
|
|
r, err := s.Query(qr)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("failed to query single node: %s", err.Error())
|
|
|
|
}
|
|
|
|
if exp, got := `["id","name"]`, asJSON(r[0].Columns); exp != got {
|
|
|
|
t.Fatalf("unexpected results for query\nexp: %s\ngot: %s", exp, got)
|
|
|
|
}
|
|
|
|
if exp, got := `[[1,"fiona"]]`, asJSON(r[0].Values); exp != got {
|
|
|
|
t.Fatalf("unexpected results for query\nexp: %s\ngot: %s", exp, got)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
er := executeRequestFromStrings([]string{
|
|
|
|
`CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)`,
|
|
|
|
`INSERT INTO foo(id, name) VALUES(1, "fiona")`,
|
|
|
|
}, false, false)
|
|
|
|
_, err := s.Execute(er)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("failed to execute on single node: %s", err.Error())
|
|
|
|
}
|
|
|
|
queryTest()
|
|
|
|
id, addr := s.ID(), s.Addr()
|
|
|
|
if err := s.Close(true); err != nil {
|
|
|
|
t.Fatalf("failed to close single-node store: %s", err.Error())
|
|
|
|
}
|
|
|
|
|
|
|
|
// Set up for Recovery during open
|
|
|
|
peers := fmt.Sprintf(`[{"id": "%s","address": "%s"}]`, id, addr)
|
|
|
|
peersPath := filepath.Join(s.Path(), "/raft/peers.json")
|
|
|
|
peersInfo := filepath.Join(s.Path(), "/raft/peers.info")
|
|
|
|
mustWriteFile(peersPath, peers)
|
|
|
|
if err := s.Open(); err != nil {
|
|
|
|
t.Fatalf("failed to re-open single-node store: %s", err.Error())
|
|
|
|
}
|
|
|
|
if _, err := s.WaitForLeader(10 * time.Second); err != nil {
|
|
|
|
t.Fatalf("Error waiting for leader: %s", err)
|
|
|
|
}
|
|
|
|
queryTest()
|
|
|
|
if err := s.Close(true); err != nil {
|
|
|
|
t.Fatalf("failed to close single-node store: %s", err.Error())
|
|
|
|
}
|
|
|
|
|
|
|
|
if pathExists(peersPath) {
|
|
|
|
t.Fatalf("Peers JSON exists at %s", peersPath)
|
|
|
|
}
|
|
|
|
if !pathExists(peersInfo) {
|
|
|
|
t.Fatalf("Peers info does not exist at %s", peersInfo)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Test_SingleNodeRecoverNetworkChange tests a node recovery that
|
|
|
|
// involves a changed-network address.
|
|
|
|
func Test_SingleNodeRecoverNetworkChange(t *testing.T) {
|
|
|
|
s0, ln0 := mustNewStore(t)
|
|
|
|
defer ln0.Close()
|
|
|
|
if err := s0.Open(); err != nil {
|
|
|
|
t.Fatalf("failed to open single-node store: %s", err.Error())
|
|
|
|
}
|
|
|
|
if err := s0.Bootstrap(NewServer(s0.ID(), s0.Addr(), true)); err != nil {
|
|
|
|
t.Fatalf("failed to bootstrap single-node store: %s", err.Error())
|
|
|
|
}
|
|
|
|
if _, err := s0.WaitForLeader(10 * time.Second); err != nil {
|
|
|
|
t.Fatalf("Error waiting for leader: %s", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
queryTest := func(s *Store) {
|
|
|
|
qr := queryRequestFromString("SELECT * FROM foo", false, false)
|
|
|
|
qr.Level = proto.QueryRequest_QUERY_REQUEST_LEVEL_NONE
|
|
|
|
r, err := s.Query(qr)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("failed to query single node: %s", err.Error())
|
|
|
|
}
|
|
|
|
if exp, got := `["id","name"]`, asJSON(r[0].Columns); exp != got {
|
|
|
|
t.Fatalf("unexpected results for query\nexp: %s\ngot: %s", exp, got)
|
|
|
|
}
|
|
|
|
if exp, got := `[[1,"fiona"]]`, asJSON(r[0].Values); exp != got {
|
|
|
|
t.Fatalf("unexpected results for query\nexp: %s\ngot: %s", exp, got)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
er := executeRequestFromStrings([]string{
|
|
|
|
`CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)`,
|
|
|
|
`INSERT INTO foo(id, name) VALUES(1, "fiona")`,
|
|
|
|
}, false, false)
|
|
|
|
_, err := s0.Execute(er)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("failed to execute on single node: %s", err.Error())
|
|
|
|
}
|
|
|
|
queryTest(s0)
|
|
|
|
|
|
|
|
id := s0.ID()
|
|
|
|
if err := s0.Close(true); err != nil {
|
|
|
|
t.Fatalf("failed to close single-node store: %s", err.Error())
|
|
|
|
}
|
|
|
|
|
|
|
|
// Create a new node, at the same path. Will presumably have a different
|
|
|
|
// Raft network address, since they are randomly assigned.
|
|
|
|
sR, srLn := mustNewStoreAtPathsLn(id, s0.Path(), "", true)
|
|
|
|
defer srLn.Close()
|
|
|
|
if IsNewNode(sR.Path()) {
|
|
|
|
t.Fatalf("store detected incorrectly as new")
|
|
|
|
}
|
|
|
|
|
|
|
|
// Set up for Recovery during open
|
|
|
|
peers := fmt.Sprintf(`[{"id": "%s","address": "%s"}]`, s0.ID(), srLn.Addr().String())
|
|
|
|
peersPath := filepath.Join(sR.Path(), "/raft/peers.json")
|
|
|
|
peersInfo := filepath.Join(sR.Path(), "/raft/peers.info")
|
|
|
|
mustWriteFile(peersPath, peers)
|
|
|
|
if err := sR.Open(); err != nil {
|
|
|
|
t.Fatalf("failed to open single-node store: %s", err.Error())
|
|
|
|
}
|
|
|
|
|
|
|
|
if _, err := sR.WaitForLeader(10 * time.Second); err != nil {
|
|
|
|
t.Fatalf("Error waiting for leader on recovered node: %s", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
queryTest(sR)
|
|
|
|
if err := sR.Close(true); err != nil {
|
|
|
|
t.Fatalf("failed to close single-node recovered store: %s", err.Error())
|
|
|
|
}
|
|
|
|
|
|
|
|
if pathExists(peersPath) {
|
|
|
|
t.Fatalf("Peers JSON exists at %s", peersPath)
|
|
|
|
}
|
|
|
|
if !pathExists(peersInfo) {
|
|
|
|
t.Fatalf("Peers info does not exist at %s", peersInfo)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Test_SingleNodeRecoverNetworkChangeSnapshot tests a node recovery that
|
|
|
|
// involves a changed-network address, with snapshots underneath.
|
|
|
|
func Test_SingleNodeRecoverNetworkChangeSnapshot(t *testing.T) {
|
|
|
|
s0, ln0 := mustNewStore(t)
|
|
|
|
defer ln0.Close()
|
|
|
|
s0.SnapshotThreshold = 4
|
|
|
|
s0.SnapshotInterval = 100 * time.Millisecond
|
|
|
|
if err := s0.Open(); err != nil {
|
|
|
|
t.Fatalf("failed to open single-node store: %s", err.Error())
|
|
|
|
}
|
|
|
|
if err := s0.Bootstrap(NewServer(s0.ID(), s0.Addr(), true)); err != nil {
|
|
|
|
t.Fatalf("failed to bootstrap single-node store: %s", err.Error())
|
|
|
|
}
|
|
|
|
if _, err := s0.WaitForLeader(10 * time.Second); err != nil {
|
|
|
|
t.Fatalf("Error waiting for leader: %s", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
queryTest := func(s *Store, c int) {
|
|
|
|
qr := queryRequestFromString("SELECT COUNT(*) FROM foo", false, false)
|
|
|
|
qr.Level = proto.QueryRequest_QUERY_REQUEST_LEVEL_NONE
|
|
|
|
r, err := s.Query(qr)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("failed to query single node: %s", err.Error())
|
|
|
|
}
|
|
|
|
if exp, got := `["COUNT(*)"]`, asJSON(r[0].Columns); exp != got {
|
|
|
|
t.Fatalf("unexpected results for query\nexp: %s\ngot: %s", exp, got)
|
|
|
|
}
|
|
|
|
if exp, got := fmt.Sprintf(`[[%d]]`, c), asJSON(r[0].Values); exp != got {
|
|
|
|
t.Fatalf("unexpected results for query\nexp: %s\ngot: %s", exp, got)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
er := executeRequestFromStrings([]string{
|
|
|
|
`CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)`,
|
|
|
|
`INSERT INTO foo(id, name) VALUES(1, "fiona")`,
|
|
|
|
}, false, false)
|
|
|
|
_, err := s0.Execute(er)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("failed to execute on single node: %s", err.Error())
|
|
|
|
}
|
|
|
|
queryTest(s0, 1)
|
|
|
|
|
|
|
|
for i := 0; i < 9; i++ {
|
|
|
|
er := executeRequestFromStrings([]string{
|
|
|
|
`INSERT INTO foo(name) VALUES("fiona")`,
|
|
|
|
}, false, false)
|
|
|
|
if _, err := s0.Execute(er); err != nil {
|
|
|
|
t.Fatalf("failed to execute on single node: %s", err.Error())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
queryTest(s0, 10)
|
|
|
|
|
|
|
|
// Wait for a snapshot to take place.
|
|
|
|
for {
|
|
|
|
time.Sleep(100 * time.Millisecond)
|
|
|
|
s0.numSnapshotsMu.Lock()
|
|
|
|
ns := s0.numSnapshots
|
|
|
|
s0.numSnapshotsMu.Unlock()
|
|
|
|
if ns > 0 {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
id := s0.ID()
|
|
|
|
if err := s0.Close(true); err != nil {
|
|
|
|
t.Fatalf("failed to close single-node store: %s", err.Error())
|
|
|
|
}
|
|
|
|
|
|
|
|
// Create a new node, at the same path. Will presumably have a different
|
|
|
|
// Raft network address, since they are randomly assigned.
|
|
|
|
sR, srLn := mustNewStoreAtPathsLn(id, s0.Path(), "", true)
|
|
|
|
if IsNewNode(sR.Path()) {
|
|
|
|
t.Fatalf("store detected incorrectly as new")
|
|
|
|
}
|
|
|
|
|
|
|
|
// Set up for Recovery during open
|
|
|
|
peers := fmt.Sprintf(`[{"id": "%s","address": "%s"}]`, id, srLn.Addr().String())
|
|
|
|
peersPath := filepath.Join(sR.Path(), "/raft/peers.json")
|
|
|
|
peersInfo := filepath.Join(sR.Path(), "/raft/peers.info")
|
|
|
|
mustWriteFile(peersPath, peers)
|
|
|
|
if err := sR.Open(); err != nil {
|
|
|
|
t.Fatalf("failed to open single-node store: %s", err.Error())
|
|
|
|
}
|
|
|
|
|
|
|
|
if _, err := sR.WaitForLeader(10 * time.Second); err != nil {
|
|
|
|
t.Fatalf("Error waiting for leader on recovered node: %s", err)
|
|
|
|
}
|
|
|
|
queryTest(sR, 10)
|
|
|
|
if err := sR.Close(true); err != nil {
|
|
|
|
t.Fatalf("failed to close single-node recovered store: %s", err.Error())
|
|
|
|
}
|
|
|
|
|
|
|
|
if pathExists(peersPath) {
|
|
|
|
t.Fatalf("Peers JSON exists at %s", peersPath)
|
|
|
|
}
|
|
|
|
if !pathExists(peersInfo) {
|
|
|
|
t.Fatalf("Peers info does not exist at %s", peersInfo)
|
|
|
|
}
|
|
|
|
}
|