|
|
|
@ -758,7 +758,7 @@ func Test_MultiNodeClusterWithNonVoter(t *testing.T) {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Test_MultiNodeClusterRecover tests recovery of a single node from a 3-node cluster,
|
|
|
|
|
// Test_MultiNodeClusterRecoverSingle tests recovery of a single node from a 3-node cluster,
|
|
|
|
|
// which no longer has quorum.
|
|
|
|
|
func Test_MultiNodeClusterRecoverSingle(t *testing.T) {
|
|
|
|
|
node1 := mustNewLeaderNode()
|
|
|
|
@ -832,3 +832,88 @@ func Test_MultiNodeClusterRecoverSingle(t *testing.T) {
|
|
|
|
|
}
|
|
|
|
|
okSingle.Close(true)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Test_MultiNodeClusterRecoverFull tests recovery of a full 3-node cluster,
|
|
|
|
|
// each node coming up with a different Raft address.
|
|
|
|
|
func Test_MultiNodeClusterRecoverFull(t *testing.T) {
|
|
|
|
|
var err error
|
|
|
|
|
|
|
|
|
|
mux1 := mustNewOpenMux("127.0.0.1:10001")
|
|
|
|
|
node1 := mustNodeEncrypted(mustTempDir(), true, false, mux1, "1")
|
|
|
|
|
_, err = node1.WaitForLeader()
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatalf("failed waiting for leader: %s", err.Error())
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
mux2 := mustNewOpenMux("127.0.0.1:10002")
|
|
|
|
|
node2 := mustNodeEncrypted(mustTempDir(), false, false, mux2, "2")
|
|
|
|
|
if err := node2.Join(node1); err != nil {
|
|
|
|
|
t.Fatalf("node failed to join leader: %s", err.Error())
|
|
|
|
|
}
|
|
|
|
|
_, err = node2.WaitForLeader()
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatalf("failed waiting for leader: %s", err.Error())
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
mux3 := mustNewOpenMux("127.0.0.1:10003")
|
|
|
|
|
node3 := mustNodeEncrypted(mustTempDir(), false, false, mux3, "3")
|
|
|
|
|
if err := node3.Join(node1); err != nil {
|
|
|
|
|
t.Fatalf("node failed to join leader: %s", err.Error())
|
|
|
|
|
}
|
|
|
|
|
_, err = node3.WaitForLeader()
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatalf("failed waiting for leader: %s", err.Error())
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if _, err := node1.Execute(`CREATE TABLE foo (id integer not null primary key, name text)`); err != nil {
|
|
|
|
|
t.Fatalf("failed to create table: %s", err.Error())
|
|
|
|
|
}
|
|
|
|
|
if _, err := node1.Execute(`INSERT INTO foo(id, name) VALUES(1, "fiona")`); err != nil {
|
|
|
|
|
t.Fatalf("failed to create table: %s", err.Error())
|
|
|
|
|
}
|
|
|
|
|
if rows, _ := node1.Query(`SELECT COUNT(*) FROM foo`); rows != `{"results":[{"columns":["COUNT(*)"],"types":[""],"values":[[1]]}]}` {
|
|
|
|
|
t.Fatalf("got incorrect results from node: %s", rows)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Shutdown all nodes
|
|
|
|
|
if err := node1.Close(true); err != nil {
|
|
|
|
|
t.Fatalf("failed to close node1: %s", err.Error())
|
|
|
|
|
}
|
|
|
|
|
if err := node2.Close(true); err != nil {
|
|
|
|
|
t.Fatalf("failed to close node2: %s", err.Error())
|
|
|
|
|
}
|
|
|
|
|
if err := node3.Close(true); err != nil {
|
|
|
|
|
t.Fatalf("failed to close node3: %s", err.Error())
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Restart cluster, each node with different Raft addresses.
|
|
|
|
|
peers := fmt.Sprintf(`[{"id": "%s","address": "%s"}, {"id": "%s","address": "%s"}, {"id": "%s","address": "%s"}]`,
|
|
|
|
|
"1", "127.0.0.1:11001",
|
|
|
|
|
"2", "127.0.0.1:11002",
|
|
|
|
|
"3", "127.0.0.1:11003",
|
|
|
|
|
)
|
|
|
|
|
mustWriteFile(node1.PeersPath, peers)
|
|
|
|
|
mustWriteFile(node2.PeersPath, peers)
|
|
|
|
|
mustWriteFile(node3.PeersPath, peers)
|
|
|
|
|
|
|
|
|
|
mux4 := mustNewOpenMux("127.0.0.1:11001")
|
|
|
|
|
node4 := mustNodeEncrypted(node1.Dir, false, false, mux4, "1")
|
|
|
|
|
defer node4.Deprovision()
|
|
|
|
|
|
|
|
|
|
mux5 := mustNewOpenMux("127.0.0.1:11002")
|
|
|
|
|
node5 := mustNodeEncrypted(node2.Dir, false, false, mux5, "2")
|
|
|
|
|
defer node5.Deprovision()
|
|
|
|
|
|
|
|
|
|
mux6 := mustNewOpenMux("127.0.0.1:11003")
|
|
|
|
|
node6 := mustNodeEncrypted(node3.Dir, false, false, mux6, "3")
|
|
|
|
|
defer node6.Deprovision()
|
|
|
|
|
|
|
|
|
|
_, err = node6.WaitForLeader()
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatalf("failed waiting for leader on recovered cluster: %s", err.Error())
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if rows, _ := node4.Query(`SELECT COUNT(*) FROM foo`); rows != `{"results":[{"columns":["COUNT(*)"],"types":[""],"values":[[1]]}]}` {
|
|
|
|
|
t.Fatalf("got incorrect results from recovered node: %s", rows)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|