package system import ( "testing" "time" ) // Test_JoinLeaderNode tests a join operation between a leader and a new node. func Test_JoinLeaderNode(t *testing.T) { leader := mustNewLeaderNode() defer leader.Deprovision() node := mustNewNode(false) defer node.Deprovision() if err := node.Join(leader); err != nil { t.Fatalf("node failed to join leader: %s", err.Error()) } _, err := node.WaitForLeader() if err != nil { t.Fatalf("failed waiting for leader: %s", err.Error()) } } // Test_MultiNodeCluster tests formation of a 3-node cluster, and its operation. func Test_MultiNodeCluster(t *testing.T) { node1 := mustNewLeaderNode() defer node1.Deprovision() node2 := mustNewNode(false) defer node2.Deprovision() if err := node2.Join(node1); err != nil { t.Fatalf("node failed to join leader: %s", err.Error()) } _, err := node2.WaitForLeader() if err != nil { t.Fatalf("failed waiting for leader: %s", err.Error()) } // Get the new leader, in case it changed. c := Cluster{node1, node2} leader, err := c.Leader() if err != nil { t.Fatalf("failed to find cluster leader: %s", err.Error()) } // Get a follower and confirm redirects work properly. followers, err := c.Followers() if err != nil { t.Fatalf("failed to get followers: %s", err.Error()) } if len(followers) != 1 { t.Fatalf("got incorrect number of followers: %d", len(followers)) } node3 := mustNewNode(false) defer node3.Deprovision() if err := node3.Join(leader); err != nil { t.Fatalf("node failed to join leader: %s", err.Error()) } _, err = node3.WaitForLeader() if err != nil { t.Fatalf("failed waiting for leader: %s", err.Error()) } // Get the new leader, in case it changed. c = Cluster{node1, node2, node3} leader, err = c.Leader() if err != nil { t.Fatalf("failed to find cluster leader: %s", err.Error()) } // Run queries against cluster. tests := []struct { stmt string expected string execute bool }{ { stmt: `CREATE TABLE foo (id integer not null primary key, name text)`, expected: `{"results":[{}]}`, execute: true, }, { stmt: `INSERT INTO foo(name) VALUES("fiona")`, expected: `{"results":[{"last_insert_id":1,"rows_affected":1}]}`, execute: true, }, { stmt: `SELECT * FROM foo`, expected: `{"results":[{"columns":["id","name"],"types":["integer","text"],"values":[[1,"fiona"]]}]}`, execute: false, }, } for i, tt := range tests { var r string var err error if tt.execute { r, err = leader.Execute(tt.stmt) } else { r, err = leader.Query(tt.stmt) } if err != nil { t.Fatalf(`test %d failed "%s": %s`, i, tt.stmt, err.Error()) } if r != tt.expected { t.Fatalf(`test %d received wrong result "%s" got: %s exp: %s`, i, tt.stmt, r, tt.expected) } } // Kill the leader and wait for the new leader. leader.Deprovision() c.RemoveNode(leader) leader, err = c.WaitForNewLeader(leader) if err != nil { t.Fatalf("failed to find new cluster leader after killing leader: %s", err.Error()) } // Run queries against the now 2-node cluster. tests = []struct { stmt string expected string execute bool }{ { stmt: `CREATE TABLE foo (id integer not null primary key, name text)`, expected: `{"results":[{"error":"table foo already exists"}]}`, execute: true, }, { stmt: `INSERT INTO foo(name) VALUES("sinead")`, expected: `{"results":[{"last_insert_id":2,"rows_affected":1}]}`, execute: true, }, { stmt: `SELECT * FROM foo`, expected: `{"results":[{"columns":["id","name"],"types":["integer","text"],"values":[[1,"fiona"],[2,"sinead"]]}]}`, execute: false, }, } for i, tt := range tests { var r string var err error if tt.execute { r, err = leader.Execute(tt.stmt) } else { r, err = leader.Query(tt.stmt) } if err != nil { t.Fatalf(`test %d failed "%s": %s`, i, tt.stmt, err.Error()) } if r != tt.expected { t.Fatalf(`test %d received wrong result "%s" got: %s exp: %s`, i, tt.stmt, r, tt.expected) } } } // Test_MultiNodeClusterSnapshot tests formation of a 3-node cluster, which involves sharing snapshots. func Test_MultiNodeClusterSnapshot(t *testing.T) { node1 := mustNewLeaderNode() defer node1.Deprovision() if _, err := node1.Execute(`CREATE TABLE foo (id integer not null primary key, name text)`); err != nil { t.Fatalf("failed to create table: %s", err.Error()) } // Force snapshots and log truncation to occur. for i := 0; i < 3*int(node1.Store.SnapshotThreshold); i++ { _, err := node1.Execute(`INSERT INTO foo(name) VALUES("sinead")`) if err != nil { t.Fatalf(`failed to write records for Snapshot test: %s`, err.Error()) } } // Join a second and third nodes, which will get database state via snapshots. node2 := mustNewNode(false) defer node2.Deprovision() if err := node2.Join(node1); err != nil { t.Fatalf("node failed to join leader: %s", err.Error()) } _, err := node2.WaitForLeader() if err != nil { t.Fatalf("failed waiting for leader: %s", err.Error()) } node3 := mustNewNode(false) defer node3.Deprovision() if err := node3.Join(node1); err != nil { t.Fatalf("node failed to join leader: %s", err.Error()) } _, err = node3.WaitForLeader() if err != nil { t.Fatalf("failed waiting for leader: %s", err.Error()) } // Create a new cluster. c := Cluster{node1, node2, node3} // Wait for followers to pick up state. followers, err := c.Followers() if err != nil { t.Fatalf("failed to determine followers: %s", err.Error()) } var n int var r string for _, f := range followers { n = 0 for { r, err = f.QueryNoneConsistency(`SELECT COUNT(*) FROM foo`) if err != nil { t.Fatalf("failed to query follower node: %s", err.Error()) } if r != `{"results":[{"columns":["COUNT(*)"],"types":[""],"values":[[300]]}]}` { if n < 20 { // Wait, and try again. time.Sleep(mustParseDuration("1s")) n++ continue } t.Fatalf("timed out waiting for snapshot state") } // The node passed! break } } // Kill original node. node1.Deprovision() c.RemoveNode(node1) var leader *Node leader, err = c.WaitForNewLeader(node1) if err != nil { t.Fatalf("failed to find new cluster leader after killing leader: %s", err.Error()) } // Test that the new leader still has the full state. n = 0 for { var r string r, err = leader.Query(`SELECT COUNT(*) FROM foo`) if err != nil { t.Fatalf("failed to query follower node: %s", err.Error()) } if r != `{"results":[{"columns":["COUNT(*)"],"types":[""],"values":[[300]]}]}` { if n < 10 { // Wait, and try again. time.Sleep(mustParseDuration("100ms")) n++ continue } t.Fatalf("timed out waiting for snapshot state") } // Test passed! break } } // Test_MultiNodeClusterWithNonVoter tests formation of a 4-node cluster, one of which is // a non-voter func Test_MultiNodeClusterWithNonVoter(t *testing.T) { node1 := mustNewLeaderNode() defer node1.Deprovision() node2 := mustNewNode(false) defer node2.Deprovision() if err := node2.Join(node1); err != nil { t.Fatalf("node failed to join leader: %s", err.Error()) } _, err := node2.WaitForLeader() if err != nil { t.Fatalf("failed waiting for leader: %s", err.Error()) } // Get the new leader, in case it changed. c := Cluster{node1, node2} leader, err := c.Leader() if err != nil { t.Fatalf("failed to find cluster leader: %s", err.Error()) } node3 := mustNewNode(false) defer node3.Deprovision() if err := node3.Join(leader); err != nil { t.Fatalf("node failed to join leader: %s", err.Error()) } _, err = node3.WaitForLeader() if err != nil { t.Fatalf("failed waiting for leader: %s", err.Error()) } // Get the new leader, in case it changed. c = Cluster{node1, node2, node3} leader, err = c.Leader() if err != nil { t.Fatalf("failed to find cluster leader: %s", err.Error()) } nonVoter := mustNewNode(false) defer nonVoter.Deprovision() if err := nonVoter.JoinAsNonVoter(leader); err != nil { t.Fatalf("non-voting node failed to join leader: %s", err.Error()) } _, err = nonVoter.WaitForLeader() if err != nil { t.Fatalf("failed waiting for leader: %s", err.Error()) } c = Cluster{node1, node2, node3, nonVoter} // Run queries against cluster. tests := []struct { stmt string expected string execute bool }{ { stmt: `CREATE TABLE foo (id integer not null primary key, name text)`, expected: `{"results":[{}]}`, execute: true, }, { stmt: `INSERT INTO foo(name) VALUES("fiona")`, expected: `{"results":[{"last_insert_id":1,"rows_affected":1}]}`, execute: true, }, { stmt: `SELECT * FROM foo`, expected: `{"results":[{"columns":["id","name"],"types":["integer","text"],"values":[[1,"fiona"]]}]}`, execute: false, }, } for i, tt := range tests { var r string var err error if tt.execute { r, err = leader.Execute(tt.stmt) } else { r, err = leader.Query(tt.stmt) } if err != nil { t.Fatalf(`test %d failed "%s": %s`, i, tt.stmt, err.Error()) } if r != tt.expected { t.Fatalf(`test %d received wrong result "%s" got: %s exp: %s`, i, tt.stmt, r, tt.expected) } } // Kill the leader and wait for the new leader. leader.Deprovision() c.RemoveNode(leader) leader, err = c.WaitForNewLeader(leader) if err != nil { t.Fatalf("failed to find new cluster leader after killing leader: %s", err.Error()) } // Run queries against the now 3-node cluster. tests = []struct { stmt string expected string execute bool }{ { stmt: `CREATE TABLE foo (id integer not null primary key, name text)`, expected: `{"results":[{"error":"table foo already exists"}]}`, execute: true, }, { stmt: `INSERT INTO foo(name) VALUES("sinead")`, expected: `{"results":[{"last_insert_id":2,"rows_affected":1}]}`, execute: true, }, { stmt: `SELECT * FROM foo`, expected: `{"results":[{"columns":["id","name"],"types":["integer","text"],"values":[[1,"fiona"],[2,"sinead"]]}]}`, execute: false, }, } for i, tt := range tests { var r string var err error if tt.execute { r, err = leader.Execute(tt.stmt) } else { r, err = leader.Query(tt.stmt) } if err != nil { t.Fatalf(`test %d failed "%s": %s`, i, tt.stmt, err.Error()) } if r != tt.expected { t.Fatalf(`test %d received wrong result "%s" got: %s exp: %s`, i, tt.stmt, r, tt.expected) } } }