Merge pull request #870 from rqlite/adv-testing

More system-level Raft advertised addresss testing

store/store.go

@@ -712,7 +712,7 @@ func (s *Store) Backup(leader bool, fmt BackupFormat, dst io.Writer) error {
 // Join joins a node, identified by id and located at addr, to this store.
 // The node must be ready to respond to Raft communications at that address.
 func (s *Store) Join(id, addr string, voter bool) error {
-	s.logger.Printf("received request to join node at %s", addr)
+	s.logger.Printf("received request from node with ID %s, at %s, to join this node", id, addr)
 	if s.raft.State() != raft.Leader {
 		return ErrNotLeader
 	}
@@ -759,7 +759,7 @@ func (s *Store) Join(id, addr string, voter bool) error {
 	}
 
 	stats.Add(numJoins, 1)
-	s.logger.Printf("node at %s joined successfully as %s", addr, prettyVoter(voter))
+	s.logger.Printf("node with ID %s, at %s, joined successfully as %s", id, addr, prettyVoter(voter))
 	return nil
 }
 

system_test/cluster_test.go

@@ -2,8 +2,11 @@ package system
 
 import (
 	"fmt"
+	"net"
 	"testing"
 	"time"
+
+	"github.com/rqlite/rqlite/tcp"
 )
 
 // Test_JoinLeaderNode tests a join operation between a leader and a new node.
@@ -157,6 +160,69 @@ func Test_MultiNodeCluster(t *testing.T) {
 	}
 }
 
+// Test_MultiNodeClusterRaftAdv tests 3-node cluster with advertised Raft addresses usage.
+func Test_MultiNodeClusterRaftAdv(t *testing.T) {
+	ln1 := mustTCPListener("0.0.0.0:0")
+	defer ln1.Close()
+	ln2 := mustTCPListener("0.0.0.0:0")
+	defer ln2.Close()
+
+	advAddr := mustGetLocalIPv4Address()
+
+	_, port1, err := net.SplitHostPort(ln1.Addr().String())
+	if err != nil {
+		t.Fatalf("failed to get host and port: %s", err.Error())
+	}
+	_, port2, err := net.SplitHostPort(ln2.Addr().String())
+	if err != nil {
+		t.Fatalf("failed to get host and port: %s", err.Error())
+	}
+
+	advAddr1, err := net.ResolveTCPAddr("tcp", net.JoinHostPort(advAddr, port1))
+	if err != nil {
+		t.Fatalf("failed to resolve TCP address: %s", err.Error())
+	}
+	advAddr2, err := net.ResolveTCPAddr("tcp", net.JoinHostPort(advAddr, port2))
+	if err != nil {
+		t.Fatalf("failed to resolve TCP address: %s", err.Error())
+	}
+
+	mux1, err := tcp.NewMux(ln1, advAddr1)
+	if err != nil {
+		t.Fatalf("failed to create node-to-node mux: %s", err.Error())
+	}
+	go mux1.Serve()
+	mux2, err := tcp.NewMux(ln2, advAddr2)
+	if err != nil {
+		t.Fatalf("failed to create node-to-node mux: %s", err.Error())
+	}
+	go mux2.Serve()
+
+	// Start two nodes, and ensure a cluster can be formed.
+	node1 := mustNodeEncrypted(mustTempDir(), true, false, mux1, "1")
+	defer node1.Deprovision()
+	leader, err := node1.WaitForLeader()
+	if err != nil {
+		t.Fatalf("failed waiting for leader on node1: %s", err.Error())
+	}
+	if exp, got := advAddr1.String(), leader; exp != got {
+		t.Fatalf("node return wrong leader from leader, exp: %s, got %s", exp, got)
+	}
+
+	node2 := mustNodeEncrypted(mustTempDir(), false, false, mux2, "2")
+	defer node2.Deprovision()
+	if err := node2.Join(node1); err != nil {
+		t.Fatalf("node2 failed to join leader: %s", err.Error())
+	}
+	leader, err = node2.WaitForLeader()
+	if err != nil {
+		t.Fatalf("failed waiting for leader on node2: %s", err.Error())
+	}
+	if exp, got := advAddr1.String(), leader; exp != got {
+		t.Fatalf("node return wrong leader from follower, exp: %s, got %s", exp, got)
+	}
+}
+
 // Test_MultiNodeClusterNodes checks nodes/ endpoint under various situations.
 func Test_MultiNodeClusterNodes(t *testing.T) {
 	node1 := mustNewLeaderNode()

system_test/helpers.go

@@ -585,6 +585,29 @@ func mustNewOpenTLSMux(certFile, keyPath, addr string) *tcp.Mux {
 	return mux
 }
 
+func mustTCPListener(bind string) net.Listener {
+	l, err := net.Listen("tcp", bind)
+	if err != nil {
+		panic(err)
+	}
+	return l
+}
+
+func mustGetLocalIPv4Address() string {
+	addrs, err := net.InterfaceAddrs()
+	if err != nil {
+		panic(fmt.Sprintf("failed to get interface addresses: %s", err.Error()))
+	}
+	for _, address := range addrs {
+		if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
+			if ipnet.IP.To4() != nil {
+				return ipnet.IP.String()
+			}
+		}
+	}
+	panic("couldn't find a local IPv4 address")
+}
+
 func mustParseDuration(d string) time.Duration {
 	if dur, err := time.ParseDuration(d); err != nil {
 		panic("failed to parse duration")