More TLS unit test fixes

cluster/service_mux_test.go

@@ -1,12 +1,14 @@
 package cluster
 
 import (
+	"crypto/tls"
 	"fmt"
 	"net"
 	"os"
 	"testing"
 	"time"
 
+	"github.com/rqlite/rqlite/rtls"
 	"github.com/rqlite/rqlite/tcp"
 	"github.com/rqlite/rqlite/testdata/x509"
 )
@@ -104,7 +106,7 @@ func mustNewTLSMux() (net.Listener, *tcp.Mux) {
 	key := x509.KeyFile("")
 	defer os.Remove(key)
 
-	mux, err := tcp.NewTLSMux(ln, nil, cert, key, "", true)
+	mux, err := tcp.NewTLSMux(ln, nil, cert, key, "", "", "", true)
 	if err != nil {
 		panic(fmt.Sprintf("failed to create TLS mux: %s", err))
 	}
@@ -113,5 +115,13 @@ func mustNewTLSMux() (net.Listener, *tcp.Mux) {
 }
 
 func mustNewDialer(header byte, remoteEncrypted, skipVerify bool) *tcp.Dialer {
-	return tcp.NewDialer(header, remoteEncrypted, skipVerify)
+	var tlsConfig *tls.Config
+	var err error
+	if remoteEncrypted {
+		tlsConfig, err = rtls.CreateClientConfig("", "", "", skipVerify, false)
+		if err != nil {
+			panic(fmt.Sprintf("failed to create client TLS config: %s", err))
+		}
+	}
+	return tcp.NewDialer(header, tlsConfig)
 }

cmd/rqlited/main.go

@@ -110,7 +110,15 @@ func main() {
 	// We want to start the HTTP server as soon as possible, so the node is responsive and external
 	// systems can see that it's running. We still have to open the Store though, so the node won't
 	// be able to do much until that happens however.
-	clstrDialer := tcp.NewDialer(cluster.MuxClusterHeader, cfg.NodeEncrypt, cfg.NoNodeVerify)
+	var dialerTLSConfig *tls.Config
+	if cfg.NodeEncrypt {
+		dialerTLSConfig, err = rtls.CreateClientConfig(cfg.NodeX509CertClient, cfg.NodeX509KeyClient,
+			cfg.NodeX509CACert, cfg.NoNodeVerify, cfg.TLS1011)
+		if err != nil {
+			log.Fatalf("failed to create TLS config for cluster dialer: %s", err.Error())
+		}
+	}
+	clstrDialer := tcp.NewDialer(cluster.MuxClusterHeader, dialerTLSConfig)
 	clstrClient := cluster.NewClient(clstrDialer, cfg.ClusterConnectTimeout)
 	if err := clstrClient.SetLocal(cfg.RaftAdv, clstr); err != nil {
 		log.Fatalf("failed to set cluster client local parameters: %s", err.Error())
@@ -129,10 +137,13 @@ func main() {
 	// Register remaining status providers.
 	httpServ.RegisterStatus("cluster", clstr)
 
-	tlsConfig, err := rtls.CreateClientConfig(cfg.X509CertClient, cfg.X509KeyClient, cfg.X509CACert,
-		cfg.NoHTTPVerify, cfg.TLS1011)
-	if err != nil {
-		log.Fatalf("failed to create TLS client config for cluster: %s", err.Error())
+	var clstrTLSConfig *tls.Config
+	if cfg.X509CertClient != "" || cfg.X509CACert != "" {
+		clstrTLSConfig, err = rtls.CreateClientConfig(cfg.X509CertClient, cfg.X509KeyClient, cfg.X509CACert,
+			cfg.NoHTTPVerify, cfg.TLS1011)
+		if err != nil {
+			log.Fatalf("failed to create TLS client config for cluster: %s", err.Error())
+		}
 	}
 
 	// Create the cluster!
@@ -140,7 +151,7 @@ func main() {
 	if err != nil {
 		log.Fatalf("failed to get nodes %s", err.Error())
 	}
-	if err := createCluster(cfg, tlsConfig, len(nodes) > 0, str, httpServ, credStr); err != nil {
+	if err := createCluster(cfg, clstrTLSConfig, len(nodes) > 0, str, httpServ, credStr); err != nil {
 		log.Fatalf("clustering failure: %s", err.Error())
 	}
 
@@ -263,6 +274,7 @@ func startHTTPService(cfg *Config, str *store.Store, cltr *cluster.Client, credS
 		s = httpd.New(cfg.HTTPAddr, str, cltr, nil)
 	}
 
+	s.ClientCACertFile = cfg.X509CACert
 	s.CertFile = cfg.X509Cert
 	s.KeyFile = cfg.X509Key
 	s.TLS1011 = cfg.TLS1011
@@ -294,7 +306,8 @@ func startNodeMux(cfg *Config, ln net.Listener) (*tcp.Mux, error) {
 	var mux *tcp.Mux
 	if cfg.NodeEncrypt {
 		log.Printf("enabling node-to-node encryption with cert: %s, key: %s", cfg.NodeX509Cert, cfg.NodeX509Key)
-		mux, err = tcp.NewTLSMux(ln, adv, cfg.NodeX509Cert, cfg.NodeX509Key, cfg.NodeX509CACert, cfg.NoNodeVerify)
+		mux, err = tcp.NewTLSMux(ln, adv, cfg.NodeX509Cert, cfg.NodeX509Key,
+			cfg.NodeX509CertClient, cfg.NodeX509KeyClient, cfg.NodeX509CACert, cfg.NoNodeVerify)
 	} else {
 		mux, err = tcp.NewMux(ln, adv)
 	}

rtls/config.go

@@ -20,10 +20,12 @@ func CreateClientConfig(certFile, keyFile, caCertFile string, noverify, tls1011
 	var err error
 
 	config := createBaseTLSConfig(noverify, tls1011)
-	config.Certificates = make([]tls.Certificate, 1)
-	config.Certificates[0], err = tls.LoadX509KeyPair(certFile, keyFile)
-	if err != nil {
-		return nil, err
+	if certFile != "" && keyFile != "" {
+		config.Certificates = make([]tls.Certificate, 1)
+		config.Certificates[0], err = tls.LoadX509KeyPair(certFile, keyFile)
+		if err != nil {
+			return nil, err
+		}
 	}
 	if caCertFile != "" {
 		asn1Data, err := ioutil.ReadFile(caCertFile)

tcp/dialer.go

@@ -8,30 +8,25 @@ import (
 )
 
 // NewDialer returns an initialized Dialer
-func NewDialer(header byte, remoteEncrypted, skipVerify bool) *Dialer {
+func NewDialer(header byte, tlsConfig *tls.Config) *Dialer {
 	return &Dialer{
-		header:          header,
-		remoteEncrypted: remoteEncrypted,
-		skipVerify:      skipVerify,
+		header:    header,
+		tlsConfig: tlsConfig,
 	}
 }
 
 // Dialer supports dialing a cluster service.
 type Dialer struct {
-	header          byte
-	remoteEncrypted bool
-	skipVerify      bool
+	header    byte
+	tlsConfig *tls.Config
 }
 
 // Dial dials the cluster service at the given addr and returns a connection.
 func (d *Dialer) Dial(addr string, timeout time.Duration) (conn net.Conn, retErr error) {
 	dialer := &net.Dialer{Timeout: timeout}
 
-	if d.remoteEncrypted {
-		conf := &tls.Config{
-			InsecureSkipVerify: d.skipVerify,
-		}
-		conn, retErr = tls.DialWithDialer(dialer, "tcp", addr, conf)
+	if d.tlsConfig != nil {
+		conn, retErr = tls.DialWithDialer(dialer, "tcp", addr, d.tlsConfig)
 	} else {
 		conn, retErr = dialer.Dial("tcp", addr)
 	}

tcp/dialer_test.go

@@ -13,14 +13,14 @@ import (
 )
 
 func Test_NewDialer(t *testing.T) {
-	d := NewDialer(1, false, false)
+	d := NewDialer(1, nil)
 	if d == nil {
 		t.Fatal("failed to create a dialer")
 	}
 }
 
 func Test_DialerNoConnect(t *testing.T) {
-	d := NewDialer(87, false, false)
+	d := NewDialer(87, nil)
 	_, err := d.Dial("127.0.0.1:0", 5*time.Second)
 	if err == nil {
 		t.Fatalf("no error connecting to bad address")
@@ -32,7 +32,7 @@ func Test_DialerHeader(t *testing.T) {
 	defer s.Close()
 	go s.Start(t)
 
-	d := NewDialer(64, false, false)
+	d := NewDialer(64, nil)
 	conn, err := d.Dial(s.Addr(), 10*time.Second)
 	if err != nil {
 		t.Fatalf("failed to dial echo server: %s", err.Error())
@@ -84,7 +84,7 @@ func Test_DialerHeaderTLSBadConnect(t *testing.T) {
 
 	// Connect to a TLS server with an unencrypted client, to make sure
 	// code can handle that misconfig.
-	d := NewDialer(56, false, false)
+	d := NewDialer(56, nil)
 	conn, err := d.Dial(s.Addr(), 5*time.Second)
 	if err != nil {
 		t.Fatalf("failed to dial TLS echo server: %s", err.Error())

tcp/mux.go

@@ -36,16 +36,16 @@ func init() {
 }
 
 // Layer represents the connection between nodes. It can be both used to
-// make connections to other nodes, and receive connections from other
-// nodes.
+// make connections to other nodes (client), and receive connections from other
+// nodes (server)
 type Layer struct {
 	ln   net.Listener
 	addr net.Addr
 
 	dialer *Dialer
 
-	nodeX509CACert string
-	tlsConfig      *tls.Config
+	tlsServerConfig *tls.Config
+	tlsClientConfig *tls.Config
 }
 
 // Dial creates a new network connection.
@@ -80,20 +80,8 @@ type Mux struct {
 	// Out-of-band error logger
 	Logger *log.Logger
 
-	// Path to x509 certifcate authority for this service. If set, then this
-	// be used to verify the identity of other nodes that connect to this node.
-	x509CACert string
-
-	// Path to X509 certificate for this service.
-	x509Cert string
-
-	// Path to X509 key for this service.
-	x509Key string
-
-	// Whether to skip verification of other nodes' certificates.
-	insecureSkipVerify bool
-
-	tlsConfig *tls.Config
+	serverTLSConfig *tls.Config
+	clientTLSConfig *tls.Config
 }
 
 // NewMux returns a new instance of Mux for ln. If adv is nil,
@@ -115,23 +103,24 @@ func NewMux(ln net.Listener, adv net.Addr) (*Mux, error) {
 
 // NewTLSMux returns a new instance of Mux for ln, and encrypts all traffic
 // using TLS. If adv is nil, then the addr of ln is used.
-func NewTLSMux(ln net.Listener, adv net.Addr, cert, key, caCert string, insecure bool) (*Mux, error) {
+func NewTLSMux(ln net.Listener, adv net.Addr, cert, key, clientCert, clientKey, caCert string, insecure bool) (*Mux, error) {
 	mux, err := NewMux(ln, adv)
 	if err != nil {
 		return nil, err
 	}
 
-	mux.tlsConfig, err = rtls.CreateServerConfig(cert, key, caCert, insecure, false)
+	mux.serverTLSConfig, err = rtls.CreateServerConfig(cert, key, caCert, insecure, false)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("cannot create Server TLS config: %s", err)
 	}
 
-	mux.ln = tls.NewListener(ln, mux.tlsConfig)
+	mux.clientTLSConfig, err = rtls.CreateClientConfig(clientCert, clientKey, caCert, insecure, false)
+	if err != nil {
+		return nil, fmt.Errorf("cannot create Client TLS config: %s", err)
+	}
+
+	mux.ln = tls.NewListener(ln, mux.serverTLSConfig)
 	mux.remoteEncrypted = true
-	mux.x509CACert = caCert
-	mux.x509Cert = cert
-	mux.x509Key = key
-	mux.insecureSkipVerify = insecure
 
 	return mux, nil
 }
@@ -139,7 +128,7 @@ func NewTLSMux(ln net.Listener, adv net.Addr, cert, key, caCert string, insecure
 // Serve handles connections from ln and multiplexes then across registered listener.
 func (mux *Mux) Serve() error {
 	tlsStr := ""
-	if mux.tlsConfig != nil {
+	if mux.serverTLSConfig != nil {
 		tlsStr = "TLS "
 	}
 	mux.Logger.Printf("%smux serving on %s, advertising %s", tlsStr, mux.ln.Addr().String(), mux.addr)
@@ -177,13 +166,6 @@ func (mux *Mux) Stats() (interface{}, error) {
 		"encrypted": strconv.FormatBool(mux.remoteEncrypted),
 	}
 
-	if mux.remoteEncrypted {
-		s["certificate"] = mux.x509Cert
-		s["key"] = mux.x509Key
-		s["ca_certificate"] = mux.x509CACert
-		s["skip_verify"] = strconv.FormatBool(mux.insecureSkipVerify)
-	}
-
 	return s, nil
 }
 
@@ -242,12 +224,12 @@ func (mux *Mux) Listen(header byte) *Layer {
 	mux.m[header] = ln
 
 	layer := &Layer{
-		ln:             ln,
-		addr:           mux.addr,
-		nodeX509CACert: mux.x509CACert,
-		tlsConfig:      mux.tlsConfig,
+		ln:              ln,
+		addr:            mux.addr,
+		tlsServerConfig: mux.serverTLSConfig,
+		tlsClientConfig: mux.clientTLSConfig,
 	}
-	layer.dialer = NewDialer(header, mux.remoteEncrypted, mux.insecureSkipVerify)
+	layer.dialer = NewDialer(header, mux.clientTLSConfig)
 
 	return layer
 }

tcp/mux_test.go

@@ -177,7 +177,7 @@ func TestTLSMux(t *testing.T) {
 	key := x509.KeyFile("")
 	defer os.Remove(key)
 
-	mux, err := NewTLSMux(tcpListener, nil, cert, key, "", true)
+	mux, err := NewTLSMux(tcpListener, nil, cert, key, "", "", "", true)
 	if err != nil {
 		t.Fatalf("failed to create mux: %s", err.Error())
 	}
@@ -200,7 +200,7 @@ func TestTLSMux(t *testing.T) {
 func TestTLSMux_Fail(t *testing.T) {
 	tcpListener := mustTCPListener("127.0.0.1:0")
 	defer tcpListener.Close()
-	_, err := NewTLSMux(tcpListener, nil, "xxxx", "yyyy", "", true)
+	_, err := NewTLSMux(tcpListener, nil, "xxxx", "yyyy", "", "", "", true)
 	if err == nil {
 		t.Fatalf("created mux unexpectedly with bad resources")
 	}