From 0d959f174ee9a1bf33ea69161460023c32b8fb86 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Sat, 18 Nov 2023 11:50:08 -0500 Subject: [PATCH] Fix system-level testing of Bootstrapping --- cmd/rqlited/main.go | 28 +++++----------------------- system_test/cluster_test.go | 18 ++++++++++-------- tcp/dialer.go | 2 +- 3 files changed, 16 insertions(+), 32 deletions(-) diff --git a/cmd/rqlited/main.go b/cmd/rqlited/main.go index d37dce5a..a4e6c535 100644 --- a/cmd/rqlited/main.go +++ b/cmd/rqlited/main.go @@ -158,18 +158,12 @@ func main() { httpServ.RegisterStatus("cluster", clstrServ) httpServ.RegisterStatus("network", tcp.NetworkReporter{}) - // Prepare the cluster-joiner - joiner, err := createJoiner(cfg, credStr) - if err != nil { - log.Fatalf("failed to create cluster joiner: %s", err.Error()) - } - // Create the cluster! nodes, err := str.Nodes() if err != nil { log.Fatalf("failed to get nodes %s", err.Error()) } - if err := createCluster(cfg, len(nodes) > 0, joiner, str, httpServ, credStr); err != nil { + if err := createCluster(cfg, len(nodes) > 0, clstrClient, str, httpServ, credStr); err != nil { log.Fatalf("clustering failure: %s", err.Error()) } @@ -427,14 +421,6 @@ func credentialStore(cfg *Config) (*auth.CredentialsStore, error) { return auth.NewCredentialsStoreFromFile(cfg.AuthFile) } -func createJoiner(cfg *Config, credStr *auth.CredentialsStore) (*cluster.Joiner, error) { - tlsConfig, err := createHTTPTLSConfig(cfg) - if err != nil { - return nil, err - } - return cluster.NewJoiner(cfg.JoinSrcIP, cfg.JoinAttempts, cfg.JoinInterval, tlsConfig), nil -} - func clusterService(cfg *Config, tn cluster.Transport, db cluster.Database, mgr cluster.Manager, credStr *auth.CredentialsStore) (*cluster.Service, error) { c := cluster.New(tn, db, mgr, credStr) c.SetAPIAddr(cfg.HTTPAdv) @@ -463,12 +449,7 @@ func createClusterClient(cfg *Config, clstr *cluster.Service) (*cluster.Client, return clstrClient, nil } -func createCluster(cfg *Config, hasPeers bool, joiner *cluster.Joiner, str *store.Store, httpServ *httpd.Service, credStr *auth.CredentialsStore) error { - tlsConfig, err := createHTTPTLSConfig(cfg) - if err != nil { - return fmt.Errorf("failed to create TLS client config for cluster: %s", err.Error()) - } - +func createCluster(cfg *Config, hasPeers bool, client *cluster.Client, str *store.Store, httpServ *httpd.Service, credStr *auth.CredentialsStore) error { joins := cfg.JoinAddresses() if joins == nil && cfg.DiscoMode == "" && !hasPeers { if cfg.RaftNonVoter { @@ -489,6 +470,7 @@ func createCluster(cfg *Config, hasPeers bool, joiner *cluster.Joiner, str *stor return leader != "" } + joiner := cluster.NewJoiner(client, cfg.JoinAttempts, cfg.JoinInterval) if joins != nil && cfg.BootstrapExpect == 0 { // Explicit join operation requested, so do it. j, err := joiner.Do(joins, str.ID(), cfg.RaftAdv, !cfg.RaftNonVoter) @@ -501,7 +483,7 @@ func createCluster(cfg *Config, hasPeers bool, joiner *cluster.Joiner, str *stor if joins != nil && cfg.BootstrapExpect > 0 { // Bootstrap with explicit join addresses requests. - bs := cluster.NewBootstrapper(cluster.NewAddressProviderString(joins), tlsConfig) + bs := cluster.NewBootstrapper(cluster.NewAddressProviderString(joins), client) return bs.Boot(str.ID(), cfg.RaftAdv, isClustered, cfg.BootstrapExpectTimeout) } @@ -543,7 +525,7 @@ func createCluster(cfg *Config, hasPeers bool, joiner *cluster.Joiner, str *stor provider = dnssrv.New(dnssrvCfg) } - bs := cluster.NewBootstrapper(provider, tlsConfig) + bs := cluster.NewBootstrapper(provider, client) httpServ.RegisterStatus("disco", provider) return bs.Boot(str.ID(), cfg.RaftAdv, isClustered, cfg.BootstrapExpectTimeout) diff --git a/system_test/cluster_test.go b/system_test/cluster_test.go index f386ed8c..0ba9a259 100644 --- a/system_test/cluster_test.go +++ b/system_test/cluster_test.go @@ -505,18 +505,20 @@ func Test_MultiNodeClusterBootstrapLaterJoinHTTPS(t *testing.T) { node3.Store.BootstrapExpect = 3 defer node3.Deprovision() - tlsConfig, err := rtls.CreateClientConfig("", "", "", true) + dialerTLSConfig, err := rtls.CreateClientConfig("", "", "", true) if err != nil { - t.Fatalf("failed to create TLS config: %s", err) + t.Fatalf("failed to create TLS config for cluster dialer: %s", err) } + clstrDialer := tcp.NewDialer(cluster.MuxClusterHeader, dialerTLSConfig) + clstrClient := cluster.NewClient(clstrDialer, 5*time.Second) provider := cluster.NewAddressProviderString( - []string{node1.APIAddr, node2.APIAddr, node3.APIAddr}) - node1Bs := cluster.NewBootstrapper(provider, tlsConfig) + []string{node1.RaftAddr, node2.RaftAddr, node3.RaftAddr}) + node1Bs := cluster.NewBootstrapper(provider, clstrClient) node1Bs.Interval = time.Second - node2Bs := cluster.NewBootstrapper(provider, tlsConfig) + node2Bs := cluster.NewBootstrapper(provider, clstrClient) node2Bs.Interval = time.Second - node3Bs := cluster.NewBootstrapper(provider, tlsConfig) + node3Bs := cluster.NewBootstrapper(provider, clstrClient) node3Bs.Interval = time.Second // Have all nodes start a bootstrap basically in parallel, @@ -571,11 +573,11 @@ func Test_MultiNodeClusterBootstrapLaterJoinHTTPS(t *testing.T) { } // Ensure a 4th node can join cluster with exactly same launch - // params. Under the cover it should just do a join. + // params. Under the covers it should just do a join. node4 := mustNewNodeEncrypted(false, true, true) node4.Store.BootstrapExpect = 3 defer node3.Deprovision() - node4Bs := cluster.NewBootstrapper(provider, tlsConfig) + node4Bs := cluster.NewBootstrapper(provider, clstrClient) node4Bs.Interval = time.Second done := func() bool { addr, _ := node4.Store.LeaderAddr() diff --git a/tcp/dialer.go b/tcp/dialer.go index 147a041b..2df40907 100644 --- a/tcp/dialer.go +++ b/tcp/dialer.go @@ -35,7 +35,7 @@ func (d *Dialer) Dial(addr string, timeout time.Duration) (conn net.Conn, retErr } defer func() { - if retErr != nil { + if retErr != nil && conn != nil { conn.Close() } }()