Merge pull request #1100 from rqlite/e2e-key-error

HTTP server now up faster, Store may not be ready

cluster/join.go

@@ -104,6 +104,8 @@ func (j *Joiner) Do(joinAddrs []string, id, addr string, voter bool) (string, er
 			if err == nil {
 				// Success!
 				return joinee, nil
+			} else {
+				j.logger.Printf("failed to join via node at %s: %s", a, err)
 			}
 		}
 		if i+1 < j.numAttempts {
@@ -166,14 +168,14 @@ func (j *Joiner) join(joinAddr, id, addr string, voter bool) (string, error) {
 			// protocol a registered node is actually using.
 			if strings.HasPrefix(fullAddr, "https://") {
 				// It's already HTTPS, give up.
-				return "", fmt.Errorf("failed to join, node returned: %s: (%s)", resp.Status, string(b))
+				return "", fmt.Errorf("%s: (%s)", resp.Status, string(b))
 			}
 
 			j.logger.Print("join via HTTP failed, trying via HTTPS")
 			fullAddr = rurl.EnsureHTTPS(fullAddr)
 			continue
 		default:
-			return "", fmt.Errorf("failed to join, node returned: %s: (%s)", resp.Status, string(b))
+			return "", fmt.Errorf("%s: (%s)", resp.Status, string(b))
 		}
 	}
 }

system_test/full_system_test.py

@@ -329,15 +329,17 @@ class Node(object):
     except requests.exceptions.ConnectionError:
       return ''
 
-  def wait_for_leader(self, timeout=TIMEOUT):
+  def wait_for_leader(self, timeout=TIMEOUT, log=True):
     lr = None
     t = 0
     while lr == None or lr['addr'] == '':
       if t > timeout:
+        if log:
+          self.dump_log("dumping log due to timeout waiting for leader")
         raise Exception('rqlite node failed to detect leader within %d seconds' % timeout)
       try:
         lr = self.status()['store']['leader']
-      except requests.exceptions.ConnectionError:
+      except (KeyError, requests.exceptions.ConnectionError):
         pass
       time.sleep(1)
       t+=1
@@ -347,6 +349,13 @@ class Node(object):
       raise Exception('leader is available but node reports not ready')
     return lr
 
+  def expect_leader_fail(self, timeout=TIMEOUT):
+    try:
+      self.wait_for_leader(self, timeout, log=False)
+    except:
+      return True
+    return false
+
   def db_applied_index(self):
     return int(self.status()['store']['db_applied_index'])
 
@@ -1102,7 +1111,7 @@ class TestAuthJoin(unittest.TestCase):
 
     n1 = Node(RQLITED_PATH, '1', auth=self.auth_file.name)
     n1.start(join=n0.APIAddr())
-    self.assertRaises(Exception, n1.wait_for_leader) # Join should fail due to lack of auth.
+    self.assertTrue(n1.expect_leader_fail()) # Join should fail due to lack of auth.
 
     n2 = Node(RQLITED_PATH, '2', auth=self.auth_file.name)
     n2.start(join=n0.APIAddr(), join_as="foo")