More testing and docs for /readyz

CHANGELOG.md

@@ -1,4 +1,7 @@
-## 6.4.4 (unreleased)
+## 6.5.0 (unreleased)
+### New features
+- [PR #896](https://github.com/rqlite/rqlite/pull/896): Add `/readyz` endpoint for easy ready-to-respond checks.
+
 ### Implementation changes and bug fixes
 - [PR #885](https://github.com/rqlite/rqlite/pull/885): Improved responses on HTTP 500.
 - [PR #888](https://github.com/rqlite/rqlite/pull/888): Expose stats about BoltDB on the `status/` endpoint.

DOC/DIAGNOSTICS.md

@@ -62,6 +62,14 @@ Welcome to the rqlite CLI. Enter ".help" for usage hints.
   leader: false
  ```
 
+ ## Readiness checks
+ rqlite nodes serve a "ready" status at `/readyz`. The endpoint will return `HTTP 200 OK` if the node is ready to respond to database requests. An example access is shown below.
+
+ ```bash
+ $ curl localhost:4001/readyz
+[+]leader ok
+```
+
 ## expvar support
 rqlite also exports [expvar](http://godoc.org/pkg/expvar/) information. The standard expvar information, as well as some custom information, is exposed. This data can be retrieved like so (assuming the node is started in its default configuration):
 

system_test/full_system_test.py

@@ -131,6 +131,10 @@ class Node(object):
     raise_for_status(r)
     return r.json()
 
+  def ready(self):
+    r = requests.get(self._ready_url())
+    return r.status_code == 200
+
   def expvar(self):
     r = requests.get(self._expvar_url())
     raise_for_status(r)
@@ -164,6 +168,9 @@ class Node(object):
       time.sleep(1)
       t+=1
 
+    # Perform a check on readyness while we're here.
+    if self.ready() is not True:
+      raise Exception('leader is available but node reports not ready')
     return lr
 
   def db_applied_index(self):
@@ -285,6 +292,8 @@ class Node(object):
     return 'http://' + self.APIAddr() + '/status'
   def _nodes_url(self):
     return 'http://' + self.APIAddr() + '/nodes?nonvoters' # Getting all nodes back makes testing easier
+  def _ready_url(self):
+    return 'http://' + self.APIAddr() + '/readyz'
   def _expvar_url(self):
     return 'http://' + self.APIAddr() + '/debug/vars'
   def _query_url(self, redirect=False):

system_test/helpers.go

@@ -241,6 +241,17 @@ func (n *Node) Status() (string, error) {
 	return string(body), nil
 }
 
+// Ready returns the ready status for the node
+func (n *Node) Ready() (bool, error) {
+	v, _ := url.Parse("http://" + n.APIAddr + "/readyz")
+
+	resp, err := http.Get(v.String())
+	if err != nil {
+		return false, err
+	}
+	return resp.StatusCode == 200, nil
+}
+
 // Expvar returns the expvar output for node.
 func (n *Node) Expvar() (string, error) {
 	v, _ := url.Parse("http://" + n.APIAddr + "/debug/vars")

system_test/single_node_test.go

@@ -32,6 +32,26 @@ func Test_SingleNodeBasicEndpoint(t *testing.T) {
 	if err != nil {
 		t.Fatalf(`failed to retrieve status for on-disk: %s`, err)
 	}
+
+	ready, err := node.Ready()
+	if err != nil {
+		t.Fatalf(`failed to retrieve readiness: %s`, err)
+	}
+	if !ready {
+		t.Fatalf("node is not ready")
+	}
+}
+
+func Test_SingleNodeNotReady(t *testing.T) {
+	node := mustNewNode(false)
+	defer node.Deprovision()
+	ready, err := node.Ready()
+	if err != nil {
+		t.Fatalf(`failed to retrieve readiness: %s`, err)
+	}
+	if ready {
+		t.Fatalf("node is ready when it should not be")
+	}
 }
 
 func Test_SingleNode(t *testing.T) {