Merge branch 'master' of github.com:rqlite/rqlite into http-before-join

CHANGELOG.md

@@ -1,4 +1,7 @@
-## 6.4.4 (unreleased)
+## 6.5.0 (unreleased)
+### New features
+- [PR #896](https://github.com/rqlite/rqlite/pull/896): Add `/readyz` endpoint for easy ready-to-respond checks.
+
 ### Implementation changes and bug fixes
 - [PR #885](https://github.com/rqlite/rqlite/pull/885): Improved responses on HTTP 500.
 - [PR #888](https://github.com/rqlite/rqlite/pull/888): Expose stats about BoltDB on the `status/` endpoint.

DOC/DIAGNOSTICS.md

@@ -62,6 +62,14 @@ Welcome to the rqlite CLI. Enter ".help" for usage hints.
   leader: false
  ```
 
+ ## Readiness checks
+ rqlite nodes serve a "ready" status at `/readyz`. The endpoint will return `HTTP 200 OK` if the node is ready to respond to database requests and cluster management operations. An example access is shown below.
+
+ ```bash
+ $ curl localhost:4001/readyz
+[+]leader ok
+```
+
 ## expvar support
 rqlite also exports [expvar](http://godoc.org/pkg/expvar/) information. The standard expvar information, as well as some custom information, is exposed. This data can be retrieved like so (assuming the node is started in its default configuration):
 

DOC/SECURITY.md

@@ -42,7 +42,8 @@ rqlite, via the configuration file, also supports user-level permissions. Each u
 - _query_: user may access the query endpoint.
 - _load_: user may load an SQLite dump file into a node.
 - _backup_: user may perform backups.
-- _status_: user can retrieve status and Go runtime information.
+- _status_: user can retrieve node status and Go runtime information.
+- _ready_: user can retrieve node readiness.
 - _join_: user can join a cluster. In practice only a node joins a cluster, so it's the joining node that must supply the credentials.
 - _remove_: user can remove a node from a cluster.
 

http/service.go

@@ -157,6 +157,8 @@ const (
 	PermQuery = "query"
 	// PermStatus means user can retrieve node status.
 	PermStatus = "status"
+	// PermReady means user can retrieve ready status.
+	PermReady = "ready"
 	// PermBackup means user can backup node.
 	PermBackup = "backup"
 	// PermLoad means user can load a SQLite dump into a node.
@@ -316,6 +318,8 @@ func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		s.handleStatus(w, r)
 	case strings.HasPrefix(r.URL.Path, "/nodes"):
 		s.handleNodes(w, r)
+	case strings.HasPrefix(r.URL.Path, "/readyz"):
+		s.handleReadyz(w, r)
 	case r.URL.Path == "/debug/vars" && s.Expvar:
 		s.handleExpvar(w, r)
 	case strings.HasPrefix(r.URL.Path, "/debug/pprof") && s.Pprof:
@@ -757,6 +761,42 @@ func (s *Service) handleNodes(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// handleReadyz returns whether the node is ready.
+func (s *Service) handleReadyz(w http.ResponseWriter, r *http.Request) {
+	if !s.CheckRequestPerm(r, PermReady) {
+		w.WriteHeader(http.StatusUnauthorized)
+		return
+	}
+
+	if r.Method != "GET" {
+		w.WriteHeader(http.StatusMethodNotAllowed)
+		return
+	}
+
+	timeout, err := timeoutParam(r, defaultTimeout)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	lAddr, err := s.store.LeaderAddr()
+	if err != nil {
+		http.Error(w, fmt.Sprintf("leader address: %s", err.Error()),
+			http.StatusInternalServerError)
+		return
+	}
+
+	if lAddr != "" {
+		if _, err := s.cluster.GetNodeAPIAddr(lAddr, timeout); err == nil {
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte("[+]leader ok"))
+			return
+		}
+	}
+
+	w.WriteHeader(http.StatusServiceUnavailable)
+}
+
 // handleExecute handles queries that modify the database.
 func (s *Service) handleExecute(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json; charset=utf-8")

http/service_test.go

@@ -412,6 +412,7 @@ func Test_401Routes_NoBasicAuth(t *testing.T) {
 		"/delete",
 		"/status",
 		"/nodes",
+		"/readyz",
 		"/debug/vars",
 		"/debug/pprof/cmdline",
 		"/debug/pprof/profile",
@@ -451,6 +452,7 @@ func Test_401Routes_BasicAuthBadPassword(t *testing.T) {
 		"/join",
 		"/status",
 		"/nodes",
+		"/readyz",
 		"/debug/vars",
 		"/debug/pprof/cmdline",
 		"/debug/pprof/profile",
@@ -495,6 +497,8 @@ func Test_401Routes_BasicAuthBadPerm(t *testing.T) {
 		"/db/load",
 		"/join",
 		"/status",
+		"/nodes",
+		"/readyz",
 		"/debug/vars",
 		"/debug/pprof/cmdline",
 		"/debug/pprof/profile",
@@ -677,6 +681,30 @@ func Test_Nodes(t *testing.T) {
 	}
 }
 
+func Test_Readyz(t *testing.T) {
+	m := &MockStore{
+		leaderAddr: "foo:1234",
+	}
+	c := &mockClusterService{
+		apiAddr: "https://bar:5678",
+	}
+	s := New("127.0.0.1:0", m, c, nil)
+	if err := s.Start(); err != nil {
+		t.Fatalf("failed to start service")
+	}
+	defer s.Close()
+
+	client := &http.Client{}
+	host := fmt.Sprintf("http://%s", s.Addr().String())
+	resp, err := client.Get(host + "/readyz")
+	if err != nil {
+		t.Fatalf("failed to make nodes request")
+	}
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("failed to get expected StatusOK for nodes, got %d", resp.StatusCode)
+	}
+}
+
 func Test_ForwardingRedirectQuery(t *testing.T) {
 	m := &MockStore{
 		leaderAddr: "foo:1234",

system_test/full_system_test.py

@@ -131,6 +131,10 @@ class Node(object):
     raise_for_status(r)
     return r.json()
 
+  def ready(self):
+    r = requests.get(self._ready_url())
+    return r.status_code == 200
+
   def expvar(self):
     r = requests.get(self._expvar_url())
     raise_for_status(r)
@@ -164,6 +168,9 @@ class Node(object):
       time.sleep(1)
       t+=1
 
+    # Perform a check on readyness while we're here.
+    if self.ready() is not True:
+      raise Exception('leader is available but node reports not ready')
     return lr
 
   def db_applied_index(self):
@@ -285,6 +292,8 @@ class Node(object):
     return 'http://' + self.APIAddr() + '/status'
   def _nodes_url(self):
     return 'http://' + self.APIAddr() + '/nodes?nonvoters' # Getting all nodes back makes testing easier
+  def _ready_url(self):
+    return 'http://' + self.APIAddr() + '/readyz'
   def _expvar_url(self):
     return 'http://' + self.APIAddr() + '/debug/vars'
   def _query_url(self, redirect=False):

system_test/helpers.go

@@ -241,6 +241,17 @@ func (n *Node) Status() (string, error) {
 	return string(body), nil
 }
 
+// Ready returns the ready status for the node
+func (n *Node) Ready() (bool, error) {
+	v, _ := url.Parse("http://" + n.APIAddr + "/readyz")
+
+	resp, err := http.Get(v.String())
+	if err != nil {
+		return false, err
+	}
+	return resp.StatusCode == 200, nil
+}
+
 // Expvar returns the expvar output for node.
 func (n *Node) Expvar() (string, error) {
 	v, _ := url.Parse("http://" + n.APIAddr + "/debug/vars")

system_test/single_node_test.go

@@ -32,6 +32,26 @@ func Test_SingleNodeBasicEndpoint(t *testing.T) {
 	if err != nil {
 		t.Fatalf(`failed to retrieve status for on-disk: %s`, err)
 	}
+
+	ready, err := node.Ready()
+	if err != nil {
+		t.Fatalf(`failed to retrieve readiness: %s`, err)
+	}
+	if !ready {
+		t.Fatalf("node is not ready")
+	}
+}
+
+func Test_SingleNodeNotReady(t *testing.T) {
+	node := mustNewNode(false)
+	defer node.Deprovision()
+	ready, err := node.Ready()
+	if err != nil {
+		t.Fatalf(`failed to retrieve readiness: %s`, err)
+	}
+	if ready {
+		t.Fatalf("node is ready when it should not be")
+	}
 }
 
 func Test_SingleNode(t *testing.T) {