Merge pull request #1018 from rqlite/no-file-fmt

Autodetect data format during load operation

.circleci/config.yml

@@ -63,6 +63,7 @@ jobs:
       - image: gcr.io/etcd-development/etcd:v3.5.1
     steps:
         - checkout
+        - run: sudo apt-get update
         - run: sudo apt-get install python3
         - run: sudo apt install python3-pip
         - run: python3 -m pip install requests

DOC/RESTORE_FROM_SQLITE.md

@@ -8,7 +8,7 @@ rqlite supports loading a node directly from two sources, either of which can be
 The following examples show a trivial database being generated by `sqlite3`, the SQLite file being backed up, converted to the corresponding list of SQL commands, and then loaded into a rqlite node listening on localhost using each form.
 
 ### HTTP
- _Be sure to set the Content-type header as shown in each case._
+ _Be sure to set the Content-type header as shown, depending on the format of the upload._
 
 ```bash
 ~ $ sqlite3 restore.sqlite
@@ -18,7 +18,7 @@ sqlite> CREATE TABLE foo (id integer not null primary key, name text);
 sqlite> INSERT INTO "foo" VALUES(1,'fiona');
 sqlite>
 # Load directly from the SQLite file.
-~ $ curl -v -XPOST localhost:4001/db/load?fmt=binary -H "Content-type: application/octet-stream" --data-binary @restore.sqlite
+~ $ curl -v -XPOST localhost:4001/db/load -H "Content-type: application/octet-stream" --data-binary @restore.sqlite
 
 # Convert SQLite database file to set of SQL commands and then load
 ~ $ echo '.dump' | sqlite3 restore.sqlite > restore.dump

README.md

@@ -1,6 +1,6 @@
 <img src="DOC/logo-text.png" height=100></img>
 
-[![appveyor](https://ci.appveyor.com/api/projects/status/github/rqlite/rqlite?branch=master&svg=true)](https://ci.appveyor.com/project/otoolep/rqlite) [![Go Report Card](https://goreportcard.com/badge/github.com/rqlite/rqlite)](https://goreportcard.com/report/github.com/rqlite/rqlite) [![Release](https://img.shields.io/github/release/rqlite/rqlite.svg)](https://github.com/rqlite/rqlite/releases) [![Docker](https://img.shields.io/docker/pulls/rqlite/rqlite?style=plastic)](https://hub.docker.com/r/rqlite/rqlite/) [![Google Group](https://img.shields.io/badge/Google%20Group--blue.svg)](https://groups.google.com/group/rqlite) [![Slack](https://img.shields.io/badge/Slack--purple.svg)](https://www.philipotoole.com/join-rqlite-slack)
+[![Circle CI](https://circleci.com/gh/rqlite/rqlite/tree/master.svg?style=svg)](https://circleci.com/gh/rqlite/rqlite/tree/master) [![appveyor](https://ci.appveyor.com/api/projects/status/github/rqlite/rqlite?branch=master&svg=true)](https://ci.appveyor.com/project/otoolep/rqlite) [![Go Report Card](https://goreportcard.com/badge/github.com/rqlite/rqlite)](https://goreportcard.com/report/github.com/rqlite/rqlite) [![Release](https://img.shields.io/github/release/rqlite/rqlite.svg)](https://github.com/rqlite/rqlite/releases) [![Docker](https://img.shields.io/docker/pulls/rqlite/rqlite?style=plastic)](https://hub.docker.com/r/rqlite/rqlite/) [![Google Group](https://img.shields.io/badge/Google%20Group--blue.svg)](https://groups.google.com/group/rqlite) [![Slack](https://img.shields.io/badge/Slack--purple.svg)](https://www.philipotoole.com/join-rqlite-slack)
 
 *rqlite* is an easy-to-use, lightweight, distributed relational database, which uses [SQLite](https://www.sqlite.org/) as its storage engine. rqlite is simple to deploy, operating it is very straightforward, and its clustering capabilities provide you with fault-tolerance and high-availability. [rqlite is available for Linux, macOS, and Microsoft Windows](https://github.com/rqlite/rqlite/releases).
 

http/service.go

@@ -593,18 +593,8 @@ func (s *Service) handleLoad(w http.ResponseWriter, r *http.Request) {
 	}
 	r.Body.Close()
 
-	format, err := fmtParam(r)
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return
-	}
-
-	if strings.ToLower(format) == "binary" {
-		if !validateSQLiteFile(b) {
-			http.Error(w, "invalid SQLite database file", http.StatusBadRequest)
-			return
-		}
-
+	if validSQLiteFile(b) {
+		s.logger.Printf("SQLite database file detected as load data")
 		lr := &command.LoadRequest{
 			Data: b,
 		}
@@ -613,7 +603,6 @@ func (s *Service) handleLoad(w http.ResponseWriter, r *http.Request) {
 			http.Error(w, err.Error(), http.StatusBadRequest)
 			return
 		}
-		s.writeResponse(w, r, resp)
 	} else {
 		// No JSON structure expected for this API.
 		queries := []string{string(b)}
@@ -638,8 +627,8 @@ func (s *Service) handleLoad(w http.ResponseWriter, r *http.Request) {
 			resp.Results.ExecuteResult = results
 		}
 		resp.end = time.Now()
-		s.writeResponse(w, r, resp)
 	}
+	s.writeResponse(w, r, resp)
 }
 
 // handleStatus returns status on the system.
@@ -1550,6 +1539,6 @@ func queryRequestFromStrings(s []string, timings, tx bool) *command.QueryRequest
 
 // validateSQLiteFile checks that the supplied data looks like a SQLite database
 // file. See https://www.sqlite.org/fileformat.html
-func validateSQLiteFile(b []byte) bool {
+func validSQLiteFile(b []byte) bool {
 	return string(b[0:13]) == "SQLite format"
 }

system_test/full_system_test.py

@@ -472,13 +472,13 @@ class Node(object):
     # This is the one API that doesn't expect JSON.
     if fmt != "binary":
       conn = sqlite3.connect(file)
-      r = requests.post(self._load_url(fmt), data='\n'.join(conn.iterdump()))
+      r = requests.post(self._load_url(), data='\n'.join(conn.iterdump()))
       raise_for_status(r)
       conn.close()
     else:
       with open(file, 'rb') as f:
         data = f.read()
-      r = requests.post(self._load_url(fmt), data=data, headers={'Content-Type': 'application/octet-stream'})
+      r = requests.post(self._load_url(), data=data, headers={'Content-Type': 'application/octet-stream'})
       raise_for_status(r)
     return r.json()
 
@@ -516,11 +516,8 @@ class Node(object):
     return 'http://' + self.APIAddr() + '/db/execute' + rd
   def _backup_url(self):
     return 'http://' + self.APIAddr() + '/db/backup'
-  def _load_url(self, fmt=None):
-    f = ""
-    if fmt is not None:
-      f = '?fmt=%s' % (fmt)
-    return 'http://' + self.APIAddr() + '/db/load' + f
+  def _load_url(self):
+    return 'http://' + self.APIAddr() + '/db/load'
   def __eq__(self, other):
     return self.node_id == other.node_id
   def __str__(self):