Remove BSON support

master
Nicolas Favre-Felix 12 years ago
parent 8a8ab845a3
commit 2c634fa071

@ -1,10 +1,10 @@
OUT=webdis
HIREDIS_OBJ=hiredis/hiredis.o hiredis/sds.o hiredis/net.o hiredis/async.o
JANSSON_OBJ=jansson/src/dump.o jansson/src/error.o jansson/src/hashtable.o jansson/src/load.o jansson/src/strbuffer.o jansson/src/utf.o jansson/src/value.o jansson/src/variadic.o
FORMAT_OBJS=formats/json.o formats/raw.o formats/common.o formats/custom-type.o formats/bson.o
FORMAT_OBJS=formats/json.o formats/raw.o formats/common.o formats/custom-type.o
HTTP_PARSER_OBJS=http-parser/http_parser.o
CFLAGS=-O3 -Wall -Wextra -I. -Ijansson/src -Ihttp-parser
CFLAGS=-O0 -ggdb -Wall -Wextra -I. -Ijansson/src -Ihttp-parser
LDFLAGS=-levent -pthread
# check for MessagePack

@ -23,7 +23,6 @@ curl -d "GET/hello" http://127.0.0.1:7379/
* GET and POST are supported, as well as `PUT` for file uploads.
* JSON output by default, optional JSONP parameter (`?jsonp=myFunction` or `?callback=myFunction`).
* Raw Redis 2.0 protocol output with `.raw` suffix
* BSON support for compact responses and MongoDB compatibility.
* MessagePack output with `.msg` suffix
* HTTP 1.1 pipelining (70,000 http requests per second on a desktop Linux machine.)
* Multi-threaded server, configurable number of worker threads.
@ -173,7 +172,6 @@ $ curl http://127.0.0.1:7379/MAKE-ME-COFFEE.raw
Several content-types are available:
* `.json` for `application/json` (this is the default Content-Type).
* `.bson` for `application/bson`. See [http://bsonspec.org/](http://bsonspec.org/) for the specs.
* `.msg` for `application/x-msgpack`. See [http://msgpack.org/](http://msgpack.org/) for the specs.
* `.txt` for `text/plain`
* `.html` for `text/html`

@ -8,7 +8,6 @@
#include "server.h"
#include "formats/json.h"
#include "formats/bson.h"
#include "formats/raw.h"
#ifdef MSGPACK
#include "formats/msgpack.h"
@ -302,7 +301,6 @@ cmd_select_format(struct http_client *client, struct cmd *cmd,
struct reply_format funs[] = {
{.s = "json", .sz = 4, .f = json_reply, .ct = "application/json"},
{.s = "raw", .sz = 3, .f = raw_reply, .ct = "binary/octet-stream"},
{.s = "bson", .sz = 4, .f = bson_reply, .ct = "application/bson"},
#ifdef MSGPACK
{.s = "msg", .sz = 3, .f = msgpack_reply, .ct = "application/x-msgpack"},

@ -1,442 +0,0 @@
#include "bson.h"
#include "common.h"
#include "cmd.h"
#include "http.h"
#include <string.h>
#include <hiredis/hiredis.h>
#include <hiredis/async.h>
static bson_t *
bson_wrap_redis_reply(const struct cmd *cmd, const redisReply *r);
static char *
bson_string_output(bson_t *b, size_t *sz);
void
bson_reply(redisAsyncContext *c, void *r, void *privdata) {
redisReply *reply = r;
struct cmd *cmd = privdata;
bson_t *b;
char *bstr = NULL;
size_t bsz;
(void)c;
if(cmd == NULL) {
/* broken connection */
return;
}
if (reply == NULL) { /* broken Redis link */
format_send_error(cmd, 503, "Service Unavailable");
return;
}
/* encode redis reply as BSON */
b = bson_wrap_redis_reply(cmd, reply);
/* get BSON as string */
bstr = bson_string_output(b, &bsz);
/* send reply */
format_send_reply(cmd, bstr, bsz, "application/bson");
/* cleanup */
free(bstr);
bson_free(b);
}
/**
* Parse info message and return object.
*/
static bson_t *
bson_info_reply(const char *s, size_t sz) {
const char *p = s;
bson_t *broot = bson_object();
/* TODO: handle new format */
while(p < s + sz) {
char *key, *val, *nl, *colon;
size_t key_len, val_len;
/* find key */
colon = strchr(p, ':');
if(!colon) {
break;
}
key = calloc(colon - p + 1, 1);
key_len = colon - p;
memcpy(key, p, key_len);
p = colon + 1;
/* find value */
nl = strchr(p, '\r');
if(!nl) {
free(key);
break;
}
val = calloc(nl - p + 1, 1);
val_len = nl - p;
memcpy(val, p, val_len);
p = nl + 1;
if(*p == '\n') p++;
/* add to object */
bson_object_set_new(broot, key, key_len,
bson_string(val, val_len));
free(key);
free(val);
}
return broot;
}
static bson_t *
bson_hgetall_reply(const redisReply *r) {
/* zip keys and values together in a bson object */
bson_t *broot;
unsigned int i;
if(r->elements % 2 != 0) {
return NULL;
}
broot = bson_object();
for(i = 0; i < r->elements; i += 2) {
redisReply *k = r->element[i], *v = r->element[i+1];
/* keys and values need to be strings */
if(k->type != REDIS_REPLY_STRING || v->type != REDIS_REPLY_STRING) {
bson_free(broot);
return NULL;
}
bson_object_set_new(broot, k->str, k->len, bson_string(v->str, v->len));
}
return broot;
}
static bson_t *
bson_wrap_redis_reply(const struct cmd *cmd, const redisReply *r) {
unsigned int i;
bson_t *blist, *broot = bson_object(); /* that's what we return */
const char *verb = cmd->argv[0];
size_t verb_sz = cmd->argv_len[0];
switch(r->type) {
case REDIS_REPLY_STATUS:
case REDIS_REPLY_ERROR:
blist = bson_array();
bson_array_append_new(blist,
r->type == REDIS_REPLY_ERROR ? bson_false() : bson_true());
bson_array_append_new(blist, bson_bin(r->str, r->len));
bson_object_set_new(broot, verb, verb_sz, blist);
break;
case REDIS_REPLY_STRING:
if(strncasecmp(verb, "INFO", verb_sz) == 0) {
bson_object_set_new(broot, verb, verb_sz,
bson_info_reply(r->str, r->len));
} else {
bson_object_set_new(broot, verb, verb_sz,
bson_bin(r->str, r->len));
}
break;
case REDIS_REPLY_INTEGER:
bson_object_set_new(broot, verb, verb_sz, bson_integer(r->integer));
break;
case REDIS_REPLY_ARRAY:
if(strcasecmp(verb, "HGETALL") == 0) {
bson_t *bobj = bson_hgetall_reply(r);
if(bobj) {
bson_object_set_new(broot, verb, verb_sz, bobj);
break;
}
}
blist = bson_array();
for(i = 0; i < r->elements; ++i) {
redisReply *e = r->element[i];
switch(e->type) {
case REDIS_REPLY_STRING:
bson_array_append_new(blist,
bson_bin(e->str, e->len));
break;
case REDIS_REPLY_INTEGER:
bson_array_append_new(blist, bson_integer(e->integer));
break;
default:
bson_array_append_new(blist, bson_null());
break;
}
}
bson_object_set_new(broot, verb, verb_sz, blist);
break;
default:
bson_object_set_new(broot, verb, verb_sz, bson_null());
break;
}
return broot;
}
static bson_t *
bson_new() {
return calloc(sizeof(bson_t), 1);
}
void
bson_free(bson_t *b) {
int i;
switch(b->type) {
case BSON_STRING:
case BSON_BIN:
free(b->data.bin.s);
break;
case BSON_OBJECT:
case BSON_ARRAY:
for(i = 0; i < b->data.array.count; ++i) {
bson_free(b->data.array.elements[i]);
}
free(b->data.array.elements);
break;
default:
break;
}
free(b);
}
bson_t *
bson_array() {
bson_t *b = bson_new();
b->type = BSON_ARRAY;
return b;
}
bson_t *
bson_object() {
bson_t *b = bson_new();
b->type = BSON_OBJECT;
return b;
}
bson_t *
bson_null() {
bson_t *b = bson_new();
b->type = BSON_NULL;
return b;
}
bson_t *
bson_true() {
bson_t *b = bson_new();
b->type = BSON_TRUE;
return b;
}
bson_t *
bson_false() {
bson_t *b = bson_new();
b->type = BSON_FALSE;
return b;
}
bson_t *
bson_integer(long long i) {
bson_t *b = bson_new();
b->type = BSON_INT;
b->data.i = i;
return b;
}
bson_t *
bson_bin(const char *s, size_t sz) {
bson_t *b = bson_new();
b->type = BSON_BIN;
b->data.bin.s = malloc(sz);
memcpy(b->data.bin.s, s, sz);
b->data.bin.sz = sz;
return b;
}
bson_t *
bson_string(const char *s, size_t sz) {
bson_t *b = bson_new();
b->type = BSON_STRING;
b->data.bin.s = malloc(sz);
memcpy(b->data.bin.s, s, sz);
b->data.bin.sz = sz;
return b;
}
static void
bson_array_append_raw(bson_t *b, bson_t *e) {
if(b->type != BSON_ARRAY && b->type != BSON_OBJECT) {
return;
}
b->data.array.count++;
b->data.array.elements = realloc(b->data.array.elements,
b->data.array.count * sizeof(bson_t*));
b->data.array.elements[b->data.array.count-1] = e;
}
void
bson_array_append_new(bson_t *b, bson_t *e) {
char s[20];
int sz = sprintf(s, "%d", b->data.array.count / 2);
bson_array_append_raw(b, bson_string(s, sz));
bson_array_append_raw(b, e);
}
void
bson_object_set_new(bson_t *b, const char *k, size_t sz, bson_t *v) {
bson_array_append_raw(b, bson_string(k, sz));
bson_array_append_raw(b, v);
}
static char *
bson_string_output_object(bson_t *b, size_t *out_sz) {
int i;
size_t sz = 0;
char *s = NULL;
for(i = 0; i < b->data.array.count; i += 2) {
char type = '\x00';
/* key is always string. */
const char *key = b->data.array.elements[i]->data.bin.s;
size_t key_sz = b->data.array.elements[i]->data.bin.sz;
/* val can be anything. */
bson_t *bval = b->data.array.elements[i+1];
switch(bval->type) {
case BSON_OBJECT:
type = '\x03';
break;
case BSON_TRUE:
case BSON_FALSE:
type = '\x08';
break;
case BSON_INT:
type = '\x12';
break;
case BSON_STRING:
type = '\x02';
break;
case BSON_BIN:
type = '\x05';
break;
case BSON_ARRAY:
type = '\x04';
break;
case BSON_NULL:
type = '\x0A';
break;
}
if(type) {
size_t val_sz;
char *val = bson_string_output(bval, &val_sz);
s = realloc(s, sz + 1 + key_sz + 1 + val_sz);
s[sz] = type; /* type */
memcpy(s + sz + 1, key, key_sz); /* key */
s[sz + 1 + key_sz] = 0; /* end of key */
memcpy(s + sz + 1 + key_sz + 1, val, val_sz);
sz += 1 + key_sz + 1 + val_sz;
free(val);
}
}
*out_sz = sz;
return s;
}
static char *
bson_string_output(bson_t *b, size_t *sz) {
char *s = NULL, *s_obj = NULL;
size_t sz_obj;
*sz = 0;
int64_t i64;
int32_t i32;
switch(b->type) {
case BSON_ARRAY:
case BSON_OBJECT:
s_obj = bson_string_output_object(b, &sz_obj);
*sz = sizeof(i32) + sz_obj + 1;
s = malloc(*sz);
i32 = *sz;
memcpy(s, &i32, sizeof(i32));
memcpy(s + sizeof(i32), s_obj, sz_obj);
s[*sz-1] = 0;
free(s_obj);
break;
case BSON_TRUE:
*sz = 1;
s = malloc(*sz);
*s = 1;
break;
case BSON_FALSE:
*sz = 1;
s = malloc(*sz);
*s = 0;
break;
case BSON_INT:
*sz = 8;
i64 = b->data.i;
s = malloc(*sz);
memcpy(s, &i64, sizeof(i64));
break;
case BSON_BIN:
i32 = b->data.bin.sz;
*sz = sizeof(i32) + 1 + i32;
s = malloc(*sz);
memcpy(s, &i32, sizeof(i32));
s[sizeof(i32)] = 0;
memcpy(s + sizeof(i32) + 1, b->data.bin.s, b->data.bin.sz);
break;
case BSON_STRING:
i32 = b->data.bin.sz + 1;
*sz = sizeof(i32) + i32;
s = malloc(*sz);
memcpy(s, &i32, sizeof(i32));
memcpy(s + sizeof(i32), b->data.bin.s, b->data.bin.sz);
s[*sz - 1] = 0;
break;
case BSON_NULL:
*sz = 0;
return NULL;
break;
}
return s;
}

@ -1,68 +0,0 @@
#ifndef BSON_H
#define BSON_H
#include <jansson.h>
#include <hiredis/hiredis.h>
#include <hiredis/async.h>
struct cmd;
void
bson_reply(redisAsyncContext *c, void *r, void *privdata);
typedef struct bson_t {
enum {BSON_OBJECT, BSON_TRUE, BSON_FALSE, BSON_INT,
BSON_BIN, BSON_STRING, BSON_ARRAY, BSON_NULL} type;
union {
long long i;
struct {
char *s;
size_t sz;
} bin;
struct {
struct bson_t **elements;
int count;
} array;
} data;
} bson_t;
/* BSON encoding */
void
bson_free(bson_t *b);
bson_t *
bson_array();
bson_t *
bson_object();
bson_t *
bson_null();
bson_t *
bson_true();
bson_t *
bson_false();
bson_t *
bson_integer(long long i);
bson_t *
bson_string(const char *s, size_t sz);
bson_t *
bson_bin(const char *s, size_t sz);
void
bson_array_append_new(bson_t *b, bson_t *e);
void
bson_object_set_new(bson_t *b, const char *k, size_t sz, bson_t *v);
#endif

@ -1,10 +1,6 @@
#!/usr/bin/python
import urllib2, unittest, json, hashlib
from functools import wraps
try:
import bson
except:
bson = None
try:
import msgpack
except:
@ -143,61 +139,6 @@ class TestRaw(TestWebdis):
f = self.query('UNKNOWN/COMMAND.raw')
self.assertTrue(f.read().startswith("-ERR "))
def need_bson(fn):
def wrapper(self):
if bson:
fn(self)
return wrapper
class TestBSon(TestWebdis):
@need_bson
def test_set(self):
"success type (+OK)"
self.query('DEL/hello')
f = self.query('SET/hello/world.bson')
self.assertTrue(f.headers.getheader('Content-Type') == 'application/bson')
obj = bson.decode_all(f.read())
self.assertTrue(obj == [{u'SET': [True, bson.Binary('OK', 0)]}])
@need_bson
def test_get(self):
"string type"
self.query('SET/hello/world')
f = self.query('GET/hello.bson')
obj = bson.decode_all(f.read())
self.assertTrue(obj == [{u'GET': bson.Binary('world', 0)}])
@need_bson
def test_incr(self):
"integer type"
self.query('DEL/hello')
f = self.query('INCR/hello.bson')
obj = bson.decode_all(f.read())
self.assertTrue(obj == [{u'INCR': 1L}])
@need_bson
def test_list(self):
"list type"
self.query('DEL/hello')
self.query('RPUSH/hello/abc')
self.query('RPUSH/hello/def')
f = self.query('LRANGE/hello/0/-1.bson')
obj = bson.decode_all(f.read())
self.assertTrue(obj == [{u'LRANGE': [bson.Binary('abc', 0), bson.Binary('def', 0)]}])
@need_bson
def test_error(self):
"error return type"
f = self.query('UNKNOWN/COMMAND.bson')
obj = bson.decode_all(f.read())
self.assertTrue(len(obj) == 1)
self.assertTrue(u'UNKNOWN' in obj[0])
self.assertTrue(isinstance(obj[0], dict))
self.assertTrue(isinstance(obj[0][u'UNKNOWN'], list))
self.assertTrue(obj[0]['UNKNOWN'][0] == False)
self.assertTrue(isinstance(obj[0]['UNKNOWN'][1], bson.Binary))
def need_msgpack(fn):
def wrapper(self):
if msgpack:

@ -6,7 +6,9 @@
"http_host": "0.0.0.0",
"http_port": 7379,
"threads": 3,
"threads": 5,
"pool_size": 20,
"daemonize": false,
"websockets": false,

Loading…
Cancel
Save