From fe752c081dd13d4a2a9b9fa4dc2796e4838b0157 Mon Sep 17 00:00:00 2001 From: Dvir Volk Date: Sun, 1 May 2016 18:34:08 +0300 Subject: [PATCH] added example module --- .gitignore | 6 + example/Makefile | 33 ++++ example/module.c | 54 +++++++ redismodule.h | 232 +++++++++++++++++++++++++++ rmutil/Makefile | 8 +- rmutil/sdsalloc.h | 42 +++++ rmutil/strings.c | 1 + rmutil/util.c | 79 ++++++++++ rmutil/util.h | 28 +++- rmutil/zmalloc.c | 395 ++++++++++++++++++++++++++++++++++++++++++++++ rmutil/zmalloc.h | 87 ++++++++++ 11 files changed, 960 insertions(+), 5 deletions(-) create mode 100644 .gitignore create mode 100644 example/Makefile create mode 100644 example/module.c create mode 100644 redismodule.h create mode 100644 rmutil/sdsalloc.h create mode 100644 rmutil/zmalloc.c create mode 100644 rmutil/zmalloc.h diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0961f3e --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +*.o +*.a +*.so +*.db +.vscode + diff --git a/example/Makefile b/example/Makefile new file mode 100644 index 0000000..2c2dc0b --- /dev/null +++ b/example/Makefile @@ -0,0 +1,33 @@ +#set environment variable RM_INCLUDE_DIR to the location of redismodule.h +ifndef RM_INCLUDE_DIR + RM_INCLUDE_DIR=../../ +endif + +# find the OS +uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') + +# Compile flags for linux / osx +ifeq ($(uname_S),Linux) + SHOBJ_CFLAGS ?= -fno-common -g -ggdb + SHOBJ_LDFLAGS ?= -shared +else + SHOBJ_CFLAGS ?= -dynamic -fno-common -g -ggdb + SHOBJ_LDFLAGS ?= -bundle -undefined dynamic_lookup +endif +CFLAGS = -I$(RM_INCLUDE_DIR) -g -fPIC -lc -lm -Og -std=gnu99 +CC=gcc +.SUFFIXES: .c .so .xo .o + +all: module.so + +.c.xo: + $(CC) -I. $(CFLAGS) $(SHOBJ_CFLAGS) -fPIC -c $< -o $@ + +#mdma.xo: ./index.h + +module.so: module.o + $(LD) -o $@ module.o $(SHOBJ_LDFLAGS) $(LIBS) -L../ -lrmutil -lc + +clean: + rm -rf *.xo *.so *.o + diff --git a/example/module.c b/example/module.c new file mode 100644 index 0000000..9c9a421 --- /dev/null +++ b/example/module.c @@ -0,0 +1,54 @@ +#include "../../redismodule.h" +#include "../util.h" +#include "../strings.h" + +/* EXAMPLE.PARSE [SUM ] | [PROD ] +* Demonstrates the automatic arg parsing utility. +* If the command receives "SUM " it returns their sum +* If it receives "PROD " it returns their product +*/ +int ParseCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + + // we must have at least 4 args + if (argc < 4) { + return RedisModule_WrongArity(ctx); + } + + // init auto memory for created strings + RedisModule_AutoMemory(ctx); + long long x,y; + + + // If we got SUM - return the sum of 2 consecutive arguments + if (RMUtil_ParseArgsAfter("SUM", argv, argc, "ll", &x, &y) == REDISMODULE_OK) { + RedisModule_ReplyWithLongLong(ctx, x+y); + return REDISMODULE_OK; + } + // If we got PROD - return the product of 2 consecutive arguments + if (RMUtil_ParseArgsAfter("PROD", argv, argc, "ll", &x, &y) == REDISMODULE_OK) { + RedisModule_ReplyWithLongLong(ctx, x*y); + return REDISMODULE_OK; + } + + // something is fishy... + RedisModule_ReplyWithError(ctx, "Invalid arguments"); + + return REDISMODULE_OK; +} + + +int RedisModule_OnLoad(RedisModuleCtx *ctx) { + + if (RedisModule_Init(ctx,"example",1, REDISMODULE_APIVER_1) == REDISMODULE_ERR) { + return REDISMODULE_ERR; + } + + + if (RedisModule_CreateCommand(ctx, "example.parse", ParseCommand, "readonly", 1,1,1) + == REDISMODULE_ERR) { + return REDISMODULE_ERR; + } + + + return REDISMODULE_OK; +} \ No newline at end of file diff --git a/redismodule.h b/redismodule.h new file mode 100644 index 0000000..d23f8cb --- /dev/null +++ b/redismodule.h @@ -0,0 +1,232 @@ +#ifndef REDISMODULE_H +#define REDISMODULE_H + +#include +#include +#include + +/* ---------------- Defines common between core and modules --------------- */ + +/* Error status return values. */ +#define REDISMODULE_OK 0 +#define REDISMODULE_ERR 1 + +/* API versions. */ +#define REDISMODULE_APIVER_1 1 + +/* API flags and constants */ +#define REDISMODULE_READ (1<<0) +#define REDISMODULE_WRITE (1<<1) + +#define REDISMODULE_LIST_HEAD 0 +#define REDISMODULE_LIST_TAIL 1 + +/* Key types. */ +#define REDISMODULE_KEYTYPE_EMPTY 0 +#define REDISMODULE_KEYTYPE_STRING 1 +#define REDISMODULE_KEYTYPE_LIST 2 +#define REDISMODULE_KEYTYPE_HASH 3 +#define REDISMODULE_KEYTYPE_SET 4 +#define REDISMODULE_KEYTYPE_ZSET 5 + +/* Reply types. */ +#define REDISMODULE_REPLY_UNKNOWN -1 +#define REDISMODULE_REPLY_STRING 0 +#define REDISMODULE_REPLY_ERROR 1 +#define REDISMODULE_REPLY_INTEGER 2 +#define REDISMODULE_REPLY_ARRAY 3 +#define REDISMODULE_REPLY_NULL 4 + +/* Postponed array length. */ +#define REDISMODULE_POSTPONED_ARRAY_LEN -1 + +/* Expire */ +#define REDISMODULE_NO_EXPIRE -1 + +/* Sorted set API flags. */ +#define REDISMODULE_ZADD_XX (1<<0) +#define REDISMODULE_ZADD_NX (1<<1) +#define REDISMODULE_ZADD_ADDED (1<<2) +#define REDISMODULE_ZADD_UPDATED (1<<3) +#define REDISMODULE_ZADD_NOP (1<<4) + +/* Hash API flags. */ +#define REDISMODULE_HASH_NONE 0 +#define REDISMODULE_HASH_NX (1<<0) +#define REDISMODULE_HASH_XX (1<<1) +#define REDISMODULE_HASH_CFIELDS (1<<2) +#define REDISMODULE_HASH_EXISTS (1<<3) + +/* A special pointer that we can use between the core and the module to signal + * field deletion, and that is impossible to be a valid pointer. */ +#define REDISMODULE_HASH_DELETE ((RedisModuleString*)(long)1) + +/* Error messages. */ +#define REDISMODULE_ERRORMSG_WRONGTYPE "WRONGTYPE Operation against a key holding the wrong kind of value" + +#define REDISMODULE_POSITIVE_INFINITE (1.0/0.0) +#define REDISMODULE_NEGATIVE_INFINITE (-1.0/0.0) + +/* ------------------------- End of common defines ------------------------ */ + +#ifndef REDISMODULE_CORE + +typedef long long mstime_t; + +/* Incomplete structures for compiler checks but opaque access. */ +typedef struct RedisModuleCtx RedisModuleCtx; +typedef struct RedisModuleKey RedisModuleKey; +typedef struct RedisModuleString RedisModuleString; +typedef struct RedisModuleCallReply RedisModuleCallReply; + +typedef int (*RedisModuleCmdFunc) (RedisModuleCtx *ctx, RedisModuleString **argv, int argc); + +#define REDISMODULE_GET_API(name) \ + RedisModule_GetApi("RedisModule_" #name, ((void **)&RedisModule_ ## name)) + +#define REDISMODULE_API_FUNC(x) (*x) + +int REDISMODULE_API_FUNC(RedisModule_GetApi)(const char *, void *); +int REDISMODULE_API_FUNC(RedisModule_CreateCommand)(RedisModuleCtx *ctx, const char *name, RedisModuleCmdFunc cmdfunc, const char *strflags, int firstkey, int lastkey, int keystep); +int REDISMODULE_API_FUNC(RedisModule_SetModuleAttribs)(RedisModuleCtx *ctx, const char *name, int ver, int apiver); +int REDISMODULE_API_FUNC(RedisModule_WrongArity)(RedisModuleCtx *ctx); +int REDISMODULE_API_FUNC(RedisModule_ReplyWithLongLong)(RedisModuleCtx *ctx, long long ll); +int REDISMODULE_API_FUNC(RedisModule_GetSelectedDb)(RedisModuleCtx *ctx); +int REDISMODULE_API_FUNC(RedisModule_SelectDb)(RedisModuleCtx *ctx, int newid); +void *REDISMODULE_API_FUNC(RedisModule_OpenKey)(RedisModuleCtx *ctx, RedisModuleString *keyname, int mode); +void REDISMODULE_API_FUNC(RedisModule_CloseKey)(RedisModuleKey *kp); +int REDISMODULE_API_FUNC(RedisModule_KeyType)(RedisModuleKey *kp); +size_t REDISMODULE_API_FUNC(RedisModule_ValueLength)(RedisModuleKey *kp); +int REDISMODULE_API_FUNC(RedisModule_ListPush)(RedisModuleKey *kp, int where, RedisModuleString *ele); +RedisModuleString *REDISMODULE_API_FUNC(RedisModule_ListPop)(RedisModuleKey *key, int where); +RedisModuleCallReply *REDISMODULE_API_FUNC(RedisModule_Call)(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...); +const char *REDISMODULE_API_FUNC(RedisModule_CallReplyProto)(RedisModuleCallReply *reply, size_t *len); +void REDISMODULE_API_FUNC(RedisModule_FreeCallReply)(RedisModuleCallReply *reply); +int REDISMODULE_API_FUNC(RedisModule_CallReplyType)(RedisModuleCallReply *reply); +long long REDISMODULE_API_FUNC(RedisModule_CallReplyInteger)(RedisModuleCallReply *reply); +size_t REDISMODULE_API_FUNC(RedisModule_CallReplyLength)(RedisModuleCallReply *reply); +RedisModuleCallReply *REDISMODULE_API_FUNC(RedisModule_CallReplyArrayElement)(RedisModuleCallReply *reply, size_t idx); +RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateString)(RedisModuleCtx *ctx, const char *ptr, size_t len); +RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromLongLong)(RedisModuleCtx *ctx, long long ll); +void REDISMODULE_API_FUNC(RedisModule_FreeString)(RedisModuleCtx *ctx, RedisModuleString *str); +const char *REDISMODULE_API_FUNC(RedisModule_StringPtrLen)(RedisModuleString *str, size_t *len); +int REDISMODULE_API_FUNC(RedisModule_ReplyWithError)(RedisModuleCtx *ctx, const char *err); +int REDISMODULE_API_FUNC(RedisModule_ReplyWithSimpleString)(RedisModuleCtx *ctx, const char *msg); +int REDISMODULE_API_FUNC(RedisModule_ReplyWithArray)(RedisModuleCtx *ctx, long len); +void REDISMODULE_API_FUNC(RedisModule_ReplySetArrayLength)(RedisModuleCtx *ctx, long len); +int REDISMODULE_API_FUNC(RedisModule_ReplyWithStringBuffer)(RedisModuleCtx *ctx, const char *buf, size_t len); +int REDISMODULE_API_FUNC(RedisModule_ReplyWithString)(RedisModuleCtx *ctx, RedisModuleString *str); +int REDISMODULE_API_FUNC(RedisModule_ReplyWithNull)(RedisModuleCtx *ctx); +int REDISMODULE_API_FUNC(RedisModule_ReplyWithDouble)(RedisModuleCtx *ctx, double d); +int REDISMODULE_API_FUNC(RedisModule_ReplyWithCallReply)(RedisModuleCtx *ctx, RedisModuleCallReply *reply); +int REDISMODULE_API_FUNC(RedisModule_StringToLongLong)(RedisModuleString *str, long long *ll); +int REDISMODULE_API_FUNC(RedisModule_StringToDouble)(RedisModuleString *str, double *d); +void REDISMODULE_API_FUNC(RedisModule_AutoMemory)(RedisModuleCtx *ctx); +int REDISMODULE_API_FUNC(RedisModule_Replicate)(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...); +int REDISMODULE_API_FUNC(RedisModule_ReplicateVerbatim)(RedisModuleCtx *ctx); +const char *REDISMODULE_API_FUNC(RedisModule_CallReplyStringPtr)(RedisModuleCallReply *reply, size_t *len); +RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromCallReply)(RedisModuleCallReply *reply); +int REDISMODULE_API_FUNC(RedisModule_DeleteKey)(RedisModuleKey *key); +int REDISMODULE_API_FUNC(RedisModule_StringSet)(RedisModuleKey *key, RedisModuleString *str); +char *REDISMODULE_API_FUNC(RedisModule_StringDMA)(RedisModuleKey *key, size_t *len, int mode); +int REDISMODULE_API_FUNC(RedisModule_StringTruncate)(RedisModuleKey *key, size_t newlen); +mstime_t REDISMODULE_API_FUNC(RedisModule_GetExpire)(RedisModuleKey *key); +int REDISMODULE_API_FUNC(RedisModule_SetExpire)(RedisModuleKey *key, mstime_t expire); +int REDISMODULE_API_FUNC(RedisModule_ZsetAdd)(RedisModuleKey *key, double score, RedisModuleString *ele, int *flagsptr); +int REDISMODULE_API_FUNC(RedisModule_ZsetIncrby)(RedisModuleKey *key, double score, RedisModuleString *ele, int *flagsptr, double *newscore); +int REDISMODULE_API_FUNC(RedisModule_ZsetScore)(RedisModuleKey *key, RedisModuleString *ele, double *score); +int REDISMODULE_API_FUNC(RedisModule_ZsetRem)(RedisModuleKey *key, RedisModuleString *ele, int *deleted); +void REDISMODULE_API_FUNC(RedisModule_ZsetRangeStop)(RedisModuleKey *key); +int REDISMODULE_API_FUNC(RedisModule_ZsetFirstInScoreRange)(RedisModuleKey *key, double min, double max, int minex, int maxex); +int REDISMODULE_API_FUNC(RedisModule_ZsetLastInScoreRange)(RedisModuleKey *key, double min, double max, int minex, int maxex); +int REDISMODULE_API_FUNC(RedisModule_ZsetFirstInLexRange)(RedisModuleKey *key, RedisModuleString *min, RedisModuleString *max); +int REDISMODULE_API_FUNC(RedisModule_ZsetLastInLexRange)(RedisModuleKey *key, RedisModuleString *min, RedisModuleString *max); +RedisModuleString *REDISMODULE_API_FUNC(RedisModule_ZsetRangeCurrentElement)(RedisModuleKey *key, double *score); +int REDISMODULE_API_FUNC(RedisModule_ZsetRangeNext)(RedisModuleKey *key); +int REDISMODULE_API_FUNC(RedisModule_ZsetRangePrev)(RedisModuleKey *key); +int REDISMODULE_API_FUNC(RedisModule_ZsetRangeEndReached)(RedisModuleKey *key); +int REDISMODULE_API_FUNC(RedisModule_HashSet)(RedisModuleKey *key, int flags, ...); +int REDISMODULE_API_FUNC(RedisModule_HashGet)(RedisModuleKey *key, int flags, ...); +int REDISMODULE_API_FUNC(RedisModule_IsKeysPositionRequest)(RedisModuleCtx *ctx); +void REDISMODULE_API_FUNC(RedisModule_KeyAtPos)(RedisModuleCtx *ctx, int pos); + +/* This is included inline inside each Redis module. */ +static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int apiver) { + void *getapifuncptr = ((void**)ctx)[0]; + RedisModule_GetApi = (int (*)(const char *, void *)) (unsigned long)getapifuncptr; + REDISMODULE_GET_API(CreateCommand); + REDISMODULE_GET_API(SetModuleAttribs); + REDISMODULE_GET_API(WrongArity); + REDISMODULE_GET_API(ReplyWithLongLong); + REDISMODULE_GET_API(ReplyWithError); + REDISMODULE_GET_API(ReplyWithSimpleString); + REDISMODULE_GET_API(ReplyWithArray); + REDISMODULE_GET_API(ReplySetArrayLength); + REDISMODULE_GET_API(ReplyWithStringBuffer); + REDISMODULE_GET_API(ReplyWithString); + REDISMODULE_GET_API(ReplyWithNull); + REDISMODULE_GET_API(ReplyWithCallReply); + REDISMODULE_GET_API(ReplyWithDouble); + REDISMODULE_GET_API(ReplySetArrayLength); + REDISMODULE_GET_API(GetSelectedDb); + REDISMODULE_GET_API(SelectDb); + REDISMODULE_GET_API(OpenKey); + REDISMODULE_GET_API(CloseKey); + REDISMODULE_GET_API(KeyType); + REDISMODULE_GET_API(ValueLength); + REDISMODULE_GET_API(ListPush); + REDISMODULE_GET_API(ListPop); + REDISMODULE_GET_API(StringToLongLong); + REDISMODULE_GET_API(StringToDouble); + REDISMODULE_GET_API(Call); + REDISMODULE_GET_API(CallReplyProto); + REDISMODULE_GET_API(FreeCallReply); + REDISMODULE_GET_API(CallReplyInteger); + REDISMODULE_GET_API(CallReplyType); + REDISMODULE_GET_API(CallReplyLength); + REDISMODULE_GET_API(CallReplyArrayElement); + REDISMODULE_GET_API(CallReplyStringPtr); + REDISMODULE_GET_API(CreateStringFromCallReply); + REDISMODULE_GET_API(CreateString); + REDISMODULE_GET_API(CreateStringFromLongLong); + REDISMODULE_GET_API(FreeString); + REDISMODULE_GET_API(StringPtrLen); + REDISMODULE_GET_API(AutoMemory); + REDISMODULE_GET_API(Replicate); + REDISMODULE_GET_API(ReplicateVerbatim); + REDISMODULE_GET_API(DeleteKey); + REDISMODULE_GET_API(StringSet); + REDISMODULE_GET_API(StringDMA); + REDISMODULE_GET_API(StringTruncate); + REDISMODULE_GET_API(GetExpire); + REDISMODULE_GET_API(SetExpire); + REDISMODULE_GET_API(ZsetAdd); + REDISMODULE_GET_API(ZsetIncrby); + REDISMODULE_GET_API(ZsetScore); + REDISMODULE_GET_API(ZsetRem); + REDISMODULE_GET_API(ZsetRangeStop); + REDISMODULE_GET_API(ZsetFirstInScoreRange); + REDISMODULE_GET_API(ZsetLastInScoreRange); + REDISMODULE_GET_API(ZsetFirstInLexRange); + REDISMODULE_GET_API(ZsetLastInLexRange); + REDISMODULE_GET_API(ZsetRangeCurrentElement); + REDISMODULE_GET_API(ZsetRangeNext); + REDISMODULE_GET_API(ZsetRangePrev); + REDISMODULE_GET_API(ZsetRangeEndReached); + REDISMODULE_GET_API(HashSet); + REDISMODULE_GET_API(HashGet); + REDISMODULE_GET_API(IsKeysPositionRequest); + REDISMODULE_GET_API(KeyAtPos); + + RedisModule_SetModuleAttribs(ctx,name,ver,apiver); + return REDISMODULE_OK; +} + +#else + +/* Things only defined for the modules core, not exported to modules + * including this file. */ +#define RedisModuleString robj + +#endif /* REDISMODULE_CORE */ +#endif /* REDISMOUDLE_H */ diff --git a/rmutil/Makefile b/rmutil/Makefile index 318becd..b33ced9 100644 --- a/rmutil/Makefile +++ b/rmutil/Makefile @@ -1,17 +1,19 @@ #set environment variable RM_INCLUDE_DIR to the location of redismodule.h +ifndef RM_INCLUDE_DIR + RM_INCLUDE_DIR=../ +endif CFLAGS = -g -fPIC -lc -lm -O3 -std=gnu99 -I$(RM_INCLUDE_DIR) CC=gcc -CFLAGS = -I$(RM_INCLUDE_DIR) .SUFFIXES: .c .so .xo .o OBJS=util.o strings.o sds.o -all: rmutil.a +all: librmutil.a clean: rm -rf *.xo *.so *.o -rmutil.a: $(OBJS) +librmutil.a: $(OBJS) ar rcs $@ $^ diff --git a/rmutil/sdsalloc.h b/rmutil/sdsalloc.h new file mode 100644 index 0000000..531d419 --- /dev/null +++ b/rmutil/sdsalloc.h @@ -0,0 +1,42 @@ +/* SDSLib 2.0 -- A C dynamic strings library + * + * Copyright (c) 2006-2015, Salvatore Sanfilippo + * Copyright (c) 2015, Redis Labs, Inc + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* SDS allocator selection. + * + * This file is used in order to change the SDS allocator at compile time. + * Just define the following defines to what you want to use. Also add + * the include of your alternate allocator if needed (not needed in order + * to use the default libc allocator). */ + +#include "zmalloc.h" +#define s_malloc zmalloc +#define s_realloc zrealloc +#define s_free zfree diff --git a/rmutil/strings.c b/rmutil/strings.c index 1d1a9e6..08502b9 100644 --- a/rmutil/strings.c +++ b/rmutil/strings.c @@ -50,4 +50,5 @@ void RMUtil_StringToUpper(RedisModuleString *s) { ++c; } + } diff --git a/rmutil/util.c b/rmutil/util.c index 85866ba..610cfd2 100644 --- a/rmutil/util.c +++ b/rmutil/util.c @@ -3,8 +3,10 @@ #include #include #include +#include #include #include +#include #include "util.h" /** @@ -35,6 +37,7 @@ RMUtilInfo *RMUtil_GetRedisInfo(RedisModuleCtx *ctx) { RMUtilInfo *info = malloc(sizeof(RMUtilInfo)); info->entries = calloc(cap, sizeof(RMUtilInfoEntry)); + int i = 0; char *text = (char *)RedisModule_StringPtrLen(RedisModule_CreateStringFromCallReply(r), NULL); char *line = text; @@ -115,3 +118,79 @@ int RMUtilInfo_GetDouble(RMUtilInfo *info, const char *key, double *d) { return 1; } + + +/* +c -- pointer to a Null terminated C string pointer. +s -- pointer to a RedisModuleString +l -- pointer to Long long integer. +d -- pointer to a Double +* -- do not parse this argument at all +*/ +int RMUtil_ParseArgs(RedisModuleString **argv, int argc, int offset, const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + int rc = rmutil_vparseArgs(argv, argc, offset, fmt, ap); + va_end(ap); + return rc; +} + + +// Internal function that parses arguments based on the format described above +int rmutil_vparseArgs(RedisModuleString **argv, int argc, int offset, const char *fmt, va_list ap) { + + int i = offset; + char *c = (char *)fmt; + while (*c && i < argc) { + + // read c string + if (*c == 'c') { + char **p = va_arg(ap, char**); + *p = (char *)RedisModule_StringPtrLen(argv[i], NULL); + } else if (*c == 's') { //read redis string + + RedisModuleString **s = va_arg(ap, void*); + *s = argv[i]; + + } else if (*c == 'l') { //read long + long long *l = va_arg(ap, long long *); + + if (RedisModule_StringToLongLong(argv[i], l) != REDISMODULE_OK) { + return REDISMODULE_ERR; + } + } else if (*c == 'd') { //read double + double *d = va_arg(ap, double *); + if (RedisModule_StringToDouble(argv[i], d) != REDISMODULE_OK) { + return REDISMODULE_ERR; + } + } else if (*c == '*') { //skip current arg + //do nothing + } else { + return REDISMODULE_ERR; //WAT? + } + c++; + i++; + } + // if the format is longer than argc, retun an error + if (*c != 0) { + return REDISMODULE_ERR; + } + return REDISMODULE_OK; + + +} + +int RMUtil_ParseArgsAfter(const char *token, RedisModuleString **argv, int argc, const char *fmt, ...) { + + int pos = RMUtil_ArgExists(token, argv, argc, 0); + if (pos == 0) { + return REDISMODULE_ERR; + } + + va_list ap; + va_start(ap, fmt); + int rc = rmutil_vparseArgs(argv, argc, pos+1, fmt, ap); + va_end(ap); + return rc; + +} \ No newline at end of file diff --git a/rmutil/util.h b/rmutil/util.h index 6c07709..7289edc 100644 --- a/rmutil/util.h +++ b/rmutil/util.h @@ -2,6 +2,7 @@ #define __UTIL_H__ #include +#include // make sure the response is not NULL or an error, and if it is sends the error to the client and exit the current function #define REDIS_ASSERT_NOERROR(r) \ @@ -18,8 +19,31 @@ /* Return the offset of an arg if it exists in the arg list, or 0 if it's not there */ int RMUtil_ArgExists(const char *arg, RedisModuleString **argv, int argc, int offset); -// NOT IMPLEMENTED YET -int RMUtil_ParseArgs(RedisModuleString **argv, int argc, const char *fmt, ...); +/* +Automatically conver the arg list to corresponding variable pointers according to a given format. +You pass it the command arg list and count, the starting offset, a parsing format, and pointers to the variables. +The format is a string consisting of the following identifiers: + + c -- pointer to a Null terminated C string pointer. + s -- pointer to a RedisModuleString + l -- pointer to Long long integer. + d -- pointer to a Double + * -- do not parse this argument at all + +Example: If I want to parse args[1], args[2] as a long long and double, I do: + double d; + long long l; + RMUtil_ParseArgs(argv, argc, 1, "ld", &l, &d); +*/ +int RMUtil_ParseArgs(RedisModuleString **argv, int argc, int offset, const char *fmt, ...); + +/** +Same as RMUtil_ParseArgs, but only parses the arguments after `token`, if it was found. +This is useful for optional stuff like [LIMIT [offset] [limit]] +*/ +int RMUtil_ParseArgsAfter(const char *token, RedisModuleString **argv, int argc, const char *fmt, ...); + +int rmutil_vparseArgs(RedisModuleString **argv, int argc, int offset, const char *fmt, va_list ap); // A single key/value entry in a redis info map typedef struct { diff --git a/rmutil/zmalloc.c b/rmutil/zmalloc.c new file mode 100644 index 0000000..da62447 --- /dev/null +++ b/rmutil/zmalloc.c @@ -0,0 +1,395 @@ +/* zmalloc - total amount of allocated memory aware version of malloc() + * + * Copyright (c) 2009-2010, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +/* This function provide us access to the original libc free(). This is useful + * for instance to free results obtained by backtrace_symbols(). We need + * to define this function before including zmalloc.h that may shadow the + * free implementation if we use jemalloc or another non standard allocator. */ +void zlibc_free(void *ptr) { + free(ptr); +} + +#include +#include +#include "config.h" +#include "zmalloc.h" +#include "atomicvar.h" + +#ifdef HAVE_MALLOC_SIZE +#define PREFIX_SIZE (0) +#else +#if defined(__sun) || defined(__sparc) || defined(__sparc__) +#define PREFIX_SIZE (sizeof(long long)) +#else +#define PREFIX_SIZE (sizeof(size_t)) +#endif +#endif + +/* Explicitly override malloc/free etc when using tcmalloc. */ +#if defined(USE_TCMALLOC) +#define malloc(size) tc_malloc(size) +#define calloc(count,size) tc_calloc(count,size) +#define realloc(ptr,size) tc_realloc(ptr,size) +#define free(ptr) tc_free(ptr) +#elif defined(USE_JEMALLOC) +#define malloc(size) je_malloc(size) +#define calloc(count,size) je_calloc(count,size) +#define realloc(ptr,size) je_realloc(ptr,size) +#define free(ptr) je_free(ptr) +#endif + +#define update_zmalloc_stat_alloc(__n) do { \ + size_t _n = (__n); \ + if (_n&(sizeof(long)-1)) _n += sizeof(long)-(_n&(sizeof(long)-1)); \ + if (zmalloc_thread_safe) { \ + atomicIncr(used_memory,__n,&used_memory_mutex); \ + } else { \ + used_memory += _n; \ + } \ +} while(0) + +#define update_zmalloc_stat_free(__n) do { \ + size_t _n = (__n); \ + if (_n&(sizeof(long)-1)) _n += sizeof(long)-(_n&(sizeof(long)-1)); \ + if (zmalloc_thread_safe) { \ + atomicDecr(used_memory,__n,&used_memory_mutex); \ + } else { \ + used_memory -= _n; \ + } \ +} while(0) + +static size_t used_memory = 0; +static int zmalloc_thread_safe = 0; +pthread_mutex_t used_memory_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void zmalloc_default_oom(size_t size) { + fprintf(stderr, "zmalloc: Out of memory trying to allocate %zu bytes\n", + size); + fflush(stderr); + abort(); +} + +static void (*zmalloc_oom_handler)(size_t) = zmalloc_default_oom; + +void *zmalloc(size_t size) { + void *ptr = malloc(size+PREFIX_SIZE); + + if (!ptr) zmalloc_oom_handler(size); +#ifdef HAVE_MALLOC_SIZE + update_zmalloc_stat_alloc(zmalloc_size(ptr)); + return ptr; +#else + *((size_t*)ptr) = size; + update_zmalloc_stat_alloc(size+PREFIX_SIZE); + return (char*)ptr+PREFIX_SIZE; +#endif +} + +void *zcalloc(size_t size) { + void *ptr = calloc(1, size+PREFIX_SIZE); + + if (!ptr) zmalloc_oom_handler(size); +#ifdef HAVE_MALLOC_SIZE + update_zmalloc_stat_alloc(zmalloc_size(ptr)); + return ptr; +#else + *((size_t*)ptr) = size; + update_zmalloc_stat_alloc(size+PREFIX_SIZE); + return (char*)ptr+PREFIX_SIZE; +#endif +} + +void *zrealloc(void *ptr, size_t size) { +#ifndef HAVE_MALLOC_SIZE + void *realptr; +#endif + size_t oldsize; + void *newptr; + + if (ptr == NULL) return zmalloc(size); +#ifdef HAVE_MALLOC_SIZE + oldsize = zmalloc_size(ptr); + newptr = realloc(ptr,size); + if (!newptr) zmalloc_oom_handler(size); + + update_zmalloc_stat_free(oldsize); + update_zmalloc_stat_alloc(zmalloc_size(newptr)); + return newptr; +#else + realptr = (char*)ptr-PREFIX_SIZE; + oldsize = *((size_t*)realptr); + newptr = realloc(realptr,size+PREFIX_SIZE); + if (!newptr) zmalloc_oom_handler(size); + + *((size_t*)newptr) = size; + update_zmalloc_stat_free(oldsize); + update_zmalloc_stat_alloc(size); + return (char*)newptr+PREFIX_SIZE; +#endif +} + +/* Provide zmalloc_size() for systems where this function is not provided by + * malloc itself, given that in that case we store a header with this + * information as the first bytes of every allocation. */ +#ifndef HAVE_MALLOC_SIZE +size_t zmalloc_size(void *ptr) { + void *realptr = (char*)ptr-PREFIX_SIZE; + size_t size = *((size_t*)realptr); + /* Assume at least that all the allocations are padded at sizeof(long) by + * the underlying allocator. */ + if (size&(sizeof(long)-1)) size += sizeof(long)-(size&(sizeof(long)-1)); + return size+PREFIX_SIZE; +} +#endif + +void zfree(void *ptr) { +#ifndef HAVE_MALLOC_SIZE + void *realptr; + size_t oldsize; +#endif + + if (ptr == NULL) return; +#ifdef HAVE_MALLOC_SIZE + update_zmalloc_stat_free(zmalloc_size(ptr)); + free(ptr); +#else + realptr = (char*)ptr-PREFIX_SIZE; + oldsize = *((size_t*)realptr); + update_zmalloc_stat_free(oldsize+PREFIX_SIZE); + free(realptr); +#endif +} + +char *zstrdup(const char *s) { + size_t l = strlen(s)+1; + char *p = zmalloc(l); + + memcpy(p,s,l); + return p; +} + +size_t zmalloc_used_memory(void) { + size_t um; + + if (zmalloc_thread_safe) { + atomicGet(used_memory,um,&used_memory_mutex); + } else { + um = used_memory; + } + return um; +} + +void zmalloc_enable_thread_safeness(void) { + zmalloc_thread_safe = 1; +} + +void zmalloc_set_oom_handler(void (*oom_handler)(size_t)) { + zmalloc_oom_handler = oom_handler; +} + +/* Get the RSS information in an OS-specific way. + * + * WARNING: the function zmalloc_get_rss() is not designed to be fast + * and may not be called in the busy loops where Redis tries to release + * memory expiring or swapping out objects. + * + * For this kind of "fast RSS reporting" usages use instead the + * function RedisEstimateRSS() that is a much faster (and less precise) + * version of the function. */ + +#if defined(HAVE_PROC_STAT) +#include +#include +#include +#include + +size_t zmalloc_get_rss(void) { + int page = sysconf(_SC_PAGESIZE); + size_t rss; + char buf[4096]; + char filename[256]; + int fd, count; + char *p, *x; + + snprintf(filename,256,"/proc/%d/stat",getpid()); + if ((fd = open(filename,O_RDONLY)) == -1) return 0; + if (read(fd,buf,4096) <= 0) { + close(fd); + return 0; + } + close(fd); + + p = buf; + count = 23; /* RSS is the 24th field in /proc//stat */ + while(p && count--) { + p = strchr(p,' '); + if (p) p++; + } + if (!p) return 0; + x = strchr(p,' '); + if (!x) return 0; + *x = '\0'; + + rss = strtoll(p,NULL,10); + rss *= page; + return rss; +} +#elif defined(HAVE_TASKINFO) +#include +#include +#include +#include +#include +#include +#include + +size_t zmalloc_get_rss(void) { + task_t task = MACH_PORT_NULL; + struct task_basic_info t_info; + mach_msg_type_number_t t_info_count = TASK_BASIC_INFO_COUNT; + + if (task_for_pid(current_task(), getpid(), &task) != KERN_SUCCESS) + return 0; + task_info(task, TASK_BASIC_INFO, (task_info_t)&t_info, &t_info_count); + + return t_info.resident_size; +} +#else +size_t zmalloc_get_rss(void) { + /* If we can't get the RSS in an OS-specific way for this system just + * return the memory usage we estimated in zmalloc().. + * + * Fragmentation will appear to be always 1 (no fragmentation) + * of course... */ + return zmalloc_used_memory(); +} +#endif + +/* Fragmentation = RSS / allocated-bytes */ +float zmalloc_get_fragmentation_ratio(size_t rss) { + return (float)rss/zmalloc_used_memory(); +} + +/* Get the sum of the specified field (converted form kb to bytes) in + * /proc/self/smaps. The field must be specified with trailing ":" as it + * apperas in the smaps output. + * + * Example: zmalloc_get_smap_bytes_by_field("Rss:"); + */ +#if defined(HAVE_PROC_SMAPS) +size_t zmalloc_get_smap_bytes_by_field(char *field) { + char line[1024]; + size_t bytes = 0; + FILE *fp = fopen("/proc/self/smaps","r"); + int flen = strlen(field); + + if (!fp) return 0; + while(fgets(line,sizeof(line),fp) != NULL) { + if (strncmp(line,field,flen) == 0) { + char *p = strchr(line,'k'); + if (p) { + *p = '\0'; + bytes += strtol(line+flen,NULL,10) * 1024; + } + } + } + fclose(fp); + return bytes; +} +#else +size_t zmalloc_get_smap_bytes_by_field(char *field) { + ((void) field); + return 0; +} +#endif + +size_t zmalloc_get_private_dirty(void) { + return zmalloc_get_smap_bytes_by_field("Private_Dirty:"); +} + +/* Returns the size of physical memory (RAM) in bytes. + * It looks ugly, but this is the cleanest way to achive cross platform results. + * Cleaned up from: + * + * http://nadeausoftware.com/articles/2012/09/c_c_tip_how_get_physical_memory_size_system + * + * Note that this function: + * 1) Was released under the following CC attribution license: + * http://creativecommons.org/licenses/by/3.0/deed.en_US. + * 2) Was originally implemented by David Robert Nadeau. + * 3) Was modified for Redis by Matt Stancliff. + * 4) This note exists in order to comply with the original license. + */ +size_t zmalloc_get_memory_size(void) { +#if defined(__unix__) || defined(__unix) || defined(unix) || \ + (defined(__APPLE__) && defined(__MACH__)) +#if defined(CTL_HW) && (defined(HW_MEMSIZE) || defined(HW_PHYSMEM64)) + int mib[2]; + mib[0] = CTL_HW; +#if defined(HW_MEMSIZE) + mib[1] = HW_MEMSIZE; /* OSX. --------------------- */ +#elif defined(HW_PHYSMEM64) + mib[1] = HW_PHYSMEM64; /* NetBSD, OpenBSD. --------- */ +#endif + int64_t size = 0; /* 64-bit */ + size_t len = sizeof(size); + if (sysctl( mib, 2, &size, &len, NULL, 0) == 0) + return (size_t)size; + return 0L; /* Failed? */ + +#elif defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE) + /* FreeBSD, Linux, OpenBSD, and Solaris. -------------------- */ + return (size_t)sysconf(_SC_PHYS_PAGES) * (size_t)sysconf(_SC_PAGESIZE); + +#elif defined(CTL_HW) && (defined(HW_PHYSMEM) || defined(HW_REALMEM)) + /* DragonFly BSD, FreeBSD, NetBSD, OpenBSD, and OSX. -------- */ + int mib[2]; + mib[0] = CTL_HW; +#if defined(HW_REALMEM) + mib[1] = HW_REALMEM; /* FreeBSD. ----------------- */ +#elif defined(HW_PYSMEM) + mib[1] = HW_PHYSMEM; /* Others. ------------------ */ +#endif + unsigned int size = 0; /* 32-bit */ + size_t len = sizeof(size); + if (sysctl(mib, 2, &size, &len, NULL, 0) == 0) + return (size_t)size; + return 0L; /* Failed? */ +#endif /* sysctl and sysconf variants */ + +#else + return 0L; /* Unknown OS. */ +#endif +} + + diff --git a/rmutil/zmalloc.h b/rmutil/zmalloc.h new file mode 100644 index 0000000..a47ea6c --- /dev/null +++ b/rmutil/zmalloc.h @@ -0,0 +1,87 @@ +/* zmalloc - total amount of allocated memory aware version of malloc() + * + * Copyright (c) 2009-2010, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __ZMALLOC_H +#define __ZMALLOC_H + +/* Double expansion needed for stringification of macro values. */ +#define __xstr(s) __str(s) +#define __str(s) #s + +#if defined(USE_TCMALLOC) +#define ZMALLOC_LIB ("tcmalloc-" __xstr(TC_VERSION_MAJOR) "." __xstr(TC_VERSION_MINOR)) +#include +#if (TC_VERSION_MAJOR == 1 && TC_VERSION_MINOR >= 6) || (TC_VERSION_MAJOR > 1) +#define HAVE_MALLOC_SIZE 1 +#define zmalloc_size(p) tc_malloc_size(p) +#else +#error "Newer version of tcmalloc required" +#endif + +#elif defined(USE_JEMALLOC) +#define ZMALLOC_LIB ("jemalloc-" __xstr(JEMALLOC_VERSION_MAJOR) "." __xstr(JEMALLOC_VERSION_MINOR) "." __xstr(JEMALLOC_VERSION_BUGFIX)) +#include +#if (JEMALLOC_VERSION_MAJOR == 2 && JEMALLOC_VERSION_MINOR >= 1) || (JEMALLOC_VERSION_MAJOR > 2) +#define HAVE_MALLOC_SIZE 1 +#define zmalloc_size(p) je_malloc_usable_size(p) +#else +#error "Newer version of jemalloc required" +#endif + +#elif defined(__APPLE__) +#include +#define HAVE_MALLOC_SIZE 1 +#define zmalloc_size(p) malloc_size(p) +#endif + +#ifndef ZMALLOC_LIB +#define ZMALLOC_LIB "libc" +#endif + +void *zmalloc(size_t size); +void *zcalloc(size_t size); +void *zrealloc(void *ptr, size_t size); +void zfree(void *ptr); +char *zstrdup(const char *s); +size_t zmalloc_used_memory(void); +void zmalloc_enable_thread_safeness(void); +void zmalloc_set_oom_handler(void (*oom_handler)(size_t)); +float zmalloc_get_fragmentation_ratio(size_t rss); +size_t zmalloc_get_rss(void); +size_t zmalloc_get_private_dirty(void); +size_t zmalloc_get_smap_bytes_by_field(char *field); +size_t zmalloc_get_memory_size(void); +void zlibc_free(void *ptr); + +#ifndef HAVE_MALLOC_SIZE +size_t zmalloc_size(void *ptr); +#endif + +#endif /* __ZMALLOC_H */