From c3e8542aa9183426e4cb4959cac7458ebdd5242e Mon Sep 17 00:00:00 2001 From: Dvir Volk Date: Wed, 4 May 2016 17:03:31 +0300 Subject: [PATCH] removed zmalloc as a dependency, and updated the vector to use size_t --- .gitignore | 2 +- rmutil/sdsalloc.h | 8 +- rmutil/test_vector.c | 2 +- rmutil/vector.c | 6 +- rmutil/vector.h | 14 +- rmutil/zmalloc.c | 395 ------------------------------------------- rmutil/zmalloc.h | 87 ---------- 7 files changed, 16 insertions(+), 498 deletions(-) delete mode 100644 rmutil/zmalloc.c delete mode 100644 rmutil/zmalloc.h diff --git a/.gitignore b/.gitignore index 0961f3e..41c423b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,4 @@ *.so *.db .vscode - +rmutil/test_vector diff --git a/rmutil/sdsalloc.h b/rmutil/sdsalloc.h index 531d419..6ba2e4c 100644 --- a/rmutil/sdsalloc.h +++ b/rmutil/sdsalloc.h @@ -36,7 +36,7 @@ * the include of your alternate allocator if needed (not needed in order * to use the default libc allocator). */ -#include "zmalloc.h" -#define s_malloc zmalloc -#define s_realloc zrealloc -#define s_free zfree +//#include "zmalloc.h" +#define s_malloc malloc +#define s_realloc realloc +#define s_free free diff --git a/rmutil/test_vector.c b/rmutil/test_vector.c index 3c32ad0..afb2a39 100644 --- a/rmutil/test_vector.c +++ b/rmutil/test_vector.c @@ -33,7 +33,7 @@ int main(int argc, char **argv) { assert(Vector_Size(v) == N); assert(Vector_Cap(v) >= N); - for (int i = 0; i < Vector_Size(v); i++) { + for (size_t i = 0; i < Vector_Size(v); i++) { char *x; int rc = Vector_Get(v, i, &x); assert (rc == 1); diff --git a/rmutil/vector.c b/rmutil/vector.c index a5a325e..73a7fdd 100644 --- a/rmutil/vector.c +++ b/rmutil/vector.c @@ -13,7 +13,7 @@ int __vector_PushPtr(Vector *v, void *elem) { } -int Vector_Get(Vector *v, int pos, void *ptr) { +int Vector_Get(Vector *v, size_t pos, void *ptr) { // return 0 if pos is out of bounds if (pos >= v->top) { return 0; @@ -24,7 +24,7 @@ int Vector_Get(Vector *v, int pos, void *ptr) { } -int __vector_PutPtr(Vector *v, int pos, void *elem) { +int __vector_PutPtr(Vector *v, size_t pos, void *elem) { // resize if pos is out of bounds if (pos >= v->cap) { Vector_Resize(v, pos+1); @@ -39,7 +39,7 @@ int __vector_PutPtr(Vector *v, int pos, void *elem) { } -int Vector_Resize(Vector *v, int newcap) { +int Vector_Resize(Vector *v, size_t newcap) { int oldcap = v->cap; v->cap = newcap; diff --git a/rmutil/vector.h b/rmutil/vector.h index a599c45..6546742 100644 --- a/rmutil/vector.h +++ b/rmutil/vector.h @@ -11,9 +11,9 @@ */ typedef struct { char *data; - int elemSize; - int cap; - int top; + size_t elemSize; + size_t cap; + size_t top; } Vector; @@ -22,7 +22,7 @@ typedef struct { Vector *__newVectorSize(size_t elemSize, size_t cap); // Put a pointer in the vector. To be used internall by the library -int __vector_PutPtr(Vector *v, int pos, void *elem); +int __vector_PutPtr(Vector *v, size_t pos, void *elem); /* * Create a new vector for a given type and a given capacity. @@ -35,7 +35,7 @@ int __vector_PutPtr(Vector *v, int pos, void *elem); * the vector capacity, we return 0 * otherwise 1 */ -int Vector_Get(Vector *v, int pos, void *ptr); +int Vector_Get(Vector *v, size_t pos, void *ptr); //#define Vector_Getx(v, pos, ptr) pos < v->cap ? 1 : 0; *ptr = //*(typeof(ptr))(v->data + v->elemSize*pos) @@ -53,7 +53,7 @@ int Vector_Get(Vector *v, int pos, void *ptr); int __vector_PushPtr(Vector *v, void *elem); /* resize capacity of v */ -int Vector_Resize(Vector *v, int newcap); +int Vector_Resize(Vector *v, size_t newcap); /* return the used size of the vector, regardless of capacity */ inline int Vector_Size(Vector *v) { return v->top; } @@ -65,6 +65,6 @@ inline int Vector_Cap(Vector *v) { return v->cap; } * they are pointers*/ void Vector_Free(Vector *v); -int __vecotr_PutPtr(Vector *v, int pos, void *elem); +int __vecotr_PutPtr(Vector *v, size_t pos, void *elem); #endif \ No newline at end of file diff --git a/rmutil/zmalloc.c b/rmutil/zmalloc.c deleted file mode 100644 index da62447..0000000 --- a/rmutil/zmalloc.c +++ /dev/null @@ -1,395 +0,0 @@ -/* zmalloc - total amount of allocated memory aware version of malloc() - * - * Copyright (c) 2009-2010, Salvatore Sanfilippo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Redis nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include - -/* This function provide us access to the original libc free(). This is useful - * for instance to free results obtained by backtrace_symbols(). We need - * to define this function before including zmalloc.h that may shadow the - * free implementation if we use jemalloc or another non standard allocator. */ -void zlibc_free(void *ptr) { - free(ptr); -} - -#include -#include -#include "config.h" -#include "zmalloc.h" -#include "atomicvar.h" - -#ifdef HAVE_MALLOC_SIZE -#define PREFIX_SIZE (0) -#else -#if defined(__sun) || defined(__sparc) || defined(__sparc__) -#define PREFIX_SIZE (sizeof(long long)) -#else -#define PREFIX_SIZE (sizeof(size_t)) -#endif -#endif - -/* Explicitly override malloc/free etc when using tcmalloc. */ -#if defined(USE_TCMALLOC) -#define malloc(size) tc_malloc(size) -#define calloc(count,size) tc_calloc(count,size) -#define realloc(ptr,size) tc_realloc(ptr,size) -#define free(ptr) tc_free(ptr) -#elif defined(USE_JEMALLOC) -#define malloc(size) je_malloc(size) -#define calloc(count,size) je_calloc(count,size) -#define realloc(ptr,size) je_realloc(ptr,size) -#define free(ptr) je_free(ptr) -#endif - -#define update_zmalloc_stat_alloc(__n) do { \ - size_t _n = (__n); \ - if (_n&(sizeof(long)-1)) _n += sizeof(long)-(_n&(sizeof(long)-1)); \ - if (zmalloc_thread_safe) { \ - atomicIncr(used_memory,__n,&used_memory_mutex); \ - } else { \ - used_memory += _n; \ - } \ -} while(0) - -#define update_zmalloc_stat_free(__n) do { \ - size_t _n = (__n); \ - if (_n&(sizeof(long)-1)) _n += sizeof(long)-(_n&(sizeof(long)-1)); \ - if (zmalloc_thread_safe) { \ - atomicDecr(used_memory,__n,&used_memory_mutex); \ - } else { \ - used_memory -= _n; \ - } \ -} while(0) - -static size_t used_memory = 0; -static int zmalloc_thread_safe = 0; -pthread_mutex_t used_memory_mutex = PTHREAD_MUTEX_INITIALIZER; - -static void zmalloc_default_oom(size_t size) { - fprintf(stderr, "zmalloc: Out of memory trying to allocate %zu bytes\n", - size); - fflush(stderr); - abort(); -} - -static void (*zmalloc_oom_handler)(size_t) = zmalloc_default_oom; - -void *zmalloc(size_t size) { - void *ptr = malloc(size+PREFIX_SIZE); - - if (!ptr) zmalloc_oom_handler(size); -#ifdef HAVE_MALLOC_SIZE - update_zmalloc_stat_alloc(zmalloc_size(ptr)); - return ptr; -#else - *((size_t*)ptr) = size; - update_zmalloc_stat_alloc(size+PREFIX_SIZE); - return (char*)ptr+PREFIX_SIZE; -#endif -} - -void *zcalloc(size_t size) { - void *ptr = calloc(1, size+PREFIX_SIZE); - - if (!ptr) zmalloc_oom_handler(size); -#ifdef HAVE_MALLOC_SIZE - update_zmalloc_stat_alloc(zmalloc_size(ptr)); - return ptr; -#else - *((size_t*)ptr) = size; - update_zmalloc_stat_alloc(size+PREFIX_SIZE); - return (char*)ptr+PREFIX_SIZE; -#endif -} - -void *zrealloc(void *ptr, size_t size) { -#ifndef HAVE_MALLOC_SIZE - void *realptr; -#endif - size_t oldsize; - void *newptr; - - if (ptr == NULL) return zmalloc(size); -#ifdef HAVE_MALLOC_SIZE - oldsize = zmalloc_size(ptr); - newptr = realloc(ptr,size); - if (!newptr) zmalloc_oom_handler(size); - - update_zmalloc_stat_free(oldsize); - update_zmalloc_stat_alloc(zmalloc_size(newptr)); - return newptr; -#else - realptr = (char*)ptr-PREFIX_SIZE; - oldsize = *((size_t*)realptr); - newptr = realloc(realptr,size+PREFIX_SIZE); - if (!newptr) zmalloc_oom_handler(size); - - *((size_t*)newptr) = size; - update_zmalloc_stat_free(oldsize); - update_zmalloc_stat_alloc(size); - return (char*)newptr+PREFIX_SIZE; -#endif -} - -/* Provide zmalloc_size() for systems where this function is not provided by - * malloc itself, given that in that case we store a header with this - * information as the first bytes of every allocation. */ -#ifndef HAVE_MALLOC_SIZE -size_t zmalloc_size(void *ptr) { - void *realptr = (char*)ptr-PREFIX_SIZE; - size_t size = *((size_t*)realptr); - /* Assume at least that all the allocations are padded at sizeof(long) by - * the underlying allocator. */ - if (size&(sizeof(long)-1)) size += sizeof(long)-(size&(sizeof(long)-1)); - return size+PREFIX_SIZE; -} -#endif - -void zfree(void *ptr) { -#ifndef HAVE_MALLOC_SIZE - void *realptr; - size_t oldsize; -#endif - - if (ptr == NULL) return; -#ifdef HAVE_MALLOC_SIZE - update_zmalloc_stat_free(zmalloc_size(ptr)); - free(ptr); -#else - realptr = (char*)ptr-PREFIX_SIZE; - oldsize = *((size_t*)realptr); - update_zmalloc_stat_free(oldsize+PREFIX_SIZE); - free(realptr); -#endif -} - -char *zstrdup(const char *s) { - size_t l = strlen(s)+1; - char *p = zmalloc(l); - - memcpy(p,s,l); - return p; -} - -size_t zmalloc_used_memory(void) { - size_t um; - - if (zmalloc_thread_safe) { - atomicGet(used_memory,um,&used_memory_mutex); - } else { - um = used_memory; - } - return um; -} - -void zmalloc_enable_thread_safeness(void) { - zmalloc_thread_safe = 1; -} - -void zmalloc_set_oom_handler(void (*oom_handler)(size_t)) { - zmalloc_oom_handler = oom_handler; -} - -/* Get the RSS information in an OS-specific way. - * - * WARNING: the function zmalloc_get_rss() is not designed to be fast - * and may not be called in the busy loops where Redis tries to release - * memory expiring or swapping out objects. - * - * For this kind of "fast RSS reporting" usages use instead the - * function RedisEstimateRSS() that is a much faster (and less precise) - * version of the function. */ - -#if defined(HAVE_PROC_STAT) -#include -#include -#include -#include - -size_t zmalloc_get_rss(void) { - int page = sysconf(_SC_PAGESIZE); - size_t rss; - char buf[4096]; - char filename[256]; - int fd, count; - char *p, *x; - - snprintf(filename,256,"/proc/%d/stat",getpid()); - if ((fd = open(filename,O_RDONLY)) == -1) return 0; - if (read(fd,buf,4096) <= 0) { - close(fd); - return 0; - } - close(fd); - - p = buf; - count = 23; /* RSS is the 24th field in /proc//stat */ - while(p && count--) { - p = strchr(p,' '); - if (p) p++; - } - if (!p) return 0; - x = strchr(p,' '); - if (!x) return 0; - *x = '\0'; - - rss = strtoll(p,NULL,10); - rss *= page; - return rss; -} -#elif defined(HAVE_TASKINFO) -#include -#include -#include -#include -#include -#include -#include - -size_t zmalloc_get_rss(void) { - task_t task = MACH_PORT_NULL; - struct task_basic_info t_info; - mach_msg_type_number_t t_info_count = TASK_BASIC_INFO_COUNT; - - if (task_for_pid(current_task(), getpid(), &task) != KERN_SUCCESS) - return 0; - task_info(task, TASK_BASIC_INFO, (task_info_t)&t_info, &t_info_count); - - return t_info.resident_size; -} -#else -size_t zmalloc_get_rss(void) { - /* If we can't get the RSS in an OS-specific way for this system just - * return the memory usage we estimated in zmalloc().. - * - * Fragmentation will appear to be always 1 (no fragmentation) - * of course... */ - return zmalloc_used_memory(); -} -#endif - -/* Fragmentation = RSS / allocated-bytes */ -float zmalloc_get_fragmentation_ratio(size_t rss) { - return (float)rss/zmalloc_used_memory(); -} - -/* Get the sum of the specified field (converted form kb to bytes) in - * /proc/self/smaps. The field must be specified with trailing ":" as it - * apperas in the smaps output. - * - * Example: zmalloc_get_smap_bytes_by_field("Rss:"); - */ -#if defined(HAVE_PROC_SMAPS) -size_t zmalloc_get_smap_bytes_by_field(char *field) { - char line[1024]; - size_t bytes = 0; - FILE *fp = fopen("/proc/self/smaps","r"); - int flen = strlen(field); - - if (!fp) return 0; - while(fgets(line,sizeof(line),fp) != NULL) { - if (strncmp(line,field,flen) == 0) { - char *p = strchr(line,'k'); - if (p) { - *p = '\0'; - bytes += strtol(line+flen,NULL,10) * 1024; - } - } - } - fclose(fp); - return bytes; -} -#else -size_t zmalloc_get_smap_bytes_by_field(char *field) { - ((void) field); - return 0; -} -#endif - -size_t zmalloc_get_private_dirty(void) { - return zmalloc_get_smap_bytes_by_field("Private_Dirty:"); -} - -/* Returns the size of physical memory (RAM) in bytes. - * It looks ugly, but this is the cleanest way to achive cross platform results. - * Cleaned up from: - * - * http://nadeausoftware.com/articles/2012/09/c_c_tip_how_get_physical_memory_size_system - * - * Note that this function: - * 1) Was released under the following CC attribution license: - * http://creativecommons.org/licenses/by/3.0/deed.en_US. - * 2) Was originally implemented by David Robert Nadeau. - * 3) Was modified for Redis by Matt Stancliff. - * 4) This note exists in order to comply with the original license. - */ -size_t zmalloc_get_memory_size(void) { -#if defined(__unix__) || defined(__unix) || defined(unix) || \ - (defined(__APPLE__) && defined(__MACH__)) -#if defined(CTL_HW) && (defined(HW_MEMSIZE) || defined(HW_PHYSMEM64)) - int mib[2]; - mib[0] = CTL_HW; -#if defined(HW_MEMSIZE) - mib[1] = HW_MEMSIZE; /* OSX. --------------------- */ -#elif defined(HW_PHYSMEM64) - mib[1] = HW_PHYSMEM64; /* NetBSD, OpenBSD. --------- */ -#endif - int64_t size = 0; /* 64-bit */ - size_t len = sizeof(size); - if (sysctl( mib, 2, &size, &len, NULL, 0) == 0) - return (size_t)size; - return 0L; /* Failed? */ - -#elif defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE) - /* FreeBSD, Linux, OpenBSD, and Solaris. -------------------- */ - return (size_t)sysconf(_SC_PHYS_PAGES) * (size_t)sysconf(_SC_PAGESIZE); - -#elif defined(CTL_HW) && (defined(HW_PHYSMEM) || defined(HW_REALMEM)) - /* DragonFly BSD, FreeBSD, NetBSD, OpenBSD, and OSX. -------- */ - int mib[2]; - mib[0] = CTL_HW; -#if defined(HW_REALMEM) - mib[1] = HW_REALMEM; /* FreeBSD. ----------------- */ -#elif defined(HW_PYSMEM) - mib[1] = HW_PHYSMEM; /* Others. ------------------ */ -#endif - unsigned int size = 0; /* 32-bit */ - size_t len = sizeof(size); - if (sysctl(mib, 2, &size, &len, NULL, 0) == 0) - return (size_t)size; - return 0L; /* Failed? */ -#endif /* sysctl and sysconf variants */ - -#else - return 0L; /* Unknown OS. */ -#endif -} - - diff --git a/rmutil/zmalloc.h b/rmutil/zmalloc.h deleted file mode 100644 index a47ea6c..0000000 --- a/rmutil/zmalloc.h +++ /dev/null @@ -1,87 +0,0 @@ -/* zmalloc - total amount of allocated memory aware version of malloc() - * - * Copyright (c) 2009-2010, Salvatore Sanfilippo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Redis nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef __ZMALLOC_H -#define __ZMALLOC_H - -/* Double expansion needed for stringification of macro values. */ -#define __xstr(s) __str(s) -#define __str(s) #s - -#if defined(USE_TCMALLOC) -#define ZMALLOC_LIB ("tcmalloc-" __xstr(TC_VERSION_MAJOR) "." __xstr(TC_VERSION_MINOR)) -#include -#if (TC_VERSION_MAJOR == 1 && TC_VERSION_MINOR >= 6) || (TC_VERSION_MAJOR > 1) -#define HAVE_MALLOC_SIZE 1 -#define zmalloc_size(p) tc_malloc_size(p) -#else -#error "Newer version of tcmalloc required" -#endif - -#elif defined(USE_JEMALLOC) -#define ZMALLOC_LIB ("jemalloc-" __xstr(JEMALLOC_VERSION_MAJOR) "." __xstr(JEMALLOC_VERSION_MINOR) "." __xstr(JEMALLOC_VERSION_BUGFIX)) -#include -#if (JEMALLOC_VERSION_MAJOR == 2 && JEMALLOC_VERSION_MINOR >= 1) || (JEMALLOC_VERSION_MAJOR > 2) -#define HAVE_MALLOC_SIZE 1 -#define zmalloc_size(p) je_malloc_usable_size(p) -#else -#error "Newer version of jemalloc required" -#endif - -#elif defined(__APPLE__) -#include -#define HAVE_MALLOC_SIZE 1 -#define zmalloc_size(p) malloc_size(p) -#endif - -#ifndef ZMALLOC_LIB -#define ZMALLOC_LIB "libc" -#endif - -void *zmalloc(size_t size); -void *zcalloc(size_t size); -void *zrealloc(void *ptr, size_t size); -void zfree(void *ptr); -char *zstrdup(const char *s); -size_t zmalloc_used_memory(void); -void zmalloc_enable_thread_safeness(void); -void zmalloc_set_oom_handler(void (*oom_handler)(size_t)); -float zmalloc_get_fragmentation_ratio(size_t rss); -size_t zmalloc_get_rss(void); -size_t zmalloc_get_private_dirty(void); -size_t zmalloc_get_smap_bytes_by_field(char *field); -size_t zmalloc_get_memory_size(void); -void zlibc_free(void *ptr); - -#ifndef HAVE_MALLOC_SIZE -size_t zmalloc_size(void *ptr); -#endif - -#endif /* __ZMALLOC_H */