use cmake

.gitignore

@@ -29,7 +29,6 @@ _deps
 _test*
 *.dll
 *.db
-cozorocks/deps/
 .DS_Store
 flamegraph.svg
 release.zip

.gitmodules

@@ -1,9 +1,12 @@
-[submodule "rocksdb"]
+[submodule "cozorocks/deps/gflags"]
-	path = cozorocks/rocksdb
+	path = cozorocks/deps/gflags
-	url = https://github.com/facebook/rocksdb
+	url = https://github.com/gflags/gflags.git
-[submodule "cozorocks/rocksdb"]
+[submodule "cozorocks/deps/lz4"]
-	path = cozorocks/rocksdb
+	path = cozorocks/deps/lz4
-	url = https://github.com/facebook/rocksdb
+	url = https://github.com/lz4/lz4.git
-[submodule "cozorocks/jemalloc"]
+[submodule "cozorocks/deps/zstd"]
-	path = cozorocks/jemalloc
+	path = cozorocks/deps/zstd
-	url = https://github.com/jemalloc/jemalloc
+	url = https://github.com/facebook/zstd.git
+[submodule "cozorocks/deps/rocksdb"]
+	path = cozorocks/deps/rocksdb
+	url = https://github.com/facebook/rocksdb.git

cozorocks/CMakeLists.txt

@@ -4,7 +4,7 @@ project(cozorocks)
 set(CMAKE_CXX_STANDARD 17)
 
 include_directories("bridge")
-include_directories("rocksdb/include")
+include_directories("deps/rocksdb/include")
 include_directories("../target/cxxbridge")
 
 add_library(cozorocks "bridge/bridge.h" "bridge/common.h" "bridge/db.cpp" "bridge/db.h" "bridge/iter.h" "bridge/opts.h"

cozorocks/bridge/db.cpp

@@ -9,8 +9,8 @@
 
 unique_ptr<Options> default_db_options() {
     auto options = make_unique<Options>();
-    options->compression = kLZ4Compression;
     options->bottommost_compression = kZSTD;
+    options->compression = kLZ4Compression;
     options->level_compaction_dynamic_level_bytes = true;
     options->max_background_compactions = 4;
     options->max_background_flushes = 2;

cozorocks/build.rs

@@ -4,8 +4,9 @@ fn main() {
     let target = var("TARGET").unwrap();
 
     let mut builder = cxx_build::bridge("src/bridge/mod.rs");
-    builder.files(["bridge/status.cpp", "bridge/db.cpp", "bridge/tx.cpp"])
+    builder
-        .include("deps/include")
+        .files(["bridge/status.cpp", "bridge/db.cpp", "bridge/tx.cpp"])
+        .include("deps/rocksdb/include")
         .include("bridge");
     if target.contains("msvc") {
         builder.flag_if_supported("-std:c++17");
@@ -16,21 +17,22 @@ fn main() {
 
     let manifest_dir = var("CARGO_MANIFEST_DIR").unwrap();
 
-    println!("cargo:rustc-link-search={}/deps/lib/", manifest_dir);
+    println!("cargo:rustc-link-search={}/deps/rocksdb/cmake_build/", manifest_dir);
-    println!("cargo:rustc-link-lib=rocksdb");
+    println!("cargo:rustc-link-search={}/deps/zstd/lib/", manifest_dir);
-    println!("cargo:rustc-link-lib=z");
+    println!("cargo:rustc-link-search={}/deps/lz4/lib/", manifest_dir);
-
+    println!("cargo:rustc-link-lib=static=zstd");
-    if target.contains("msvc") {
+    println!("cargo:rustc-link-lib=static=rocksdb");
-        println!("cargo:rustc-link-lib=rpcrt4");
+    println!("cargo:rustc-link-lib=static=lz4");
-        println!("cargo:rustc-link-lib=shlwapi");
+    // println!("cargo:rustc-link-lib=z");
-    }
+    //
-    else if target.contains("darwin") {
+    // if target.contains("msvc") {
-        println!("cargo:rustc-link-lib=bz2");
+    //     println!("cargo:rustc-link-lib=rpcrt4");
-    }
+    //     println!("cargo:rustc-link-lib=shlwapi");
-
+    // } else if target.contains("darwin") {
-    println!("cargo:rustc-link-lib=lz4");
+    //     println!("cargo:rustc-link-lib=bz2");
-    println!("cargo:rustc-link-lib=snappy");
+    // }
-    println!("cargo:rustc-link-lib=zstd");
+    //
+    // println!("cargo:rustc-link-lib=snappy");
     // println!("cargo:rustc-link-lib=jemalloc");
     println!("cargo:rerun-if-changed=src/bridge/mod.rs");
     println!("cargo:rerun-if-changed=bridge/bridge.h");

cozorocks/build_deps.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+set -e
+
+if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+  export CC=/usr/bin/clang-12
+  export CPP=/usr/bin/clang-cpp-12
+  export CXX=/usr/bin/clang++-12
+  export LD=/usr/bin/ld.lld-12
+fi
+
+# gflags
+
+cd gflags
+mkdir cmake_build && cd cmake_build
+cmake ..
+make -j 8
+cd ../..
+
+# lz4
+
+cd lz4
+make -j 8
+cd ..
+
+# zstd
+
+cd zstd
+make -j 8
+cd ..
+
+# rocksdb
+
+cd rocksdb
+rm -fr cmake_build
+mkdir cmake_build && cd cmake_build
+cp ../../thirdparty.inc ../thirdparty.inc
+cmake -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_MODULE_PATH="${PWD}/../../lz4;${PWD}/../../zstd" \
+  -DROCKSDB_BUILD_SHARED=0 \
+  -Dlz4_INCLUDE_DIRS=${PWD}/../../lz4/lib \
+  -Dlz4_LIBRARIES=${PWD}/../../lz4/lib \
+  -Dzstd_INCLUDE_DIRS=${PWD}/../../zstd/lib \
+  -Dzstd_LIBRARIES=${PWD}/../../zstd/lib \
+  -DCMAKE_CXX_STANDARD=20 -DWITH_GFLAGS=1 -DWITH_LZ4=1 -DWITH_ZSTD=1 -DUSE_RTTI=1 -DWITH_TESTS=0 \
+  -DWITH_BENCHMARK_TOOLS=0 -DWITH_CORE_TOOLS=0 -DWITH_TOOLS=0 ..

cozorocks/deps/gflags

@@ -0,0 +1 @@
+Subproject commit a738fdf9338412f83ab3f26f31ac11ed3f3ec4bd

cozorocks/deps/lz4

@@ -0,0 +1 @@
+Subproject commit 5ff839680134437dbf4678f3d0c7b371d84f4964

cozorocks/deps/rocksdb

@@ -0,0 +1 @@
+Subproject commit e072c129d57a0883370034cecf6774c745c40f76

cozorocks/deps/thirdparty.inc

@@ -0,0 +1,268 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+# Edit definitions below to specify paths to include files and libraries of all 3rd party libraries
+
+# TODO: Make this work with find_package and/or get rid of it
+#
+# This example assumes all the libraries locate in directories under THIRDPARTY_HOME environment variable
+# Set environment variable THIRDPARTY_HOME to point to your third party libraries home (Unix style dir separators)
+# or change the paths below to reflect where the libraries actually reside
+#
+set (THIRDPARTY_LIBS "")         # Initialization, don't touch
+
+#
+# Defaults
+#
+set(GFLAGS_HOME $ENV{THIRDPARTY_HOME}/gflags/cmake_build)
+set(GFLAGS_INCLUDE ${GFLAGS_HOME}/include)
+set(GFLAGS_LIB_DEBUG ${GFLAGS_HOME}/lib/libgflags.a)
+set(GFLAGS_LIB_RELEASE ${GFLAGS_HOME}/lib/libgflags.a)
+
+# ================================================== GFLAGS ==================================================
+# For compatibility
+if (GFLAGS)
+  set(WITH_GFLAGS ON)
+endif ()
+
+if (WITH_GFLAGS)
+  message(STATUS "GFLAGS library is enabled")
+  
+  if(DEFINED ENV{GFLAGS_INCLUDE})
+    set(GFLAGS_INCLUDE $ENV{GFLAGS_INCLUDE})
+  endif()
+  
+  if(DEFINED ENV{GFLAGS_LIB_DEBUG})
+    set(GFLAGS_LIB_DEBUG $ENV{GFLAGS_LIB_DEBUG})
+  endif()
+
+  if(DEFINED ENV{GFLAGS_LIB_RELEASE})
+    set(GFLAGS_LIB_RELEASE $ENV{GFLAGS_LIB_RELEASE})
+  endif()
+  
+  set(GFLAGS_CXX_FLAGS -DGFLAGS=gflags)
+  set(GFLAGS_LIBS debug ${GFLAGS_LIB_DEBUG} optimized ${GFLAGS_LIB_RELEASE})
+
+  add_definitions(${GFLAGS_CXX_FLAGS})
+  include_directories(${GFLAGS_INCLUDE})
+  set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} ${GFLAGS_LIBS})
+else ()
+  message(STATUS "GFLAGS library is disabled")
+endif ()
+
+# ================================================== SNAPPY ==================================================
+#
+# Edit these 4 lines to define paths to Snappy
+#
+set(SNAPPY_HOME $ENV{THIRDPARTY_HOME}/Snappy.Library)
+set(SNAPPY_INCLUDE ${SNAPPY_HOME}/build/native/inc/inc)
+set(SNAPPY_LIB_DEBUG ${SNAPPY_HOME}/lib/native/debug/amd64/snappy.lib)
+set(SNAPPY_LIB_RELEASE ${SNAPPY_HOME}/lib/native/retail/amd64/snappy.lib)
+
+# For compatibility
+if(SNAPPY)
+  set(WITH_SNAPPY ON)
+endif ()
+
+if (WITH_SNAPPY)
+  message(STATUS "SNAPPY library is enabled")
+
+  if(DEFINED ENV{SNAPPY_INCLUDE})
+    set(SNAPPY_INCLUDE $ENV{SNAPPY_INCLUDE})
+  endif()
+
+  if(DEFINED ENV{SNAPPY_LIB_DEBUG})
+    set(SNAPPY_LIB_DEBUG $ENV{SNAPPY_LIB_DEBUG})
+  endif()
+
+  if(DEFINED ENV{SNAPPY_LIB_RELEASE})
+    set(SNAPPY_LIB_RELEASE $ENV{SNAPPY_LIB_RELEASE})
+  endif()
+
+  set(SNAPPY_CXX_FLAGS -DSNAPPY)
+  set(SNAPPY_LIBS debug ${SNAPPY_LIB_DEBUG} optimized ${SNAPPY_LIB_RELEASE})
+
+  add_definitions(${SNAPPY_CXX_FLAGS})
+  include_directories(${SNAPPY_INCLUDE})
+  set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} ${SNAPPY_LIBS})
+else ()
+  message(STATUS "SNAPPY library is disabled")
+endif ()
+
+# ================================================== LZ4 ==================================================
+#
+# Edit these 4 lines to define paths to LZ4
+#
+set(LZ4_HOME $ENV{THIRDPARTY_HOME}/lz4/lib)
+set(LZ4_INCLUDE ${LZ4_HOME})
+set(LZ4_LIB_DEBUG ${LZ4_HOME}/liblz4.a)
+set(LZ4_LIB_RELEASE ${LZ4_HOME}/liblz4.a)
+
+
+# For compatibility
+if (LZ4)
+  set(WITH_LZ4 ON)
+endif ()
+
+if (WITH_LZ4)
+  message(STATUS "LZ4 library is enabled")
+  
+  if(DEFINED ENV{LZ4_INCLUDE})
+    set(LZ4_INCLUDE $ENV{LZ4_INCLUDE})
+  endif()
+  
+  if(DEFINED ENV{LZ4_LIB_DEBUG})
+    set(LZ4_LIB_DEBUG $ENV{LZ4_LIB_DEBUG})
+  endif()
+
+  if(DEFINED ENV{LZ4_LIB_RELEASE})
+    set(LZ4_LIB_RELEASE $ENV{LZ4_LIB_RELEASE})
+  endif()
+  
+  set(LZ4_CXX_FLAGS -DLZ4)
+  set(LZ4_LIBS debug ${LZ4_LIB_DEBUG} optimized ${LZ4_LIB_RELEASE})
+
+  add_definitions(${LZ4_CXX_FLAGS})
+  include_directories(${LZ4_INCLUDE})
+  set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} ${LZ4_LIBS})
+else ()
+  message(STATUS "LZ4 library is disabled")
+endif ()
+
+# ================================================== ZLIB ==================================================
+#
+# Edit these 4 lines to define paths to ZLIB
+#
+set(ZLIB_HOME $ENV{THIRDPARTY_HOME}/ZLIB.Library)
+set(ZLIB_INCLUDE ${ZLIB_HOME}/build/native/inc/inc)
+set(ZLIB_LIB_DEBUG ${ZLIB_HOME}/lib/native/debug/amd64/zlib.lib)
+set(ZLIB_LIB_RELEASE ${ZLIB_HOME}/lib/native/retail/amd64/zlib.lib)
+
+# For compatibilty
+if (ZLIB)
+  set(WITH_ZLIB ON)
+endif ()
+
+if (WITH_ZLIB)
+  message(STATUS "ZLIB library is enabled")
+
+  if(DEFINED ENV{ZLIB_INCLUDE})
+    set(ZLIB_INCLUDE $ENV{ZLIB_INCLUDE})
+  endif()
+  
+  if(DEFINED ENV{ZLIB_LIB_DEBUG})
+    set(ZLIB_LIB_DEBUG $ENV{ZLIB_LIB_DEBUG})
+  endif()
+
+  if(DEFINED ENV{ZLIB_LIB_RELEASE})
+    set(ZLIB_LIB_RELEASE $ENV{ZLIB_LIB_RELEASE})
+  endif()
+  
+  set(ZLIB_CXX_FLAGS -DZLIB)
+  set(ZLIB_LIBS debug ${ZLIB_LIB_DEBUG} optimized ${ZLIB_LIB_RELEASE})
+
+  add_definitions(${ZLIB_CXX_FLAGS})
+  include_directories(${ZLIB_INCLUDE})
+  set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} ${ZLIB_LIBS})
+else ()
+  message(STATUS "ZLIB library is disabled")
+endif ()
+
+# ================================================== XPRESS ==================================================
+# This makes use of built-in Windows API, no additional includes, links to a system lib
+
+# For compatibilty
+if (XPRESS)
+  set(WITH_XPRESS ON)
+endif ()
+
+if (WITH_XPRESS)
+  message(STATUS "XPRESS is enabled")
+
+  add_definitions(-DXPRESS)
+  
+  # We are using the implementation provided by the system
+  set (SYSTEM_LIBS ${SYSTEM_LIBS} Cabinet.lib)
+else ()
+  message(STATUS "XPRESS is disabled")
+endif ()
+
+
+# ================================================== ZSTD ==================================================
+#
+# Edit these 4 lines to define paths to ZSTD
+#
+set(ZSTD_HOME $ENV{THIRDPARTY_HOME}/zstd/lib)
+set(ZSTD_INCLUDE ${ZSTD_HOME})
+set(ZSTD_LIB_DEBUG ${ZSTD_HOME}/libzstd.a)
+set(ZSTD_LIB_RELEASE ${ZSTD_HOME}/libzstd.a)
+
+# For compatibility
+if (ZSTD)
+  set(WITH_ZSTD ON)
+endif ()
+
+if (WITH_ZSTD)
+  message(STATUS "ZSTD library is enabled")
+
+  if(DEFINED ENV{ZSTD_INCLUDE})
+    set(ZSTD_INCLUDE $ENV{ZSTD_INCLUDE})
+  endif()
+  
+  if(DEFINED ENV{ZSTD_LIB_DEBUG})
+    set(ZSTD_LIB_DEBUG $ENV{ZSTD_LIB_DEBUG})
+  endif()
+
+  if(DEFINED ENV{ZSTD_LIB_RELEASE})
+    set(ZSTD_LIB_RELEASE $ENV{ZSTD_LIB_RELEASE})
+  endif()
+
+  # ZSTD_STATIC_LINKING_ONLY only allows us to create an allocation functions override
+  # When jemalloc is in use
+  set(ZSTD_LIBS debug ${ZSTD_LIB_DEBUG} optimized ${ZSTD_LIB_RELEASE})
+
+  add_definitions(-DZSTD -DZSTD_STATIC_LINKING_ONLY)
+  include_directories(${ZSTD_INCLUDE})
+  set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} ${ZSTD_LIBS})
+else ()
+  message(STATUS "ZSTD library is disabled")
+endif ()
+
+#
+# Edit these 4 lines to define paths to Jemalloc
+#
+set(JEMALLOC_HOME $ENV{THIRDPARTY_HOME}/Jemalloc.Library)
+set(JEMALLOC_INCLUDE ${JEMALLOC_HOME}/build/native/inc)
+set(JEMALLOC_LIB_DEBUG ${JEMALLOC_HOME}/lib/native/debug/amd64/jemalloc.lib)
+set(JEMALLOC_LIB_RELEASE ${JEMALLOC_HOME}/lib/native/retail/amd64/jemalloc.lib)
+
+# ================================================== JEMALLOC ==================================================
+if(JEMALLOC)
+  set(WITH_JEMALLOC ON)
+endif()
+
+if (WITH_JEMALLOC)
+  message(STATUS "JEMALLOC library is enabled")
+  set(JEMALLOC_CXX_FLAGS "-DROCKSDB_JEMALLOC -DJEMALLOC_EXPORT= -DJEMALLOC_NO_RENAME")
+  
+  if(DEFINED ENV{JEMALLOC_INCLUDE})
+    set(JEMALLOC_INCLUDE $ENV{JEMALLOC_INCLUDE})
+  endif()
+  
+  if(DEFINED ENV{JEMALLOC_LIB_DEBUG})
+    set(JEMALLOC_LIB_DEBUG $ENV{JEMALLOC_LIB_DEBUG})
+  endif()
+
+  if(DEFINED ENV{JEMALLOC_LIB_RELEASE})
+    set(JEMALLOC_LIB_RELEASE $ENV{JEMALLOC_LIB_RELEASE})
+  endif()
+
+  set(JEMALLOC_LIBS debug ${JEMALLOC_LIB_DEBUG} optimized ${JEMALLOC_LIB_RELEASE})
+
+  add_definitions(${JEMALLOC_CXX_FLAGS})
+  include_directories(${JEMALLOC_INCLUDE})
+  set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} ${JEMALLOC_LIBS})
+  set (ARTIFACT_SUFFIX "_je")
+
+else ()
+  set (ARTIFACT_SUFFIX "")
+  message(STATUS "JEMALLOC library is disabled")
+endif ()

cozorocks/deps/zstd

@@ -0,0 +1 @@
+Subproject commit e47e674cd09583ff0503f0f6defd6d23d8b718d3

cozorocks/rocksdb

@@ -1 +0,0 @@
-Subproject commit 35cdd3e71e0627af2b0306e4322efd134906a74e