From 3843ce721817680df783b2d9462fe73b399cfc10 Mon Sep 17 00:00:00 2001 From: Ziyang Hu Date: Sun, 21 Aug 2022 13:53:01 +0800 Subject: [PATCH] use cmake --- .gitignore | 1 - .gitmodules | 21 +-- cozorocks/CMakeLists.txt | 2 +- cozorocks/bridge/db.cpp | 2 +- cozorocks/build.rs | 36 ++--- cozorocks/build_deps.sh | 46 ++++++ cozorocks/deps/gflags | 1 + cozorocks/deps/lz4 | 1 + cozorocks/deps/rocksdb | 1 + cozorocks/deps/thirdparty.inc | 268 ++++++++++++++++++++++++++++++++++ cozorocks/deps/zstd | 1 + cozorocks/rocksdb | 1 - 12 files changed, 351 insertions(+), 30 deletions(-) create mode 100644 cozorocks/build_deps.sh create mode 160000 cozorocks/deps/gflags create mode 160000 cozorocks/deps/lz4 create mode 160000 cozorocks/deps/rocksdb create mode 100644 cozorocks/deps/thirdparty.inc create mode 160000 cozorocks/deps/zstd delete mode 160000 cozorocks/rocksdb diff --git a/.gitignore b/.gitignore index 8d476f81..0b5d34f7 100644 --- a/.gitignore +++ b/.gitignore @@ -29,7 +29,6 @@ _deps _test* *.dll *.db -cozorocks/deps/ .DS_Store flamegraph.svg release.zip \ No newline at end of file diff --git a/.gitmodules b/.gitmodules index 8f818e9e..02906c51 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,12 @@ -[submodule "rocksdb"] - path = cozorocks/rocksdb - url = https://github.com/facebook/rocksdb -[submodule "cozorocks/rocksdb"] - path = cozorocks/rocksdb - url = https://github.com/facebook/rocksdb -[submodule "cozorocks/jemalloc"] - path = cozorocks/jemalloc - url = https://github.com/jemalloc/jemalloc +[submodule "cozorocks/deps/gflags"] + path = cozorocks/deps/gflags + url = https://github.com/gflags/gflags.git +[submodule "cozorocks/deps/lz4"] + path = cozorocks/deps/lz4 + url = https://github.com/lz4/lz4.git +[submodule "cozorocks/deps/zstd"] + path = cozorocks/deps/zstd + url = https://github.com/facebook/zstd.git +[submodule "cozorocks/deps/rocksdb"] + path = cozorocks/deps/rocksdb + url = https://github.com/facebook/rocksdb.git diff --git a/cozorocks/CMakeLists.txt b/cozorocks/CMakeLists.txt index 915ad650..011a6e23 100644 --- a/cozorocks/CMakeLists.txt +++ b/cozorocks/CMakeLists.txt @@ -4,7 +4,7 @@ project(cozorocks) set(CMAKE_CXX_STANDARD 17) include_directories("bridge") -include_directories("rocksdb/include") +include_directories("deps/rocksdb/include") include_directories("../target/cxxbridge") add_library(cozorocks "bridge/bridge.h" "bridge/common.h" "bridge/db.cpp" "bridge/db.h" "bridge/iter.h" "bridge/opts.h" diff --git a/cozorocks/bridge/db.cpp b/cozorocks/bridge/db.cpp index 0183be88..9945151e 100644 --- a/cozorocks/bridge/db.cpp +++ b/cozorocks/bridge/db.cpp @@ -9,8 +9,8 @@ unique_ptr default_db_options() { auto options = make_unique(); - options->compression = kLZ4Compression; options->bottommost_compression = kZSTD; + options->compression = kLZ4Compression; options->level_compaction_dynamic_level_bytes = true; options->max_background_compactions = 4; options->max_background_flushes = 2; diff --git a/cozorocks/build.rs b/cozorocks/build.rs index f8e5a3f1..4e121553 100644 --- a/cozorocks/build.rs +++ b/cozorocks/build.rs @@ -4,8 +4,9 @@ fn main() { let target = var("TARGET").unwrap(); let mut builder = cxx_build::bridge("src/bridge/mod.rs"); - builder.files(["bridge/status.cpp", "bridge/db.cpp", "bridge/tx.cpp"]) - .include("deps/include") + builder + .files(["bridge/status.cpp", "bridge/db.cpp", "bridge/tx.cpp"]) + .include("deps/rocksdb/include") .include("bridge"); if target.contains("msvc") { builder.flag_if_supported("-std:c++17"); @@ -16,21 +17,22 @@ fn main() { let manifest_dir = var("CARGO_MANIFEST_DIR").unwrap(); - println!("cargo:rustc-link-search={}/deps/lib/", manifest_dir); - println!("cargo:rustc-link-lib=rocksdb"); - println!("cargo:rustc-link-lib=z"); - - if target.contains("msvc") { - println!("cargo:rustc-link-lib=rpcrt4"); - println!("cargo:rustc-link-lib=shlwapi"); - } - else if target.contains("darwin") { - println!("cargo:rustc-link-lib=bz2"); - } - - println!("cargo:rustc-link-lib=lz4"); - println!("cargo:rustc-link-lib=snappy"); - println!("cargo:rustc-link-lib=zstd"); + println!("cargo:rustc-link-search={}/deps/rocksdb/cmake_build/", manifest_dir); + println!("cargo:rustc-link-search={}/deps/zstd/lib/", manifest_dir); + println!("cargo:rustc-link-search={}/deps/lz4/lib/", manifest_dir); + println!("cargo:rustc-link-lib=static=zstd"); + println!("cargo:rustc-link-lib=static=rocksdb"); + println!("cargo:rustc-link-lib=static=lz4"); + // println!("cargo:rustc-link-lib=z"); + // + // if target.contains("msvc") { + // println!("cargo:rustc-link-lib=rpcrt4"); + // println!("cargo:rustc-link-lib=shlwapi"); + // } else if target.contains("darwin") { + // println!("cargo:rustc-link-lib=bz2"); + // } + // + // println!("cargo:rustc-link-lib=snappy"); // println!("cargo:rustc-link-lib=jemalloc"); println!("cargo:rerun-if-changed=src/bridge/mod.rs"); println!("cargo:rerun-if-changed=bridge/bridge.h"); diff --git a/cozorocks/build_deps.sh b/cozorocks/build_deps.sh new file mode 100644 index 00000000..de9f6993 --- /dev/null +++ b/cozorocks/build_deps.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +set -e + +if [[ "$OSTYPE" == "linux-gnu"* ]]; then + export CC=/usr/bin/clang-12 + export CPP=/usr/bin/clang-cpp-12 + export CXX=/usr/bin/clang++-12 + export LD=/usr/bin/ld.lld-12 +fi + +# gflags + +cd gflags +mkdir cmake_build && cd cmake_build +cmake .. +make -j 8 +cd ../.. + +# lz4 + +cd lz4 +make -j 8 +cd .. + +# zstd + +cd zstd +make -j 8 +cd .. + +# rocksdb + +cd rocksdb +rm -fr cmake_build +mkdir cmake_build && cd cmake_build +cp ../../thirdparty.inc ../thirdparty.inc +cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_MODULE_PATH="${PWD}/../../lz4;${PWD}/../../zstd" \ + -DROCKSDB_BUILD_SHARED=0 \ + -Dlz4_INCLUDE_DIRS=${PWD}/../../lz4/lib \ + -Dlz4_LIBRARIES=${PWD}/../../lz4/lib \ + -Dzstd_INCLUDE_DIRS=${PWD}/../../zstd/lib \ + -Dzstd_LIBRARIES=${PWD}/../../zstd/lib \ + -DCMAKE_CXX_STANDARD=20 -DWITH_GFLAGS=1 -DWITH_LZ4=1 -DWITH_ZSTD=1 -DUSE_RTTI=1 -DWITH_TESTS=0 \ + -DWITH_BENCHMARK_TOOLS=0 -DWITH_CORE_TOOLS=0 -DWITH_TOOLS=0 .. \ No newline at end of file diff --git a/cozorocks/deps/gflags b/cozorocks/deps/gflags new file mode 160000 index 00000000..a738fdf9 --- /dev/null +++ b/cozorocks/deps/gflags @@ -0,0 +1 @@ +Subproject commit a738fdf9338412f83ab3f26f31ac11ed3f3ec4bd diff --git a/cozorocks/deps/lz4 b/cozorocks/deps/lz4 new file mode 160000 index 00000000..5ff83968 --- /dev/null +++ b/cozorocks/deps/lz4 @@ -0,0 +1 @@ +Subproject commit 5ff839680134437dbf4678f3d0c7b371d84f4964 diff --git a/cozorocks/deps/rocksdb b/cozorocks/deps/rocksdb new file mode 160000 index 00000000..e072c129 --- /dev/null +++ b/cozorocks/deps/rocksdb @@ -0,0 +1 @@ +Subproject commit e072c129d57a0883370034cecf6774c745c40f76 diff --git a/cozorocks/deps/thirdparty.inc b/cozorocks/deps/thirdparty.inc new file mode 100644 index 00000000..c11fccef --- /dev/null +++ b/cozorocks/deps/thirdparty.inc @@ -0,0 +1,268 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +# Edit definitions below to specify paths to include files and libraries of all 3rd party libraries + +# TODO: Make this work with find_package and/or get rid of it +# +# This example assumes all the libraries locate in directories under THIRDPARTY_HOME environment variable +# Set environment variable THIRDPARTY_HOME to point to your third party libraries home (Unix style dir separators) +# or change the paths below to reflect where the libraries actually reside +# +set (THIRDPARTY_LIBS "") # Initialization, don't touch + +# +# Defaults +# +set(GFLAGS_HOME $ENV{THIRDPARTY_HOME}/gflags/cmake_build) +set(GFLAGS_INCLUDE ${GFLAGS_HOME}/include) +set(GFLAGS_LIB_DEBUG ${GFLAGS_HOME}/lib/libgflags.a) +set(GFLAGS_LIB_RELEASE ${GFLAGS_HOME}/lib/libgflags.a) + +# ================================================== GFLAGS ================================================== +# For compatibility +if (GFLAGS) + set(WITH_GFLAGS ON) +endif () + +if (WITH_GFLAGS) + message(STATUS "GFLAGS library is enabled") + + if(DEFINED ENV{GFLAGS_INCLUDE}) + set(GFLAGS_INCLUDE $ENV{GFLAGS_INCLUDE}) + endif() + + if(DEFINED ENV{GFLAGS_LIB_DEBUG}) + set(GFLAGS_LIB_DEBUG $ENV{GFLAGS_LIB_DEBUG}) + endif() + + if(DEFINED ENV{GFLAGS_LIB_RELEASE}) + set(GFLAGS_LIB_RELEASE $ENV{GFLAGS_LIB_RELEASE}) + endif() + + set(GFLAGS_CXX_FLAGS -DGFLAGS=gflags) + set(GFLAGS_LIBS debug ${GFLAGS_LIB_DEBUG} optimized ${GFLAGS_LIB_RELEASE}) + + add_definitions(${GFLAGS_CXX_FLAGS}) + include_directories(${GFLAGS_INCLUDE}) + set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} ${GFLAGS_LIBS}) +else () + message(STATUS "GFLAGS library is disabled") +endif () + +# ================================================== SNAPPY ================================================== +# +# Edit these 4 lines to define paths to Snappy +# +set(SNAPPY_HOME $ENV{THIRDPARTY_HOME}/Snappy.Library) +set(SNAPPY_INCLUDE ${SNAPPY_HOME}/build/native/inc/inc) +set(SNAPPY_LIB_DEBUG ${SNAPPY_HOME}/lib/native/debug/amd64/snappy.lib) +set(SNAPPY_LIB_RELEASE ${SNAPPY_HOME}/lib/native/retail/amd64/snappy.lib) + +# For compatibility +if(SNAPPY) + set(WITH_SNAPPY ON) +endif () + +if (WITH_SNAPPY) + message(STATUS "SNAPPY library is enabled") + + if(DEFINED ENV{SNAPPY_INCLUDE}) + set(SNAPPY_INCLUDE $ENV{SNAPPY_INCLUDE}) + endif() + + if(DEFINED ENV{SNAPPY_LIB_DEBUG}) + set(SNAPPY_LIB_DEBUG $ENV{SNAPPY_LIB_DEBUG}) + endif() + + if(DEFINED ENV{SNAPPY_LIB_RELEASE}) + set(SNAPPY_LIB_RELEASE $ENV{SNAPPY_LIB_RELEASE}) + endif() + + set(SNAPPY_CXX_FLAGS -DSNAPPY) + set(SNAPPY_LIBS debug ${SNAPPY_LIB_DEBUG} optimized ${SNAPPY_LIB_RELEASE}) + + add_definitions(${SNAPPY_CXX_FLAGS}) + include_directories(${SNAPPY_INCLUDE}) + set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} ${SNAPPY_LIBS}) +else () + message(STATUS "SNAPPY library is disabled") +endif () + +# ================================================== LZ4 ================================================== +# +# Edit these 4 lines to define paths to LZ4 +# +set(LZ4_HOME $ENV{THIRDPARTY_HOME}/lz4/lib) +set(LZ4_INCLUDE ${LZ4_HOME}) +set(LZ4_LIB_DEBUG ${LZ4_HOME}/liblz4.a) +set(LZ4_LIB_RELEASE ${LZ4_HOME}/liblz4.a) + + +# For compatibility +if (LZ4) + set(WITH_LZ4 ON) +endif () + +if (WITH_LZ4) + message(STATUS "LZ4 library is enabled") + + if(DEFINED ENV{LZ4_INCLUDE}) + set(LZ4_INCLUDE $ENV{LZ4_INCLUDE}) + endif() + + if(DEFINED ENV{LZ4_LIB_DEBUG}) + set(LZ4_LIB_DEBUG $ENV{LZ4_LIB_DEBUG}) + endif() + + if(DEFINED ENV{LZ4_LIB_RELEASE}) + set(LZ4_LIB_RELEASE $ENV{LZ4_LIB_RELEASE}) + endif() + + set(LZ4_CXX_FLAGS -DLZ4) + set(LZ4_LIBS debug ${LZ4_LIB_DEBUG} optimized ${LZ4_LIB_RELEASE}) + + add_definitions(${LZ4_CXX_FLAGS}) + include_directories(${LZ4_INCLUDE}) + set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} ${LZ4_LIBS}) +else () + message(STATUS "LZ4 library is disabled") +endif () + +# ================================================== ZLIB ================================================== +# +# Edit these 4 lines to define paths to ZLIB +# +set(ZLIB_HOME $ENV{THIRDPARTY_HOME}/ZLIB.Library) +set(ZLIB_INCLUDE ${ZLIB_HOME}/build/native/inc/inc) +set(ZLIB_LIB_DEBUG ${ZLIB_HOME}/lib/native/debug/amd64/zlib.lib) +set(ZLIB_LIB_RELEASE ${ZLIB_HOME}/lib/native/retail/amd64/zlib.lib) + +# For compatibilty +if (ZLIB) + set(WITH_ZLIB ON) +endif () + +if (WITH_ZLIB) + message(STATUS "ZLIB library is enabled") + + if(DEFINED ENV{ZLIB_INCLUDE}) + set(ZLIB_INCLUDE $ENV{ZLIB_INCLUDE}) + endif() + + if(DEFINED ENV{ZLIB_LIB_DEBUG}) + set(ZLIB_LIB_DEBUG $ENV{ZLIB_LIB_DEBUG}) + endif() + + if(DEFINED ENV{ZLIB_LIB_RELEASE}) + set(ZLIB_LIB_RELEASE $ENV{ZLIB_LIB_RELEASE}) + endif() + + set(ZLIB_CXX_FLAGS -DZLIB) + set(ZLIB_LIBS debug ${ZLIB_LIB_DEBUG} optimized ${ZLIB_LIB_RELEASE}) + + add_definitions(${ZLIB_CXX_FLAGS}) + include_directories(${ZLIB_INCLUDE}) + set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} ${ZLIB_LIBS}) +else () + message(STATUS "ZLIB library is disabled") +endif () + +# ================================================== XPRESS ================================================== +# This makes use of built-in Windows API, no additional includes, links to a system lib + +# For compatibilty +if (XPRESS) + set(WITH_XPRESS ON) +endif () + +if (WITH_XPRESS) + message(STATUS "XPRESS is enabled") + + add_definitions(-DXPRESS) + + # We are using the implementation provided by the system + set (SYSTEM_LIBS ${SYSTEM_LIBS} Cabinet.lib) +else () + message(STATUS "XPRESS is disabled") +endif () + + +# ================================================== ZSTD ================================================== +# +# Edit these 4 lines to define paths to ZSTD +# +set(ZSTD_HOME $ENV{THIRDPARTY_HOME}/zstd/lib) +set(ZSTD_INCLUDE ${ZSTD_HOME}) +set(ZSTD_LIB_DEBUG ${ZSTD_HOME}/libzstd.a) +set(ZSTD_LIB_RELEASE ${ZSTD_HOME}/libzstd.a) + +# For compatibility +if (ZSTD) + set(WITH_ZSTD ON) +endif () + +if (WITH_ZSTD) + message(STATUS "ZSTD library is enabled") + + if(DEFINED ENV{ZSTD_INCLUDE}) + set(ZSTD_INCLUDE $ENV{ZSTD_INCLUDE}) + endif() + + if(DEFINED ENV{ZSTD_LIB_DEBUG}) + set(ZSTD_LIB_DEBUG $ENV{ZSTD_LIB_DEBUG}) + endif() + + if(DEFINED ENV{ZSTD_LIB_RELEASE}) + set(ZSTD_LIB_RELEASE $ENV{ZSTD_LIB_RELEASE}) + endif() + + # ZSTD_STATIC_LINKING_ONLY only allows us to create an allocation functions override + # When jemalloc is in use + set(ZSTD_LIBS debug ${ZSTD_LIB_DEBUG} optimized ${ZSTD_LIB_RELEASE}) + + add_definitions(-DZSTD -DZSTD_STATIC_LINKING_ONLY) + include_directories(${ZSTD_INCLUDE}) + set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} ${ZSTD_LIBS}) +else () + message(STATUS "ZSTD library is disabled") +endif () + +# +# Edit these 4 lines to define paths to Jemalloc +# +set(JEMALLOC_HOME $ENV{THIRDPARTY_HOME}/Jemalloc.Library) +set(JEMALLOC_INCLUDE ${JEMALLOC_HOME}/build/native/inc) +set(JEMALLOC_LIB_DEBUG ${JEMALLOC_HOME}/lib/native/debug/amd64/jemalloc.lib) +set(JEMALLOC_LIB_RELEASE ${JEMALLOC_HOME}/lib/native/retail/amd64/jemalloc.lib) + +# ================================================== JEMALLOC ================================================== +if(JEMALLOC) + set(WITH_JEMALLOC ON) +endif() + +if (WITH_JEMALLOC) + message(STATUS "JEMALLOC library is enabled") + set(JEMALLOC_CXX_FLAGS "-DROCKSDB_JEMALLOC -DJEMALLOC_EXPORT= -DJEMALLOC_NO_RENAME") + + if(DEFINED ENV{JEMALLOC_INCLUDE}) + set(JEMALLOC_INCLUDE $ENV{JEMALLOC_INCLUDE}) + endif() + + if(DEFINED ENV{JEMALLOC_LIB_DEBUG}) + set(JEMALLOC_LIB_DEBUG $ENV{JEMALLOC_LIB_DEBUG}) + endif() + + if(DEFINED ENV{JEMALLOC_LIB_RELEASE}) + set(JEMALLOC_LIB_RELEASE $ENV{JEMALLOC_LIB_RELEASE}) + endif() + + set(JEMALLOC_LIBS debug ${JEMALLOC_LIB_DEBUG} optimized ${JEMALLOC_LIB_RELEASE}) + + add_definitions(${JEMALLOC_CXX_FLAGS}) + include_directories(${JEMALLOC_INCLUDE}) + set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} ${JEMALLOC_LIBS}) + set (ARTIFACT_SUFFIX "_je") + +else () + set (ARTIFACT_SUFFIX "") + message(STATUS "JEMALLOC library is disabled") +endif () diff --git a/cozorocks/deps/zstd b/cozorocks/deps/zstd new file mode 160000 index 00000000..e47e674c --- /dev/null +++ b/cozorocks/deps/zstd @@ -0,0 +1 @@ +Subproject commit e47e674cd09583ff0503f0f6defd6d23d8b718d3 diff --git a/cozorocks/rocksdb b/cozorocks/rocksdb deleted file mode 160000 index 35cdd3e7..00000000 --- a/cozorocks/rocksdb +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 35cdd3e71e0627af2b0306e4322efd134906a74e