diff --git a/Cargo.lock b/Cargo.lock index 4667521c..1c0763f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,17 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "ahash" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43bb833f0bf979d8475d38fbf09ed3b8a55e1885fe93ad3f93239fc6a4f17b98" +dependencies = [ + "getrandom", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "0.7.18" @@ -400,6 +411,15 @@ dependencies = [ "wasi", ] +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +dependencies = [ + "ahash", +] + [[package]] name = "hermit-abi" version = "0.1.19" @@ -963,12 +983,15 @@ dependencies = [ "clap", "dashmap", "env_logger", + "hashbrown", "jemallocator", "libc", "libsky", "libstress", "log", + "num_cpus", "openssl", + "parking_lot", "rand", "regex", "serde", @@ -1186,6 +1209,12 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" +[[package]] +name = "version_check" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" + [[package]] name = "wasi" version = "0.10.0+wasi-snapshot-preview1" diff --git a/server/Cargo.toml b/server/Cargo.toml index c93c3468..322fb39b 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -23,6 +23,9 @@ chrono = "0.4.19" regex = "1.5.4" tokio-openssl = "0.6.2" openssl = { version = "0.10.35", features = ["vendored"] } +hashbrown = { version = "*", features = ["raw"] } +parking_lot = "0.11.1" +num_cpus = "1.13.0" [target.'cfg(not(target_env = "msvc"))'.dependencies] # external deps diff --git a/server/src/corestore/map/bref.rs b/server/src/corestore/map/bref.rs new file mode 100644 index 00000000..ecafa1f3 --- /dev/null +++ b/server/src/corestore/map/bref.rs @@ -0,0 +1,25 @@ +/* + * Created on Mon Aug 09 2021 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2021, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ diff --git a/server/src/corestore/map/mod.rs b/server/src/corestore/map/mod.rs new file mode 100644 index 00000000..c91bc316 --- /dev/null +++ b/server/src/corestore/map/mod.rs @@ -0,0 +1,238 @@ +/* + * Created on Mon Aug 09 2021 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2021, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +#![allow(dead_code)] // TODO(@ohsayan): Remove this lint once we're done +#![allow(clippy::manual_map)] // avoid LLVM bloat + +use core::borrow::Borrow; +use core::hash::BuildHasher; +use core::hash::Hash; +use core::hash::Hasher; +use core::mem; +use parking_lot::RwLock; +use parking_lot::RwLockReadGuard; +use parking_lot::RwLockWriteGuard; +use std::collections::hash_map::RandomState; + +type LowMap = hashbrown::raw::RawTable<(K, V)>; +type ShardSlice = [RwLock>]; +type SRlock<'a, K, V> = RwLockReadGuard<'a, hashbrown::raw::RawTable<(K, V)>>; +type SWlock<'a, K, V> = RwLockWriteGuard<'a, hashbrown::raw::RawTable<(K, V)>>; +const BITS_IN_USIZE: usize = mem::size_of::() * 8; +const DEFAULT_CAP: usize = 128; + +fn make_hash(hash_builder: &S, val: &Q) -> u64 +where + K: Borrow, + Q: Hash + ?Sized, + S: BuildHasher, +{ + let mut state = hash_builder.build_hasher(); + val.hash(&mut state); + state.finish() +} + +fn make_insert_hash(hash_builder: &S, val: &K) -> u64 +where + K: Hash, + S: BuildHasher, +{ + let mut state = hash_builder.build_hasher(); + val.hash(&mut state); + state.finish() +} + +fn make_hasher(hash_builder: &S) -> impl Fn(&(Q, V)) -> u64 + '_ +where + K: Borrow, + Q: Hash, + S: BuildHasher, +{ + move |val| make_hash::(hash_builder, &val.0) +} + +fn ceq(k: &Q) -> impl Fn(&(K, V)) -> bool + '_ +where + K: Borrow, + Q: ?Sized + Eq, +{ + move |x| k.eq(x.0.borrow()) +} + +fn get_shard_count() -> usize { + (num_cpus::get() * 4).next_power_of_two() +} + +const fn cttz(amount: usize) -> usize { + amount.trailing_zeros() as usize +} + +pub struct Skymap { + shards: Box>, + hasher: S, + shift: usize, +} + +impl Default for Skymap { + fn default() -> Self { + Self::with_hasher(RandomState::default()) + } +} + +// basic impls +impl Skymap +where + S: BuildHasher + Default, +{ + pub fn new() -> Self { + Self::with_hasher(S::default()) + } + pub fn with_capacity(cap: usize) -> Self { + Self::with_capacity_and_hasher(cap, S::default()) + } + pub fn with_capacity_and_hasher(mut cap: usize, hasher: S) -> Self { + let shard_count = get_shard_count(); + let shift = BITS_IN_USIZE - cttz(shard_count); + if cap != 0 { + cap = (cap + (shard_count - 1)) & !(shard_count - 1); + } + + let cap_per_shard = cap / shard_count; + Self { + shards: (0..shard_count) + .map(|_| RwLock::new(LowMap::with_capacity(cap_per_shard))) + .collect(), + hasher, + shift, + } + } + pub fn with_hasher(hasher: S) -> Self { + Self::with_capacity_and_hasher(DEFAULT_CAP, hasher) + } +} + +// const impls +impl Skymap { + const fn shards(&self) -> &ShardSlice { + &self.shards + } + const fn determine_shard(&self, hash: usize) -> usize { + (hash << 7) >> self.shift + } + const fn h(&self) -> &S { + &self.hasher + } +} + +// insert/get/remove impls + +impl Skymap +where + K: Eq + Hash, + S: BuildHasher + Clone, +{ + pub fn insert(&self, k: K, v: V) -> Option { + let hash = make_insert_hash::(&self.hasher, &k); + let idx = self.determine_shard(hash as usize); + unsafe { + // begin critical section + let mut lowtable = self.get_wshard_unchecked(idx); + if let Some((_, item)) = lowtable.get_mut(hash, ceq(&k)) { + Some(mem::replace(item, v)) + } else { + lowtable.insert(hash, (k, v), make_hasher::(self.h())); + None + } + // end critical section + } + } + pub fn remove(&self, k: &Q) -> Option<(K, V)> + where + K: Borrow, + Q: Hash + Eq + ?Sized, + { + let hash = make_hash::(self.h(), k); + let idx = self.determine_shard(hash as usize); + unsafe { + // begin critical section + let mut lowtable = self.get_wshard_unchecked(idx); + match lowtable.remove_entry(hash, ceq(k)) { + Some(kv) => Some(kv), + None => None, + } + // end critical section + } + } + pub fn remove_if(&self, k: &Q, f: impl Fn(&(K, V)) -> bool) -> Option<(K, V)> + where + K: Borrow, + Q: Hash + Eq + ?Sized, + { + let hash = make_hash::(self.h(), k); + let idx = self.determine_shard(hash as usize); + unsafe { + // begin critical section + let mut lowtable = self.get_wshard_unchecked(idx); + match lowtable.find(hash, ceq(k)) { + Some(bucket) => { + if f(bucket.as_ref()) { + Some(lowtable.remove(bucket)) + } else { + None + } + } + None => None, + } + // end critical section + } + } +} + +// inner impls +impl Skymap { + unsafe fn get_rshard_unchecked(&self, shard: usize) -> SRlock { + self.shards.get_unchecked(shard).read() + } + unsafe fn get_wshard_unchecked(&self, shard: usize) -> SWlock { + self.shards.get_unchecked(shard).write() + } +} + +#[test] +fn test_insert_remove() { + let map = Skymap::default(); + map.insert("hello", "world"); + assert_eq!(map.remove("hello").unwrap().1, "world"); +} + +#[test] +fn test_remove_if() { + let map = Skymap::default(); + map.insert("hello", "world"); + assert!(map + .remove_if("hello", |(_k, v)| { (*v).eq("notworld") }) + .is_none()); +} diff --git a/server/src/corestore/mod.rs b/server/src/corestore/mod.rs index 0f0431d6..87dc07ca 100644 --- a/server/src/corestore/mod.rs +++ b/server/src/corestore/mod.rs @@ -52,6 +52,7 @@ pub mod htable; pub mod iarray; pub mod lazy; pub mod lock; +pub mod map; pub mod memstore; pub mod table; #[cfg(test)]