Add preload generation
Sayan Nandan
3 years ago
parent
bca8df5863
commit
87d79650ce
@ -0,0 +1,113 @@
|
|||||||
|
/*
|
||||||
|
* Created on Sat Jul 17 2021
|
||||||
|
*
|
||||||
|
* This file is a part of Skytable
|
||||||
|
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
||||||
|
* NoSQL database written by Sayan Nandan ("the Author") with the
|
||||||
|
* vision to provide flexibility in data modelling without compromising
|
||||||
|
* on performance, queryability or scalability.
|
||||||
|
*
|
||||||
|
* Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
|
||||||
|
*
|
||||||
|
* This program is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License as published by
|
||||||
|
* the Free Software Foundation, either version 3 of the License, or
|
||||||
|
* (at your option) any later version.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
|
||||||
|
#[cfg(target_endian = "big")]
|
||||||
|
macro_rules! endian_mark {
|
||||||
|
() => {
|
||||||
|
1_u64
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_endian = "little")]
|
||||||
|
macro_rules! endian_mark {
|
||||||
|
() => {
|
||||||
|
0_u64
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
macro_rules! little_endian {
|
||||||
|
($block:block) => {
|
||||||
|
#[cfg(target_endian = "little")]
|
||||||
|
{
|
||||||
|
$block
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
macro_rules! big_endian {
|
||||||
|
($block:block) => {
|
||||||
|
#[cfg(target_endian = "big")]
|
||||||
|
{
|
||||||
|
$block
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
macro_rules! not_64_bit {
|
||||||
|
($block:block) => {
|
||||||
|
#[cfg(not(target_pointer_width = "64"))]
|
||||||
|
{
|
||||||
|
$block
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
macro_rules! is_64_bit {
|
||||||
|
($block:block) => {
|
||||||
|
#[cfg(target_pointer_width = "64")]
|
||||||
|
{
|
||||||
|
$block
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_endian = "big")]
|
||||||
|
macro_rules! to_64bit_little_endian {
|
||||||
|
($e:expr) => {
|
||||||
|
($e as u64).swap_bytes()
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_endian = "little")]
|
||||||
|
macro_rules! to_64bit_little_endian {
|
||||||
|
($e:expr) => {
|
||||||
|
($e as u64)
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
macro_rules! try_dir_ignore_existing {
|
||||||
|
($dir:expr) => {{
|
||||||
|
match std::fs::create_dir_all($dir) {
|
||||||
|
Ok(_) => Ok(()),
|
||||||
|
Err(e) => match e.kind() {
|
||||||
|
std::io::ErrorKind::AlreadyExists => Ok(()),
|
||||||
|
_ => Err(e),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}};
|
||||||
|
($($dir:expr),*) => {
|
||||||
|
$(try_dir_ignore_existing!($dir)?;)*
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[macro_export]
|
||||||
|
macro_rules! concat_path {
|
||||||
|
($($s:expr),*) => {{ {
|
||||||
|
let mut path = std::path::PathBuf::with_capacity($(($s).len()+)*0);
|
||||||
|
$(path.push($s);)*
|
||||||
|
path
|
||||||
|
}}};
|
||||||
|
}
|
||||||
@ -0,0 +1,69 @@
|
|||||||
|
/*
|
||||||
|
* Created on Sat Jul 17 2021
|
||||||
|
*
|
||||||
|
* This file is a part of Skytable
|
||||||
|
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
||||||
|
* NoSQL database written by Sayan Nandan ("the Author") with the
|
||||||
|
* vision to provide flexibility in data modelling without compromising
|
||||||
|
* on performance, queryability or scalability.
|
||||||
|
*
|
||||||
|
* Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
|
||||||
|
*
|
||||||
|
* This program is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License as published by
|
||||||
|
* the Free Software Foundation, either version 3 of the License, or
|
||||||
|
* (at your option) any later version.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
|
||||||
|
use super::raw_byte_repr;
|
||||||
|
use crate::coredb::memstore::Memstore;
|
||||||
|
use std::io::Result as IoResult;
|
||||||
|
use std::io::Write;
|
||||||
|
|
||||||
|
const VERSION_MARK: u64 = 1u64.swap_bytes();
|
||||||
|
|
||||||
|
/// Add padding bytes to align to 8B boundaries
|
||||||
|
fn pad_nul_align8<W: Write>(l: usize, w: &mut W) -> IoResult<()> {
|
||||||
|
// ignore handled amount
|
||||||
|
let _ = w.write(&[b'0'].repeat(64 - l))?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Generate the `PRELOAD` disk file for this instance
|
||||||
|
/// ```text
|
||||||
|
/// [8B: Endian Mark/Version Mark (padded)] => Meta segment
|
||||||
|
/// [8B: Extent header] => Predata Segment
|
||||||
|
/// ([8B: Parition ID (nul padded)])* => Data segment
|
||||||
|
/// ```
|
||||||
|
///
|
||||||
|
/// The meta segment need not be 8B, but it is done for easier alignment
|
||||||
|
pub fn raw_generate_preload<W: Write>(w: &mut W, store: Memstore) -> IoResult<()> {
|
||||||
|
unsafe {
|
||||||
|
// generate the meta segment
|
||||||
|
#[allow(clippy::identity_op)] // clippy doesn't understand endian
|
||||||
|
let meta_segment = endian_mark!() | VERSION_MARK;
|
||||||
|
w.write_all(&raw_byte_repr(&meta_segment))?;
|
||||||
|
|
||||||
|
// generate and write the extent header (predata)
|
||||||
|
w.write_all(&raw_byte_repr(&to_64bit_little_endian!(store
|
||||||
|
.keyspaces
|
||||||
|
.len())))?;
|
||||||
|
}
|
||||||
|
// start writing the parition IDs
|
||||||
|
for partition in store.keyspaces.iter() {
|
||||||
|
let partition_id = partition.key();
|
||||||
|
w.write_all(&partition_id)?;
|
||||||
|
// pad
|
||||||
|
pad_nul_align8(partition_id.len(), w)?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
@ -0,0 +1,177 @@
|
|||||||
|
/*
|
||||||
|
* Created on Sat Jul 17 2021
|
||||||
|
*
|
||||||
|
* This file is a part of Skytable
|
||||||
|
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
||||||
|
* NoSQL database written by Sayan Nandan ("the Author") with the
|
||||||
|
* vision to provide flexibility in data modelling without compromising
|
||||||
|
* on performance, queryability or scalability.
|
||||||
|
*
|
||||||
|
* Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
|
||||||
|
*
|
||||||
|
* This program is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License as published by
|
||||||
|
* the Free Software Foundation, either version 3 of the License, or
|
||||||
|
* (at your option) any later version.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_serialize_deserialize_empty() {
|
||||||
|
let cmap = Coremap::new();
|
||||||
|
let ser = serialize_map(&cmap).unwrap();
|
||||||
|
let de = deserialize(ser).unwrap();
|
||||||
|
assert!(de.len() == 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_ser_de_few_elements() {
|
||||||
|
let cmap = Coremap::new();
|
||||||
|
cmap.upsert("sayan".into(), "writes code".into());
|
||||||
|
cmap.upsert("supersayan".into(), "writes super code".into());
|
||||||
|
let ser = serialize_map(&cmap).unwrap();
|
||||||
|
let de = deserialize(ser).unwrap();
|
||||||
|
assert!(de.len() == cmap.len());
|
||||||
|
assert!(de
|
||||||
|
.iter()
|
||||||
|
.all(|kv| cmap.get(kv.key()).unwrap().eq(kv.value())));
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg_test!(
|
||||||
|
use libstress::utils::generate_random_string_vector;
|
||||||
|
use rand::thread_rng;
|
||||||
|
#[test]
|
||||||
|
fn roast_the_serializer() {
|
||||||
|
const COUNT: usize = 1000_usize;
|
||||||
|
const LEN: usize = 8_usize;
|
||||||
|
let mut rng = thread_rng();
|
||||||
|
let (keys, values) = (
|
||||||
|
generate_random_string_vector(COUNT, LEN, &mut rng, true),
|
||||||
|
generate_random_string_vector(COUNT, LEN, &mut rng, false),
|
||||||
|
);
|
||||||
|
let cmap: Coremap<Data, Data> = keys
|
||||||
|
.iter()
|
||||||
|
.zip(values.iter())
|
||||||
|
.map(|(k, v)| (Data::from(k.to_owned()), Data::from(v.to_owned())))
|
||||||
|
.collect();
|
||||||
|
let ser = serialize_map(&cmap).unwrap();
|
||||||
|
let de = deserialize(ser).unwrap();
|
||||||
|
assert!(de
|
||||||
|
.iter()
|
||||||
|
.all(|kv| cmap.get(kv.key()).unwrap().eq(kv.value())));
|
||||||
|
assert!(de.len() == cmap.len());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_ser_de_safety() {
|
||||||
|
const COUNT: usize = 1000_usize;
|
||||||
|
const LEN: usize = 8_usize;
|
||||||
|
let mut rng = thread_rng();
|
||||||
|
let (keys, values) = (
|
||||||
|
generate_random_string_vector(COUNT, LEN, &mut rng, true),
|
||||||
|
generate_random_string_vector(COUNT, LEN, &mut rng, false),
|
||||||
|
);
|
||||||
|
let cmap: Coremap<Data, Data> = keys
|
||||||
|
.iter()
|
||||||
|
.zip(values.iter())
|
||||||
|
.map(|(k, v)| (Data::from(k.to_owned()), Data::from(v.to_owned())))
|
||||||
|
.collect();
|
||||||
|
let mut se = serialize_map(&cmap).unwrap();
|
||||||
|
// random chop
|
||||||
|
se.truncate(124);
|
||||||
|
// corrupted
|
||||||
|
assert!(deserialize(se).is_none());
|
||||||
|
}
|
||||||
|
#[test]
|
||||||
|
fn test_ser_de_excess_bytes() {
|
||||||
|
// this test needs a lot of auxiliary space
|
||||||
|
// we can approximate this to be: 100,000 x 30 bytes = 3,000,000 bytes
|
||||||
|
// and then we may have a clone overhead + heap allocation by the map
|
||||||
|
// so ~9,000,000 bytes or ~9MB
|
||||||
|
const COUNT: usize = 1000_usize;
|
||||||
|
const LEN: usize = 8_usize;
|
||||||
|
let mut rng = thread_rng();
|
||||||
|
let (keys, values) = (
|
||||||
|
generate_random_string_vector(COUNT, LEN, &mut rng, true),
|
||||||
|
generate_random_string_vector(COUNT, LEN, &mut rng, false),
|
||||||
|
);
|
||||||
|
let cmap: Coremap<Data, Data> = keys
|
||||||
|
.iter()
|
||||||
|
.zip(values.iter())
|
||||||
|
.map(|(k, v)| (Data::from(k.to_owned()), Data::from(v.to_owned())))
|
||||||
|
.collect();
|
||||||
|
let mut se = serialize_map(&cmap).unwrap();
|
||||||
|
// random patch
|
||||||
|
let patch: Vec<u8> = (0u16..500u16).into_iter().map(|v| (v >> 7) as u8).collect();
|
||||||
|
se.extend(patch);
|
||||||
|
assert!(deserialize(se).is_none());
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
#[cfg(target_pointer_width = "32")]
|
||||||
|
#[test]
|
||||||
|
#[should_panic]
|
||||||
|
fn test_runtime_panic_32bit_or_lower() {
|
||||||
|
let max = u64::MAX;
|
||||||
|
let byte_stream = unsafe { raw_byte_repr(&max).to_owned() };
|
||||||
|
let ptr = byte_stream.as_ptr();
|
||||||
|
unsafe { transmute_len(ptr) };
|
||||||
|
}
|
||||||
|
|
||||||
|
mod interface_tests {
|
||||||
|
use super::interface::{cow_file, create_tree, DIR_KSROOT, DIR_ROOT, DIR_SNAPROOT};
|
||||||
|
use crate::concat_path;
|
||||||
|
use crate::coredb::memstore::Memstore;
|
||||||
|
use std::fs;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
#[test]
|
||||||
|
fn test_tree() {
|
||||||
|
create_tree(Memstore::new_default()).unwrap();
|
||||||
|
let read_ks: Vec<String> = fs::read_dir(DIR_KSROOT)
|
||||||
|
.unwrap()
|
||||||
|
.map(|dir| {
|
||||||
|
let v = dir.unwrap().file_name();
|
||||||
|
v.to_string_lossy().to_string()
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
assert_eq!(read_ks, vec!["default".to_owned()]);
|
||||||
|
// just read one level of the snaps dir
|
||||||
|
let read_snaps: Vec<String> = fs::read_dir(DIR_SNAPROOT)
|
||||||
|
.unwrap()
|
||||||
|
.map(|dir| {
|
||||||
|
let v = dir.unwrap().file_name();
|
||||||
|
v.to_string_lossy().to_string()
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
assert_eq!(read_snaps, vec!["default".to_owned()]);
|
||||||
|
// now read level two: snaps/default
|
||||||
|
let read_snaps: Vec<String> = fs::read_dir(concat_path!(DIR_SNAPROOT, "default"))
|
||||||
|
.unwrap()
|
||||||
|
.map(|dir| {
|
||||||
|
let v = dir.unwrap().file_name();
|
||||||
|
v.to_string_lossy().to_string()
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
assert_veceq!(read_snaps, vec!["_system".to_owned(), "default".to_owned()]);
|
||||||
|
assert!(PathBuf::from("data/backups").is_dir());
|
||||||
|
// clean up
|
||||||
|
fs::remove_dir_all(DIR_ROOT).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_cowfile() {
|
||||||
|
let cow_file = cow_file(10);
|
||||||
|
assert_eq!(cow_file, "10_".to_owned());
|
||||||
|
assert_eq!(&cow_file[..cow_file.len() - 1], "10".to_owned());
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue