From 610144f78ed7b7788f63c4c360bc1f78c607e8e0 Mon Sep 17 00:00:00 2001 From: Sayan Nandan Date: Sat, 17 Jul 2021 21:12:44 +0530 Subject: [PATCH] Add unflush routines --- server/src/storage/flush.rs | 142 ++++++++++++++++++---------------- server/src/storage/macros.rs | 7 ++ server/src/storage/mod.rs | 2 + server/src/storage/unflush.rs | 76 ++++++++++++++++++ 4 files changed, 161 insertions(+), 66 deletions(-) create mode 100644 server/src/storage/unflush.rs diff --git a/server/src/storage/flush.rs b/server/src/storage/flush.rs index 26a231cb..8009de03 100644 --- a/server/src/storage/flush.rs +++ b/server/src/storage/flush.rs @@ -29,88 +29,98 @@ //! This module contains multiple flush routines: at the memstore level, the keyspace level and //! the table level +use super::interface; use crate::coredb::memstore::Keyspace; use crate::coredb::memstore::Memstore; use crate::coredb::memstore::ObjectID; -use crate::coredb::table::{DataModel, Table}; -use crate::storage::interface::DIR_KSROOT; -use std::fs::{self, File}; use std::io::Result as IoResult; -const PRELOAD_FILE_PATH_TEMP: &str = "data/ks/PRELOAD_"; -const PRELOAD_FILE_PATH_TEMP_LEN_CHOP: usize = PRELOAD_FILE_PATH_TEMP.len() - 1; - -macro_rules! tbl_path { - ($ksid:expr, $tableid:expr) => { - unsafe { concat_str!(DIR_KSROOT, "/", $ksid.as_str(), "/", $tableid.as_str()) } - }; +/// Flushes the entire **keyspace + partmap** +pub fn flush_keyspace_full(ksid: &ObjectID, keyspace: &Keyspace) -> IoResult<()> { + self::oneshot::flush_partmap(ksid, keyspace)?; + self::oneshot::flush_keyspace(ksid, keyspace) } -/// No `partmap` handling. Just flushes the table to the expected location -pub fn flush_table(tableid: &ObjectID, ksid: &ObjectID, table: &Table) -> IoResult<()> { - let path = tbl_path!(tableid, ksid); - let mut file = File::create(&path)?; - let modelcode = table.get_model_code(); - match table.get_model_ref() { - DataModel::KV(kve) => super::interface::serialize_map_into_slow_buffer( - &mut file, - kve.__get_inner_ref(), - modelcode, - )?, +/// Flush the entire **preload + keyspaces + their partmaps** +pub fn flush_full(store: &Memstore) -> IoResult<()> { + self::oneshot::flush_preload(store)?; + for keyspace in store.keyspaces.iter() { + self::flush_keyspace_full(keyspace.key(), keyspace.value())?; } - file.sync_all()?; - fs::rename(&path, &path[..path.len() - 1]) + Ok(()) } -/// Flushes an entire keyspace to the expected location. No `partmap` or `preload` handling -pub fn flush_keyspace(ksid: &ObjectID, keyspace: &Keyspace) -> IoResult<()> { - for table in keyspace.tables.iter() { - self::flush_table(table.key(), &ksid, table.value())?; +pub mod oneshot { + //! # Irresponsible flushing + //! + //! Every function does **exactly what it says** and nothing more. No partition + //! files et al are handled + //! + use super::*; + use crate::coredb::table::{DataModel, Table}; + use crate::storage::interface::DIR_KSROOT; + use std::fs::{self, File}; + + const PRELOAD_FILE_PATH_TEMP: &str = "data/ks/PRELOAD_"; + const PRELOAD_FILE_PATH_TEMP_LEN_CHOP: usize = PRELOAD_FILE_PATH_TEMP.len() - 1; + + macro_rules! tbl_path { + ($ksid:expr, $tableid:expr) => { + unsafe { concat_str!(DIR_KSROOT, "/", $ksid.as_str(), "/", $tableid.as_str()) } + }; } - Ok(()) -} -/// Flushes a single partmap -pub fn flush_partmap(ksid: &ObjectID, keyspace: &Keyspace) -> IoResult<()> { - let path = unsafe { concat_str!(DIR_KSROOT, "/", ksid.as_str(), "/", "PARTMAP_") }; - let mut file = File::create(&path)?; - super::interface::serialize_partmap_into_slow_buffer(&mut file, keyspace)?; - file.sync_all()?; - fs::rename(&path, &path[..path.len() - 1])?; - Ok(()) -} + /// No `partmap` handling. Just flushes the table to the expected location + pub fn flush_table(tableid: &ObjectID, ksid: &ObjectID, table: &Table) -> IoResult<()> { + let path = tbl_path!(tableid, ksid); + let mut file = File::create(&path)?; + let modelcode = table.get_model_code(); + match table.get_model_ref() { + DataModel::KV(kve) => super::interface::serialize_map_into_slow_buffer( + &mut file, + kve.__get_inner_ref(), + modelcode, + )?, + } + file.sync_all()?; + fs::rename(&path, &path[..path.len() - 1]) + } -/// Flushes the entire **keyspace + partmap** -pub fn flush_keyspace_full(ksid: &ObjectID, keyspace: &Keyspace) -> IoResult<()> { - self::flush_partmap(ksid, keyspace)?; - self::flush_keyspace(ksid, keyspace) -} + /// Flushes an entire keyspace to the expected location. No `partmap` or `preload` handling + pub fn flush_keyspace(ksid: &ObjectID, keyspace: &Keyspace) -> IoResult<()> { + for table in keyspace.tables.iter() { + self::flush_table(table.key(), &ksid, table.value())?; + } + Ok(()) + } -/// Flushes everything in memory. No `partmap` or `preload` handling -pub fn flush(store: &Memstore) -> IoResult<()> { - for keyspace in store.keyspaces.iter() { - self::flush_keyspace(keyspace.key(), keyspace.value())?; + /// Flushes a single partmap + pub fn flush_partmap(ksid: &ObjectID, keyspace: &Keyspace) -> IoResult<()> { + let path = unsafe { concat_str!(DIR_KSROOT, "/", ksid.as_str(), "/", "PARTMAP_") }; + let mut file = File::create(&path)?; + super::interface::serialize_partmap_into_slow_buffer(&mut file, keyspace)?; + file.sync_all()?; + fs::rename(&path, &path[..path.len() - 1])?; + Ok(()) } - Ok(()) -} -// Flush the `PRELOAD` -pub fn flush_preload(store: &Memstore) -> IoResult<()> { - let mut file = File::create(PRELOAD_FILE_PATH_TEMP)?; - super::interface::serialize_preload_into_slow_buffer(&mut file, store)?; - file.sync_all()?; - fs::rename( - &PRELOAD_FILE_PATH_TEMP, - &PRELOAD_FILE_PATH_TEMP[..PRELOAD_FILE_PATH_TEMP_LEN_CHOP], - )?; - Ok(()) -} + /// Flushes everything in memory. No `partmap` or `preload` handling + pub fn flush(store: &Memstore) -> IoResult<()> { + for keyspace in store.keyspaces.iter() { + self::flush_keyspace(keyspace.key(), keyspace.value())?; + } + Ok(()) + } -/// Flush the entire **preload + keyspaces + their partmaps** -pub fn flush_full(store: &Memstore) -> IoResult<()> { - self::flush_preload(store)?; - for keyspace in store.keyspaces.iter() { - self::flush_keyspace_full(keyspace.key(), keyspace.value())?; + // Flush the `PRELOAD` + pub fn flush_preload(store: &Memstore) -> IoResult<()> { + let mut file = File::create(PRELOAD_FILE_PATH_TEMP)?; + super::interface::serialize_preload_into_slow_buffer(&mut file, store)?; + file.sync_all()?; + fs::rename( + &PRELOAD_FILE_PATH_TEMP, + &PRELOAD_FILE_PATH_TEMP[..PRELOAD_FILE_PATH_TEMP_LEN_CHOP], + )?; + Ok(()) } - Ok(()) } diff --git a/server/src/storage/macros.rs b/server/src/storage/macros.rs index 9156cefb..e08958bd 100644 --- a/server/src/storage/macros.rs +++ b/server/src/storage/macros.rs @@ -106,3 +106,10 @@ macro_rules! concat_str { st }}}; } + +#[macro_export] +macro_rules! bad_data { + () => { + std::io::Error::from(std::io::ErrorKind::InvalidData) + }; +} diff --git a/server/src/storage/mod.rs b/server/src/storage/mod.rs index 92be4ee5..5189adbc 100644 --- a/server/src/storage/mod.rs +++ b/server/src/storage/mod.rs @@ -66,6 +66,8 @@ mod macros; pub mod flush; pub mod interface; pub mod preload; +pub mod unflush; +// test #[cfg(test)] mod tests; diff --git a/server/src/storage/unflush.rs b/server/src/storage/unflush.rs new file mode 100644 index 00000000..3f32d22e --- /dev/null +++ b/server/src/storage/unflush.rs @@ -0,0 +1,76 @@ +/* + * Created on Sat Jul 17 2021 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2021, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +//! # Unflush routines +//! +//! Routines for unflushing data + +use crate::coredb::memstore::ObjectID; +use crate::coredb::table::Table; +use crate::storage::interface::DIR_KSROOT; +use crate::storage::Coremap; +use std::collections::HashMap; +use std::fs; +use std::io::Error as IoError; +use std::io::ErrorKind; +use std::io::Result as IoResult; +use std::sync::Arc; + +/// Read a given table into a [`Table`] object +pub fn read_table(ksid: &ObjectID, tblid: &ObjectID, volatile: bool) -> IoResult { + let filepath = unsafe { concat_path!(DIR_KSROOT, ksid.as_str(), tblid.as_str()) }; + let read = fs::read(filepath)?; + // TODO(@ohsayan): Mod this to de based on data model + let (data, model) = super::de::deserialize_map(read).ok_or_else(|| bad_data!())?; + match model { + 0 => { + // kve + Table::kve_from_model_code_and_data(model, volatile, data).ok_or_else(|| bad_data!()) + } + _ => { + // some model that we don't know + Err(IoError::from(ErrorKind::Unsupported)) + } + } +} + +/// Read an entire keyspace into a Coremap. You'll need to initialize the rest +pub fn read_keyspace(ksid: ObjectID) -> IoResult>> { + let filepath = unsafe { concat_path!(DIR_KSROOT, ksid.as_str(), "PARTMAP") }; + let partmap: HashMap = + super::de::deserialize_set_ctype_bytemark(&fs::read(filepath)?) + .ok_or_else(|| bad_data!())?; + let ks: Coremap> = Coremap::with_capacity(partmap.len()); + for (tableid, table_storage_type) in partmap.into_iter() { + if table_storage_type > 1 { + return Err(bad_data!()); + } + let is_volatile = table_storage_type == 1; + let tbl = self::read_table(&ksid, &tableid, is_volatile)?; + ks.true_if_insert(tableid, Arc::new(tbl)); + } + Ok(ks) +}