Add unflush routines

server/src/storage/flush.rs

@@ -29,88 +29,98 @@
 //! This module contains multiple flush routines: at the memstore level, the keyspace level and
 //! the table level
 
+use super::interface;
 use crate::coredb::memstore::Keyspace;
 use crate::coredb::memstore::Memstore;
 use crate::coredb::memstore::ObjectID;
-use crate::coredb::table::{DataModel, Table};
-use crate::storage::interface::DIR_KSROOT;
-use std::fs::{self, File};
 use std::io::Result as IoResult;
 
-const PRELOAD_FILE_PATH_TEMP: &str = "data/ks/PRELOAD_";
+/// Flushes the entire **keyspace + partmap**
-const PRELOAD_FILE_PATH_TEMP_LEN_CHOP: usize = PRELOAD_FILE_PATH_TEMP.len() - 1;
+pub fn flush_keyspace_full(ksid: &ObjectID, keyspace: &Keyspace) -> IoResult<()> {
-
+    self::oneshot::flush_partmap(ksid, keyspace)?;
-macro_rules! tbl_path {
+    self::oneshot::flush_keyspace(ksid, keyspace)
-    ($ksid:expr, $tableid:expr) => {
-        unsafe { concat_str!(DIR_KSROOT, "/", $ksid.as_str(), "/", $tableid.as_str()) }
-    };
 }
 
-/// No `partmap` handling. Just flushes the table to the expected location
+/// Flush the entire **preload + keyspaces + their partmaps**
-pub fn flush_table(tableid: &ObjectID, ksid: &ObjectID, table: &Table) -> IoResult<()> {
+pub fn flush_full(store: &Memstore) -> IoResult<()> {
-    let path = tbl_path!(tableid, ksid);
+    self::oneshot::flush_preload(store)?;
-    let mut file = File::create(&path)?;
+    for keyspace in store.keyspaces.iter() {
-    let modelcode = table.get_model_code();
+        self::flush_keyspace_full(keyspace.key(), keyspace.value())?;
-    match table.get_model_ref() {
-        DataModel::KV(kve) => super::interface::serialize_map_into_slow_buffer(
-            &mut file,
-            kve.__get_inner_ref(),
-            modelcode,
-        )?,
     }
-    file.sync_all()?;
+    Ok(())
-    fs::rename(&path, &path[..path.len() - 1])
 }
 
-/// Flushes an entire keyspace to the expected location. No `partmap` or `preload` handling
+pub mod oneshot {
-pub fn flush_keyspace(ksid: &ObjectID, keyspace: &Keyspace) -> IoResult<()> {
+    //! # Irresponsible flushing
-    for table in keyspace.tables.iter() {
+    //!
-        self::flush_table(table.key(), &ksid, table.value())?;
+    //! Every function does **exactly what it says** and nothing more. No partition
+    //! files et al are handled
+    //!
+    use super::*;
+    use crate::coredb::table::{DataModel, Table};
+    use crate::storage::interface::DIR_KSROOT;
+    use std::fs::{self, File};
+
+    const PRELOAD_FILE_PATH_TEMP: &str = "data/ks/PRELOAD_";
+    const PRELOAD_FILE_PATH_TEMP_LEN_CHOP: usize = PRELOAD_FILE_PATH_TEMP.len() - 1;
+
+    macro_rules! tbl_path {
+        ($ksid:expr, $tableid:expr) => {
+            unsafe { concat_str!(DIR_KSROOT, "/", $ksid.as_str(), "/", $tableid.as_str()) }
+        };
     }
-    Ok(())
-}
 
-/// Flushes a single partmap
+    /// No `partmap` handling. Just flushes the table to the expected location
-pub fn flush_partmap(ksid: &ObjectID, keyspace: &Keyspace) -> IoResult<()> {
+    pub fn flush_table(tableid: &ObjectID, ksid: &ObjectID, table: &Table) -> IoResult<()> {
-    let path = unsafe { concat_str!(DIR_KSROOT, "/", ksid.as_str(), "/", "PARTMAP_") };
+        let path = tbl_path!(tableid, ksid);
-    let mut file = File::create(&path)?;
+        let mut file = File::create(&path)?;
-    super::interface::serialize_partmap_into_slow_buffer(&mut file, keyspace)?;
+        let modelcode = table.get_model_code();
-    file.sync_all()?;
+        match table.get_model_ref() {
-    fs::rename(&path, &path[..path.len() - 1])?;
+            DataModel::KV(kve) => super::interface::serialize_map_into_slow_buffer(
-    Ok(())
+                &mut file,
-}
+                kve.__get_inner_ref(),
+                modelcode,
+            )?,
+        }
+        file.sync_all()?;
+        fs::rename(&path, &path[..path.len() - 1])
+    }
 
-/// Flushes the entire **keyspace + partmap**
+    /// Flushes an entire keyspace to the expected location. No `partmap` or `preload` handling
-pub fn flush_keyspace_full(ksid: &ObjectID, keyspace: &Keyspace) -> IoResult<()> {
+    pub fn flush_keyspace(ksid: &ObjectID, keyspace: &Keyspace) -> IoResult<()> {
-    self::flush_partmap(ksid, keyspace)?;
+        for table in keyspace.tables.iter() {
-    self::flush_keyspace(ksid, keyspace)
+            self::flush_table(table.key(), &ksid, table.value())?;
-}
+        }
+        Ok(())
+    }
 
-/// Flushes everything in memory. No `partmap` or `preload` handling
+    /// Flushes a single partmap
-pub fn flush(store: &Memstore) -> IoResult<()> {
+    pub fn flush_partmap(ksid: &ObjectID, keyspace: &Keyspace) -> IoResult<()> {
-    for keyspace in store.keyspaces.iter() {
+        let path = unsafe { concat_str!(DIR_KSROOT, "/", ksid.as_str(), "/", "PARTMAP_") };
-        self::flush_keyspace(keyspace.key(), keyspace.value())?;
+        let mut file = File::create(&path)?;
+        super::interface::serialize_partmap_into_slow_buffer(&mut file, keyspace)?;
+        file.sync_all()?;
+        fs::rename(&path, &path[..path.len() - 1])?;
+        Ok(())
     }
-    Ok(())
-}
 
-// Flush the `PRELOAD`
+    /// Flushes everything in memory. No `partmap` or `preload` handling
-pub fn flush_preload(store: &Memstore) -> IoResult<()> {
+    pub fn flush(store: &Memstore) -> IoResult<()> {
-    let mut file = File::create(PRELOAD_FILE_PATH_TEMP)?;
+        for keyspace in store.keyspaces.iter() {
-    super::interface::serialize_preload_into_slow_buffer(&mut file, store)?;
+            self::flush_keyspace(keyspace.key(), keyspace.value())?;
-    file.sync_all()?;
+        }
-    fs::rename(
+        Ok(())
-        &PRELOAD_FILE_PATH_TEMP,
+    }
-        &PRELOAD_FILE_PATH_TEMP[..PRELOAD_FILE_PATH_TEMP_LEN_CHOP],
-    )?;
-    Ok(())
-}
 
-/// Flush the entire **preload + keyspaces + their partmaps**
+    // Flush the `PRELOAD`
-pub fn flush_full(store: &Memstore) -> IoResult<()> {
+    pub fn flush_preload(store: &Memstore) -> IoResult<()> {
-    self::flush_preload(store)?;
+        let mut file = File::create(PRELOAD_FILE_PATH_TEMP)?;
-    for keyspace in store.keyspaces.iter() {
+        super::interface::serialize_preload_into_slow_buffer(&mut file, store)?;
-        self::flush_keyspace_full(keyspace.key(), keyspace.value())?;
+        file.sync_all()?;
+        fs::rename(
+            &PRELOAD_FILE_PATH_TEMP,
+            &PRELOAD_FILE_PATH_TEMP[..PRELOAD_FILE_PATH_TEMP_LEN_CHOP],
+        )?;
+        Ok(())
     }
-    Ok(())
 }

server/src/storage/macros.rs

@@ -106,3 +106,10 @@ macro_rules! concat_str {
         st
     }}};
 }
+
+#[macro_export]
+macro_rules! bad_data {
+    () => {
+        std::io::Error::from(std::io::ErrorKind::InvalidData)
+    };
+}

server/src/storage/mod.rs

@@ -66,6 +66,8 @@ mod macros;
 pub mod flush;
 pub mod interface;
 pub mod preload;
+pub mod unflush;
+// test
 #[cfg(test)]
 mod tests;
 

server/src/storage/unflush.rs

@@ -0,0 +1,76 @@
+/*
+ * Created on Sat Jul 17 2021
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+//! # Unflush routines
+//!
+//! Routines for unflushing data
+
+use crate::coredb::memstore::ObjectID;
+use crate::coredb::table::Table;
+use crate::storage::interface::DIR_KSROOT;
+use crate::storage::Coremap;
+use std::collections::HashMap;
+use std::fs;
+use std::io::Error as IoError;
+use std::io::ErrorKind;
+use std::io::Result as IoResult;
+use std::sync::Arc;
+
+/// Read a given table into a [`Table`] object
+pub fn read_table(ksid: &ObjectID, tblid: &ObjectID, volatile: bool) -> IoResult<Table> {
+    let filepath = unsafe { concat_path!(DIR_KSROOT, ksid.as_str(), tblid.as_str()) };
+    let read = fs::read(filepath)?;
+    // TODO(@ohsayan): Mod this to de based on data model
+    let (data, model) = super::de::deserialize_map(read).ok_or_else(|| bad_data!())?;
+    match model {
+        0 => {
+            // kve
+            Table::kve_from_model_code_and_data(model, volatile, data).ok_or_else(|| bad_data!())
+        }
+        _ => {
+            // some model that we don't know
+            Err(IoError::from(ErrorKind::Unsupported))
+        }
+    }
+}
+
+/// Read an entire keyspace into a Coremap. You'll need to initialize the rest
+pub fn read_keyspace(ksid: ObjectID) -> IoResult<Coremap<ObjectID, Arc<Table>>> {
+    let filepath = unsafe { concat_path!(DIR_KSROOT, ksid.as_str(), "PARTMAP") };
+    let partmap: HashMap<ObjectID, u8> =
+        super::de::deserialize_set_ctype_bytemark(&fs::read(filepath)?)
+            .ok_or_else(|| bad_data!())?;
+    let ks: Coremap<ObjectID, Arc<Table>> = Coremap::with_capacity(partmap.len());
+    for (tableid, table_storage_type) in partmap.into_iter() {
+        if table_storage_type > 1 {
+            return Err(bad_data!());
+        }
+        let is_volatile = table_storage_type == 1;
+        let tbl = self::read_table(&ksid, &tableid, is_volatile)?;
+        ks.true_if_insert(tableid, Arc::new(tbl));
+    }
+    Ok(ks)
+}