Add persistence suite for keymap string table types

Also fixed persistence bug with tables in the `system` keyspace

Makefile

@@ -141,6 +141,22 @@ test: .pre
 	@echo "Waiting for server to shut down ..."
 	@${STOP_SERVER}
 	@sleep 10
+  # now run persistence suite
+	@${SEP}
+	@echo "Running persistence suite ..."
+	@${SEP}
+	@mkdir -p server1 && cd server1 && ${START_SERVER}
+	@mkdir -p server2 && cd server2 && ${START_SERVER2}
+	@echo "Sleeping for 10 seconds to let the server start up ..."
+	@sleep 10
+	@echo "Finished sleeping"
+	@${SEP}
+	@${SEP}
+	@echo "Running all persistence tests ..."
+	@${TEST} --features persist-suite
+	@echo "Waiting for server to shut down ..."
+	@${STOP_SERVER}
+	@sleep 10
 	@echo "Removing temporary files ..."
 	@rm -rf .sky_pid server1 server2 .skytest_*
 	@${SEP}

server/Cargo.toml

@@ -56,6 +56,7 @@ libc = "0.2.119"
 
 [features]
 nightly = []
+persist-suite = []
 
 [package.metadata.deb]
 name = "skytable"

server/src/storage/v1/flush.rs

@@ -226,11 +226,11 @@ pub fn flush_full<T: StorageTarget>(target: T, store: &Memstore) -> IoResult<()>
         self::flush_keyspace_full(&target, keyspace.key(), keyspace.value().as_ref())?;
     }
     // flush system tables
-    // HACK(@ohsayan): DO NOT REORDER THIS. THE above loop will flush a PARTMAP once. But
-    // this has to be done again! The system keyspace in the above loop is a dummy one
-    // because it is located in a different field. So, we need to flush the actual list
-    // of tables
-    self::oneshot::flush_partmap(&target, &SYSTEM, &store.system)?;
+    // HACK(@ohsayan): DO NOT REORDER THIS. THE above loop will flush a PARTMAP and an empty
+    // keyspace once. But this has to be done again! The system keyspace in the above loop is a
+    // dummy one because it is located in a different field. So, we need to flush the actual
+    // tables
+    self::flush_keyspace_full(&target, &SYSTEM, &store.system)?;
     Ok(())
 }
 

server/src/tests/macros.rs

@@ -39,6 +39,41 @@ macro_rules! setkeys {
             Element::UnsignedInt(count)
         );
     };
+    ($con:ident, $($key:expr => $value:expr),*) => {
+        let mut q = ::skytable::Query::new();
+        q.push("MSET");
+        let mut count = 0;
+        $(
+            q.push($key);
+            q.push($value);
+            count += 1;
+        )*
+        assert_eq!(
+            $con.run_query_raw(&q).await.unwrap(),
+            ::skytable::Element::UnsignedInt(count)
+        );
+    };
+}
+
+macro_rules! switch_entity {
+    ($con:expr, $entity:expr) => {
+        runeq!(
+            $con,
+            ::skytable::query!("use", $entity),
+            ::skytable::Element::RespCode(::skytable::RespCode::Okay)
+        )
+    };
+}
+
+macro_rules! create_table_and_switch {
+    ($con:expr, $table:expr, $decl:expr) => {{
+        runeq!(
+            $con,
+            ::skytable::query!("create", "table", $table, $decl),
+            ::skytable::Element::RespCode(::skytable::RespCode::Okay)
+        );
+        switch_entity!($con, $table);
+    }};
 }
 
 macro_rules! push {

server/src/tests/mod.rs

@@ -28,12 +28,14 @@
 
 #[macro_use]
 mod macros;
+#[cfg(not(feature = "persist-suite"))]
 mod auth;
 mod ddl_tests;
 mod inspect_tests;
 mod kvengine;
 mod kvengine_encoding;
 mod kvengine_list;
+mod persist;
 mod pipeline;
 
 mod tls {

server/src/tests/persist.rs

@@ -0,0 +1,204 @@
+/*
+ * Created on Thu Mar 17 2022
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use sky_macros::dbtest_func as dbtest;
+use skytable::{aio::Connection, query, types::RawString, Element, Query, RespCode};
+
+const PERIST_TEST_SET_SIZE: usize = 4;
+
+trait AsQueryItem {
+    fn push_into_query(&self, query: &mut Query);
+    fn as_element(&self) -> Element;
+}
+
+impl AsQueryItem for &'static [u8] {
+    fn push_into_query(&self, query: &mut Query) {
+        query.push(RawString::from(self.to_vec()));
+    }
+    fn as_element(&self) -> Element {
+        Element::Binstr(self.to_vec())
+    }
+}
+
+impl AsQueryItem for &'static str {
+    fn push_into_query(&self, query: &mut Query) {
+        query.push(*self);
+    }
+    fn as_element(&self) -> Element {
+        Element::String(self.to_string())
+    }
+}
+
+async fn persist_store<K: AsQueryItem, V: AsQueryItem>(
+    con: &mut Connection,
+    table_id: &str,
+    declaration: &str,
+    input: [(K, V); PERIST_TEST_SET_SIZE],
+) {
+    create_table_and_switch!(con, table_id, declaration);
+    for (key, value) in input {
+        let mut query = Query::from("set");
+        key.push_into_query(&mut query);
+        value.push_into_query(&mut query);
+        runeq!(con, query, Element::RespCode(RespCode::Okay))
+    }
+}
+
+async fn persist_load<K: AsQueryItem, V: AsQueryItem>(
+    con: &mut Connection,
+    table_id: &str,
+    input: [(K, V); PERIST_TEST_SET_SIZE],
+) {
+    switch_entity!(con, table_id);
+    for (key, value) in input {
+        let mut q = Query::from("get");
+        key.push_into_query(&mut q);
+        runeq!(con, q, value.as_element());
+    }
+    // now delete this table, freeing it up for the next suite run
+    switch_entity!(con, "default:default");
+    runeq!(
+        con,
+        query!("drop", "table", table_id),
+        Element::RespCode(RespCode::Okay)
+    );
+}
+
+const PERSIST_CFG_KEYMAP_BIN_BIN_TABLE: &str = "testsuite:persist_bin_bin_tbl";
+const PERSIST_DATA_KEYMAP_BIN_BIN_TABLE: [(&[u8], &[u8]); PERIST_TEST_SET_SIZE] = [
+    (b"mykey1\xF0\x90\x80", b"myval1\xF0\x90\x80"),
+    (b"mykey2\xF0\x90\x80", b"myval2\xF0\x90\x80"),
+    (b"mykey3\xF0\x90\x80", b"myval3\xF0\x90\x80"),
+    (b"mykey4\xF0\x90\x80", b"myval4\xF0\x90\x80"),
+];
+
+#[dbtest(skip_if_cfg = "persist-suite", norun = true)]
+async fn persist_store_keymap_bin_bin() {
+    persist_store(
+        &mut con,
+        PERSIST_CFG_KEYMAP_BIN_BIN_TABLE,
+        "keymap(binstr,binstr)",
+        PERSIST_DATA_KEYMAP_BIN_BIN_TABLE,
+    )
+    .await;
+}
+
+#[dbtest(run_if_cfg = "persist-suite", norun = true)]
+async fn persist_load_keymap_bin_bin() {
+    persist_load(
+        &mut con,
+        PERSIST_CFG_KEYMAP_BIN_BIN_TABLE,
+        PERSIST_DATA_KEYMAP_BIN_BIN_TABLE,
+    )
+    .await;
+}
+
+const PERSIST_CFG_KEYMAP_BIN_STR_TABLE: &str = "testsuite:persist_bin_str_tbl";
+const PERSIST_DATA_KEYMAP_BIN_STR_TABLE: [(&[u8], &str); PERIST_TEST_SET_SIZE] = [
+    (b"mykey1\xF0\x90\x80", "myval1"),
+    (b"mykey2\xF0\x90\x80", "myval2"),
+    (b"mykey3\xF0\x90\x80", "myval3"),
+    (b"mykey4\xF0\x90\x80", "myval4"),
+];
+
+#[dbtest(skip_if_cfg = "persist-suite", norun = true)]
+async fn persist_store_keymap_bin_str() {
+    persist_store(
+        &mut con,
+        PERSIST_CFG_KEYMAP_BIN_STR_TABLE,
+        "keymap(binstr,str)",
+        PERSIST_DATA_KEYMAP_BIN_STR_TABLE,
+    )
+    .await;
+}
+
+#[dbtest(run_if_cfg = "persist-suite", norun = true)]
+async fn persist_load_keymap_bin_str() {
+    persist_load(
+        &mut con,
+        PERSIST_CFG_KEYMAP_BIN_STR_TABLE,
+        PERSIST_DATA_KEYMAP_BIN_STR_TABLE,
+    )
+    .await;
+}
+
+const PERSIST_CFG_KEYMAP_STR_STR_TABLE: &str = "testsuite:persist_str_str_tbl";
+const PERSIST_DATA_KEYMAP_STR_STR_TABLE: [(&str, &str); PERIST_TEST_SET_SIZE] = [
+    ("mykey1", "myval1"),
+    ("mykey2", "myval2"),
+    ("mykey3", "myval3"),
+    ("mykey4", "myval4"),
+];
+
+#[dbtest(skip_if_cfg = "persist-suite", norun = true)]
+async fn persist_store_keymap_str_str() {
+    persist_store(
+        &mut con,
+        PERSIST_CFG_KEYMAP_STR_STR_TABLE,
+        "keymap(str,str)",
+        PERSIST_DATA_KEYMAP_STR_STR_TABLE,
+    )
+    .await;
+}
+
+#[dbtest(run_if_cfg = "persist-suite", norun = true)]
+async fn persist_load_keymap_str_str() {
+    persist_load(
+        &mut con,
+        PERSIST_CFG_KEYMAP_STR_STR_TABLE,
+        PERSIST_DATA_KEYMAP_STR_STR_TABLE,
+    )
+    .await;
+}
+
+const PERSIST_CFG_KEYMAP_STR_BIN_TABLE: &str = "testsuite:persist_str_bin_tbl";
+const PERSIST_DATA_KEYMAP_STR_BIN_TABLE: [(&str, &[u8]); PERIST_TEST_SET_SIZE] = [
+    ("mykey1", b"myval1\xF0\x90\x80"),
+    ("mykey2", b"myval2\xF0\x90\x80"),
+    ("mykey3", b"myval3\xF0\x90\x80"),
+    ("mykey4", b"myval4\xF0\x90\x80"),
+];
+
+#[dbtest(skip_if_cfg = "persist-suite", norun = true)]
+async fn persist_store_keymap_str_bin() {
+    persist_store(
+        &mut con,
+        PERSIST_CFG_KEYMAP_STR_BIN_TABLE,
+        "keymap(str,binstr)",
+        PERSIST_DATA_KEYMAP_STR_BIN_TABLE,
+    )
+    .await;
+}
+
+#[dbtest(run_if_cfg = "persist-suite", norun = true)]
+async fn persist_load_keymap_str_bin() {
+    persist_load(
+        &mut con,
+        PERSIST_CFG_KEYMAP_STR_BIN_TABLE,
+        PERSIST_DATA_KEYMAP_STR_BIN_TABLE,
+    )
+    .await;
+}

sky-macros/src/dbtest_fn.rs

@@ -40,6 +40,7 @@ pub struct DBTestFunctionConfig {
     testuser: bool,
     rootuser: bool,
     norun: bool,
+    skip_cfg: quote::__private::TokenStream,
 }
 
 impl DBTestFunctionConfig {
@@ -53,6 +54,7 @@ impl DBTestFunctionConfig {
             testuser: false,
             rootuser: false,
             norun: false,
+            skip_cfg: quote! {},
         }
     }
     pub fn get_connection_tokens(&self) -> impl quote::ToTokens {
@@ -90,6 +92,9 @@ impl DBTestFunctionConfig {
             ).await.unwrap()
         }
     }
+    pub fn get_skip_cfg_tokens(&self) -> &impl quote::ToTokens {
+        &self.skip_cfg
+    }
     pub fn get_login_tokens(&self) -> Option<impl quote::ToTokens> {
         let Self {
             login,
@@ -175,6 +180,18 @@ pub fn parse_dbtest_func_args(
             fcfg.rootuser = util::parse_bool(lit, span, "auth_testuser").expect("Expected a bool")
         }
         "norun" => fcfg.norun = util::parse_bool(lit, span, "norun").expect("Expected a bool"),
+        "run_if_cfg" => {
+            let cfg_name = util::parse_string(lit, span, "run_if_cfg").expect("Expected a string");
+            fcfg.skip_cfg = quote! {
+                #[cfg_attr(not(feature = #cfg_name), ignore)]
+            };
+        }
+        "skip_if_cfg" => {
+            let cfg_name = util::parse_string(lit, span, "run_if_cfg").expect("Expected a string");
+            fcfg.skip_cfg = quote! {
+                #[cfg_attr(feature = #cfg_name, ignore)]
+            };
+        }
         x => panic!("unknown attribute {x} specified"),
     }
 }
@@ -302,7 +319,9 @@ fn generate_dbtest(
             }
         };
     }
+    let skip_cfg = fcfg.get_skip_cfg_tokens();
     let result = quote! {
+        #skip_cfg
         #header
         #(#attrs)*
         #vis #sig {