From 1dbba0c06e61e53db9505a568f2aca0c7fc2f897 Mon Sep 17 00:00:00 2001 From: Sayan Nandan Date: Thu, 17 Mar 2022 11:31:27 +0530 Subject: [PATCH] Add persistence suite for keymap string table types Also fixed persistence bug with tables in the `system` keyspace --- Makefile | 16 +++ server/Cargo.toml | 1 + server/src/storage/v1/flush.rs | 10 +- server/src/tests/macros.rs | 35 ++++++ server/src/tests/mod.rs | 2 + server/src/tests/persist.rs | 204 +++++++++++++++++++++++++++++++++ sky-macros/src/dbtest_fn.rs | 19 +++ 7 files changed, 282 insertions(+), 5 deletions(-) create mode 100644 server/src/tests/persist.rs diff --git a/Makefile b/Makefile index 8c29398b..5f701d05 100644 --- a/Makefile +++ b/Makefile @@ -141,6 +141,22 @@ test: .pre @echo "Waiting for server to shut down ..." @${STOP_SERVER} @sleep 10 + # now run persistence suite + @${SEP} + @echo "Running persistence suite ..." + @${SEP} + @mkdir -p server1 && cd server1 && ${START_SERVER} + @mkdir -p server2 && cd server2 && ${START_SERVER2} + @echo "Sleeping for 10 seconds to let the server start up ..." + @sleep 10 + @echo "Finished sleeping" + @${SEP} + @${SEP} + @echo "Running all persistence tests ..." + @${TEST} --features persist-suite + @echo "Waiting for server to shut down ..." + @${STOP_SERVER} + @sleep 10 @echo "Removing temporary files ..." @rm -rf .sky_pid server1 server2 .skytest_* @${SEP} diff --git a/server/Cargo.toml b/server/Cargo.toml index 77c5a62d..78140385 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -56,6 +56,7 @@ libc = "0.2.119" [features] nightly = [] +persist-suite = [] [package.metadata.deb] name = "skytable" diff --git a/server/src/storage/v1/flush.rs b/server/src/storage/v1/flush.rs index 795bdf54..6bad52c5 100644 --- a/server/src/storage/v1/flush.rs +++ b/server/src/storage/v1/flush.rs @@ -226,11 +226,11 @@ pub fn flush_full(target: T, store: &Memstore) -> IoResult<()> self::flush_keyspace_full(&target, keyspace.key(), keyspace.value().as_ref())?; } // flush system tables - // HACK(@ohsayan): DO NOT REORDER THIS. THE above loop will flush a PARTMAP once. But - // this has to be done again! The system keyspace in the above loop is a dummy one - // because it is located in a different field. So, we need to flush the actual list - // of tables - self::oneshot::flush_partmap(&target, &SYSTEM, &store.system)?; + // HACK(@ohsayan): DO NOT REORDER THIS. THE above loop will flush a PARTMAP and an empty + // keyspace once. But this has to be done again! The system keyspace in the above loop is a + // dummy one because it is located in a different field. So, we need to flush the actual + // tables + self::flush_keyspace_full(&target, &SYSTEM, &store.system)?; Ok(()) } diff --git a/server/src/tests/macros.rs b/server/src/tests/macros.rs index 0f07ee92..174e8eb0 100644 --- a/server/src/tests/macros.rs +++ b/server/src/tests/macros.rs @@ -39,6 +39,41 @@ macro_rules! setkeys { Element::UnsignedInt(count) ); }; + ($con:ident, $($key:expr => $value:expr),*) => { + let mut q = ::skytable::Query::new(); + q.push("MSET"); + let mut count = 0; + $( + q.push($key); + q.push($value); + count += 1; + )* + assert_eq!( + $con.run_query_raw(&q).await.unwrap(), + ::skytable::Element::UnsignedInt(count) + ); + }; +} + +macro_rules! switch_entity { + ($con:expr, $entity:expr) => { + runeq!( + $con, + ::skytable::query!("use", $entity), + ::skytable::Element::RespCode(::skytable::RespCode::Okay) + ) + }; +} + +macro_rules! create_table_and_switch { + ($con:expr, $table:expr, $decl:expr) => {{ + runeq!( + $con, + ::skytable::query!("create", "table", $table, $decl), + ::skytable::Element::RespCode(::skytable::RespCode::Okay) + ); + switch_entity!($con, $table); + }}; } macro_rules! push { diff --git a/server/src/tests/mod.rs b/server/src/tests/mod.rs index bcf0538a..f518f2a6 100644 --- a/server/src/tests/mod.rs +++ b/server/src/tests/mod.rs @@ -28,12 +28,14 @@ #[macro_use] mod macros; +#[cfg(not(feature = "persist-suite"))] mod auth; mod ddl_tests; mod inspect_tests; mod kvengine; mod kvengine_encoding; mod kvengine_list; +mod persist; mod pipeline; mod tls { diff --git a/server/src/tests/persist.rs b/server/src/tests/persist.rs new file mode 100644 index 00000000..32ae6ea0 --- /dev/null +++ b/server/src/tests/persist.rs @@ -0,0 +1,204 @@ +/* + * Created on Thu Mar 17 2022 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2022, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use sky_macros::dbtest_func as dbtest; +use skytable::{aio::Connection, query, types::RawString, Element, Query, RespCode}; + +const PERIST_TEST_SET_SIZE: usize = 4; + +trait AsQueryItem { + fn push_into_query(&self, query: &mut Query); + fn as_element(&self) -> Element; +} + +impl AsQueryItem for &'static [u8] { + fn push_into_query(&self, query: &mut Query) { + query.push(RawString::from(self.to_vec())); + } + fn as_element(&self) -> Element { + Element::Binstr(self.to_vec()) + } +} + +impl AsQueryItem for &'static str { + fn push_into_query(&self, query: &mut Query) { + query.push(*self); + } + fn as_element(&self) -> Element { + Element::String(self.to_string()) + } +} + +async fn persist_store( + con: &mut Connection, + table_id: &str, + declaration: &str, + input: [(K, V); PERIST_TEST_SET_SIZE], +) { + create_table_and_switch!(con, table_id, declaration); + for (key, value) in input { + let mut query = Query::from("set"); + key.push_into_query(&mut query); + value.push_into_query(&mut query); + runeq!(con, query, Element::RespCode(RespCode::Okay)) + } +} + +async fn persist_load( + con: &mut Connection, + table_id: &str, + input: [(K, V); PERIST_TEST_SET_SIZE], +) { + switch_entity!(con, table_id); + for (key, value) in input { + let mut q = Query::from("get"); + key.push_into_query(&mut q); + runeq!(con, q, value.as_element()); + } + // now delete this table, freeing it up for the next suite run + switch_entity!(con, "default:default"); + runeq!( + con, + query!("drop", "table", table_id), + Element::RespCode(RespCode::Okay) + ); +} + +const PERSIST_CFG_KEYMAP_BIN_BIN_TABLE: &str = "testsuite:persist_bin_bin_tbl"; +const PERSIST_DATA_KEYMAP_BIN_BIN_TABLE: [(&[u8], &[u8]); PERIST_TEST_SET_SIZE] = [ + (b"mykey1\xF0\x90\x80", b"myval1\xF0\x90\x80"), + (b"mykey2\xF0\x90\x80", b"myval2\xF0\x90\x80"), + (b"mykey3\xF0\x90\x80", b"myval3\xF0\x90\x80"), + (b"mykey4\xF0\x90\x80", b"myval4\xF0\x90\x80"), +]; + +#[dbtest(skip_if_cfg = "persist-suite", norun = true)] +async fn persist_store_keymap_bin_bin() { + persist_store( + &mut con, + PERSIST_CFG_KEYMAP_BIN_BIN_TABLE, + "keymap(binstr,binstr)", + PERSIST_DATA_KEYMAP_BIN_BIN_TABLE, + ) + .await; +} + +#[dbtest(run_if_cfg = "persist-suite", norun = true)] +async fn persist_load_keymap_bin_bin() { + persist_load( + &mut con, + PERSIST_CFG_KEYMAP_BIN_BIN_TABLE, + PERSIST_DATA_KEYMAP_BIN_BIN_TABLE, + ) + .await; +} + +const PERSIST_CFG_KEYMAP_BIN_STR_TABLE: &str = "testsuite:persist_bin_str_tbl"; +const PERSIST_DATA_KEYMAP_BIN_STR_TABLE: [(&[u8], &str); PERIST_TEST_SET_SIZE] = [ + (b"mykey1\xF0\x90\x80", "myval1"), + (b"mykey2\xF0\x90\x80", "myval2"), + (b"mykey3\xF0\x90\x80", "myval3"), + (b"mykey4\xF0\x90\x80", "myval4"), +]; + +#[dbtest(skip_if_cfg = "persist-suite", norun = true)] +async fn persist_store_keymap_bin_str() { + persist_store( + &mut con, + PERSIST_CFG_KEYMAP_BIN_STR_TABLE, + "keymap(binstr,str)", + PERSIST_DATA_KEYMAP_BIN_STR_TABLE, + ) + .await; +} + +#[dbtest(run_if_cfg = "persist-suite", norun = true)] +async fn persist_load_keymap_bin_str() { + persist_load( + &mut con, + PERSIST_CFG_KEYMAP_BIN_STR_TABLE, + PERSIST_DATA_KEYMAP_BIN_STR_TABLE, + ) + .await; +} + +const PERSIST_CFG_KEYMAP_STR_STR_TABLE: &str = "testsuite:persist_str_str_tbl"; +const PERSIST_DATA_KEYMAP_STR_STR_TABLE: [(&str, &str); PERIST_TEST_SET_SIZE] = [ + ("mykey1", "myval1"), + ("mykey2", "myval2"), + ("mykey3", "myval3"), + ("mykey4", "myval4"), +]; + +#[dbtest(skip_if_cfg = "persist-suite", norun = true)] +async fn persist_store_keymap_str_str() { + persist_store( + &mut con, + PERSIST_CFG_KEYMAP_STR_STR_TABLE, + "keymap(str,str)", + PERSIST_DATA_KEYMAP_STR_STR_TABLE, + ) + .await; +} + +#[dbtest(run_if_cfg = "persist-suite", norun = true)] +async fn persist_load_keymap_str_str() { + persist_load( + &mut con, + PERSIST_CFG_KEYMAP_STR_STR_TABLE, + PERSIST_DATA_KEYMAP_STR_STR_TABLE, + ) + .await; +} + +const PERSIST_CFG_KEYMAP_STR_BIN_TABLE: &str = "testsuite:persist_str_bin_tbl"; +const PERSIST_DATA_KEYMAP_STR_BIN_TABLE: [(&str, &[u8]); PERIST_TEST_SET_SIZE] = [ + ("mykey1", b"myval1\xF0\x90\x80"), + ("mykey2", b"myval2\xF0\x90\x80"), + ("mykey3", b"myval3\xF0\x90\x80"), + ("mykey4", b"myval4\xF0\x90\x80"), +]; + +#[dbtest(skip_if_cfg = "persist-suite", norun = true)] +async fn persist_store_keymap_str_bin() { + persist_store( + &mut con, + PERSIST_CFG_KEYMAP_STR_BIN_TABLE, + "keymap(str,binstr)", + PERSIST_DATA_KEYMAP_STR_BIN_TABLE, + ) + .await; +} + +#[dbtest(run_if_cfg = "persist-suite", norun = true)] +async fn persist_load_keymap_str_bin() { + persist_load( + &mut con, + PERSIST_CFG_KEYMAP_STR_BIN_TABLE, + PERSIST_DATA_KEYMAP_STR_BIN_TABLE, + ) + .await; +} diff --git a/sky-macros/src/dbtest_fn.rs b/sky-macros/src/dbtest_fn.rs index b505836a..f0827371 100644 --- a/sky-macros/src/dbtest_fn.rs +++ b/sky-macros/src/dbtest_fn.rs @@ -40,6 +40,7 @@ pub struct DBTestFunctionConfig { testuser: bool, rootuser: bool, norun: bool, + skip_cfg: quote::__private::TokenStream, } impl DBTestFunctionConfig { @@ -53,6 +54,7 @@ impl DBTestFunctionConfig { testuser: false, rootuser: false, norun: false, + skip_cfg: quote! {}, } } pub fn get_connection_tokens(&self) -> impl quote::ToTokens { @@ -90,6 +92,9 @@ impl DBTestFunctionConfig { ).await.unwrap() } } + pub fn get_skip_cfg_tokens(&self) -> &impl quote::ToTokens { + &self.skip_cfg + } pub fn get_login_tokens(&self) -> Option { let Self { login, @@ -175,6 +180,18 @@ pub fn parse_dbtest_func_args( fcfg.rootuser = util::parse_bool(lit, span, "auth_testuser").expect("Expected a bool") } "norun" => fcfg.norun = util::parse_bool(lit, span, "norun").expect("Expected a bool"), + "run_if_cfg" => { + let cfg_name = util::parse_string(lit, span, "run_if_cfg").expect("Expected a string"); + fcfg.skip_cfg = quote! { + #[cfg_attr(not(feature = #cfg_name), ignore)] + }; + } + "skip_if_cfg" => { + let cfg_name = util::parse_string(lit, span, "run_if_cfg").expect("Expected a string"); + fcfg.skip_cfg = quote! { + #[cfg_attr(feature = #cfg_name, ignore)] + }; + } x => panic!("unknown attribute {x} specified"), } } @@ -302,7 +319,9 @@ fn generate_dbtest( } }; } + let skip_cfg = fcfg.get_skip_cfg_tokens(); let result = quote! { + #skip_cfg #header #(#attrs)* #vis #sig {