Add enc/dec for space

server/src/engine/core/mod.rs

@@ -28,7 +28,7 @@ mod dml;
 mod index;
 pub(in crate::engine) mod model;
 pub(in crate::engine) mod query_meta;
-mod space;
+pub mod space;
 mod util;
 // test
 #[cfg(test)]

server/src/engine/core/model/alt.rs

@@ -81,7 +81,7 @@ fn no_field(mr: &IWModel, new: &str) -> bool {
 
 fn check_nullable(props: &mut HashMap<Box<str>, DictEntryGeneric>) -> DatabaseResult<bool> {
     match props.remove("nullable") {
-        Some(DictEntryGeneric::Lit(b)) if b.kind() == TagClass::Bool => Ok(b.bool()),
+        Some(DictEntryGeneric::Data(b)) if b.kind() == TagClass::Bool => Ok(b.bool()),
         Some(_) => Err(DatabaseError::DdlModelAlterBadProperty),
         None => Ok(false),
     }

server/src/engine/core/space.rs

@@ -27,7 +27,7 @@
 use {
     crate::engine::{
         core::{model::Model, RWLIdx},
-        data::{dict, uuid::Uuid, DictEntryGeneric, MetaDict},
+        data::{dict, uuid::Uuid, DictEntryGeneric, DictGeneric},
         error::{DatabaseError, DatabaseResult},
         idx::{IndexST, STIndex},
         ql::ddl::{alt::AlterSpace, crt::CreateSpace, drop::DropSpace},
@@ -46,16 +46,19 @@ pub struct Space {
 #[derive(Debug, Default)]
 /// Space metadata
 pub struct SpaceMeta {
-    pub(super) env: RwLock<MetaDict>,
+    pub(super) env: RwLock<DictGeneric>,
 }
 
 impl SpaceMeta {
     pub const KEY_ENV: &str = "env";
-    pub fn with_env(env: MetaDict) -> Self {
+    pub fn with_env(env: DictGeneric) -> Self {
         Self {
             env: RWLIdx::new(env),
         }
     }
+    pub fn env(&self) -> &RwLock<DictGeneric> {
+        &self.env
+    }
 }
 
 #[derive(Debug)]
@@ -92,6 +95,9 @@ impl Space {
     pub(super) fn models(&self) -> &RWLIdx<Box<str>, Model> {
         &self.mns
     }
+    pub fn metadata(&self) -> &SpaceMeta {
+        &self.meta
+    }
     pub fn with_model<T>(
         &self,
         model: &str,
@@ -103,6 +109,9 @@ impl Space {
         };
         f(model)
     }
+    pub(crate) fn new_restore_empty(meta: SpaceMeta, uuid: Uuid) -> Space {
+        Self::new_with_uuid(Default::default(), meta, uuid)
+    }
 }
 
 impl Space {
@@ -139,7 +148,7 @@ impl Space {
         // check env
         let env = match props.remove(SpaceMeta::KEY_ENV) {
             Some(DictEntryGeneric::Map(m)) if props.is_empty() => m,
-            Some(DictEntryGeneric::Lit(l)) if l.is_null() => IndexST::default(),
+            Some(DictEntryGeneric::Data(l)) if l.is_null() => IndexST::default(),
             None if props.is_empty() => IndexST::default(),
             _ => {
                 return Err(DatabaseError::DdlSpaceBadProperty);
@@ -185,7 +194,7 @@ impl Space {
                         return Err(DatabaseError::DdlSpaceBadProperty);
                     }
                 }
-                Some(DictEntryGeneric::Lit(l)) if updated_props.is_empty() & l.is_null() => {
+                Some(DictEntryGeneric::Data(l)) if updated_props.is_empty() & l.is_null() => {
                     space_env.clear()
                 }
                 None => {}

server/src/engine/data/dict.rs

@@ -35,73 +35,51 @@ use {
     std::collections::HashMap,
 };
 
-/*
-    dict kinds: one is from a generic parse while the other one is the stored metadata
-    - MetaDict -> only non-null values
-    - DictGeneric -> null allowed
-*/
-
-/// A metadata dictionary
-pub type MetaDict = HashMap<Box<str>, MetaDictEntry>;
 /// A generic dictionary built from scratch from syntactical elements
 pub type DictGeneric = HashMap<Box<str>, DictEntryGeneric>;
 
 #[derive(Debug, PartialEq)]
+#[cfg_attr(test, derive(Clone))]
 /// A generic dict entry: either a literal or a recursive dictionary
 pub enum DictEntryGeneric {
     /// A literal
-    Lit(Datacell),
+    Data(Datacell),
     /// A map
     Map(DictGeneric),
 }
 
-#[derive(Debug, PartialEq)]
-#[cfg_attr(test, derive(Clone))]
-/// A metadata dictionary
-pub enum MetaDictEntry {
-    Data(Datacell),
-    Map(MetaDict),
-}
-
 /*
     patchsets
 */
 
 #[derive(Debug, PartialEq, Default)]
-struct MetaDictPatch(HashMap<Box<str>, Option<MetaDictPatchEntry>>);
+struct DictGenericPatch(HashMap<Box<str>, Option<DictGenericPatchEntry>>);
 #[derive(Debug, PartialEq)]
-enum MetaDictPatchEntry {
+enum DictGenericPatchEntry {
     Data(Datacell),
-    Map(MetaDictPatch),
+    Map(DictGenericPatch),
 }
 
-/// Recursively flatten a [`DictGeneric`] into a [`MetaDict`]
-pub fn rflatten_metadata(new: DictGeneric) -> MetaDict {
-    let mut empty = MetaDict::new();
-    _rflatten_metadata(new, &mut empty);
-    empty
+/// Accepts a dict with possible null values, and removes those null values
+pub fn rflatten_metadata(mut new: DictGeneric) -> DictGeneric {
+    _rflatten_metadata(&mut new);
+    new
 }
 
-fn _rflatten_metadata(new: DictGeneric, empty: &mut MetaDict) {
-    for (key, val) in new {
-        match val {
-            DictEntryGeneric::Lit(l) if l.is_init() => {
-                empty.insert(key, MetaDictEntry::Data(l));
-            }
-            DictEntryGeneric::Lit(_) => {}
-            DictEntryGeneric::Map(m) => {
-                let mut rnew = MetaDict::new();
-                _rflatten_metadata(m, &mut rnew);
-                empty.insert(key, MetaDictEntry::Map(rnew));
-            }
+fn _rflatten_metadata(new: &mut DictGeneric) {
+    new.retain(|_, v| match v {
+        DictEntryGeneric::Data(d) => d.is_init(),
+        DictEntryGeneric::Map(m) => {
+            _rflatten_metadata(m);
+            true
         }
-    }
+    });
 }
 
-/// Recursively merge a [`DictGeneric`] into a [`MetaDict`] with the use of an intermediary
+/// Recursively merge a [`DictGeneric`] into a [`DictGeneric`] with the use of an intermediary
 /// patchset to avoid inconsistent states
-pub fn rmerge_metadata(current: &mut MetaDict, new: DictGeneric) -> bool {
-    let mut patch = MetaDictPatch::default();
+pub fn rmerge_metadata(current: &mut DictGeneric, new: DictGeneric) -> bool {
+    let mut patch = DictGenericPatch::default();
     let current_ref = current as &_;
     let r = rmerge_metadata_prepare_patch(current_ref, new, &mut patch);
     if r {
@@ -110,15 +88,15 @@ pub fn rmerge_metadata(current: &mut MetaDict, new: DictGeneric) -> bool {
     r
 }
 
-fn merge_data_with_patch(current: &mut MetaDict, patch: MetaDictPatch) {
+fn merge_data_with_patch(current: &mut DictGeneric, patch: DictGenericPatch) {
     for (key, patch) in patch.0 {
         match patch {
-            Some(MetaDictPatchEntry::Data(d)) => {
-                current.st_upsert(key, MetaDictEntry::Data(d));
+            Some(DictGenericPatchEntry::Data(d)) => {
+                current.st_upsert(key, DictEntryGeneric::Data(d));
             }
-            Some(MetaDictPatchEntry::Map(m)) => match current.get_mut(&key) {
+            Some(DictGenericPatchEntry::Map(m)) => match current.get_mut(&key) {
                 Some(current_recursive) => match current_recursive {
-                    MetaDictEntry::Map(current_m) => {
+                    DictEntryGeneric::Map(current_m) => {
                         merge_data_with_patch(current_m, m);
                     }
                     _ => {
@@ -127,7 +105,7 @@ fn merge_data_with_patch(current: &mut MetaDict, patch: MetaDictPatch) {
                     }
                 },
                 None => {
-                    let mut new = MetaDict::new();
+                    let mut new = DictGeneric::new();
                     merge_data_with_patch(&mut new, m);
                 }
             },
@@ -139,9 +117,9 @@ fn merge_data_with_patch(current: &mut MetaDict, patch: MetaDictPatch) {
 }
 
 fn rmerge_metadata_prepare_patch(
-    current: &MetaDict,
+    current: &DictGeneric,
     new: DictGeneric,
-    patch: &mut MetaDictPatch,
+    patch: &mut DictGenericPatch,
 ) -> bool {
     let mut new = new.into_iter();
     let mut okay = true;
@@ -149,25 +127,25 @@ fn rmerge_metadata_prepare_patch(
         let (key, new_entry) = new.next().unwrap();
         match (current.get(&key), new_entry) {
             // non-null -> non-null: merge flatten update
-            (Some(MetaDictEntry::Data(this_data)), DictEntryGeneric::Lit(new_data))
+            (Some(DictEntryGeneric::Data(this_data)), DictEntryGeneric::Data(new_data))
                 if new_data.is_init() =>
             {
                 if this_data.kind() == new_data.kind() {
                     patch
                         .0
-                        .insert(key, Some(MetaDictPatchEntry::Data(new_data)));
+                        .insert(key, Some(DictGenericPatchEntry::Data(new_data)));
                 } else {
                     okay = false;
                 }
             }
-            (Some(MetaDictEntry::Data(_)), DictEntryGeneric::Map(_)) => {
+            (Some(DictEntryGeneric::Data(_)), DictEntryGeneric::Map(_)) => {
                 okay = false;
             }
             (
-                Some(MetaDictEntry::Map(this_recursive_data)),
+                Some(DictEntryGeneric::Map(this_recursive_data)),
                 DictEntryGeneric::Map(new_recursive_data),
             ) => {
-                let mut this_patch = MetaDictPatch::default();
+                let mut this_patch = DictGenericPatch::default();
                 let _okay = rmerge_metadata_prepare_patch(
                     this_recursive_data,
                     new_recursive_data,
@@ -175,26 +153,26 @@ fn rmerge_metadata_prepare_patch(
                 );
                 patch
                     .0
-                    .insert(key, Some(MetaDictPatchEntry::Map(this_patch)));
+                    .insert(key, Some(DictGenericPatchEntry::Map(this_patch)));
                 okay &= _okay;
             }
             // null -> non-null: flatten insert
-            (None, DictEntryGeneric::Lit(l)) if l.is_init() => {
-                let _ = patch.0.insert(key, Some(MetaDictPatchEntry::Data(l)));
+            (None, DictEntryGeneric::Data(l)) if l.is_init() => {
+                let _ = patch.0.insert(key, Some(DictGenericPatchEntry::Data(l)));
             }
             (None, DictEntryGeneric::Map(m)) => {
-                let mut this_patch = MetaDictPatch::default();
+                let mut this_patch = DictGenericPatch::default();
                 okay &= rmerge_metadata_prepare_patch(&into_dict!(), m, &mut this_patch);
                 let _ = patch
                     .0
-                    .insert(key, Some(MetaDictPatchEntry::Map(this_patch)));
+                    .insert(key, Some(DictGenericPatchEntry::Map(this_patch)));
             }
             // non-null -> null: remove
-            (Some(_), DictEntryGeneric::Lit(l)) => {
+            (Some(_), DictEntryGeneric::Data(l)) => {
                 debug_assert!(l.is_null());
                 patch.0.insert(key, None);
             }
-            (None, DictEntryGeneric::Lit(l)) => {
+            (None, DictEntryGeneric::Data(l)) => {
                 debug_assert!(l.is_null());
                 // ignore
             }
@@ -209,26 +187,19 @@ fn rmerge_metadata_prepare_patch(
 
 impl<'a> From<LitIR<'a>> for DictEntryGeneric {
     fn from(l: LitIR<'a>) -> Self {
-        Self::Lit(Datacell::from(l))
+        Self::Data(Datacell::from(l))
     }
 }
 
 impl<'a> From<Lit<'a>> for DictEntryGeneric {
     fn from(value: Lit<'a>) -> Self {
-        Self::Lit(Datacell::from(value))
+        Self::Data(Datacell::from(value))
     }
 }
 
 direct_from! {
     DictEntryGeneric => {
-        Datacell as Lit,
-        DictGeneric as Map,
-    }
-}
-
-direct_from! {
-    MetaDictEntry => {
         Datacell as Data,
-        MetaDict as Map,
+        DictGeneric as Map,
     }
 }

server/src/engine/data/mod.rs

@@ -36,4 +36,4 @@ pub mod uuid;
 #[cfg(test)]
 mod tests;
 
-pub use dict::{DictEntryGeneric, DictGeneric, MetaDict};
+pub use dict::{DictEntryGeneric, DictGeneric};

server/src/engine/data/tests/md_dict_tests.rs

@@ -26,16 +26,16 @@
 
 use crate::engine::data::{
     cell::Datacell,
-    dict::{self, DictEntryGeneric, DictGeneric, MetaDict, MetaDictEntry},
+    dict::{self, DictEntryGeneric, DictGeneric},
 };
 
 #[test]
 fn t_simple_flatten() {
     let generic_dict: DictGeneric = into_dict! {
-        "a_valid_key" => DictEntryGeneric::Lit(100u64.into()),
+        "a_valid_key" => DictEntryGeneric::Data(100u64.into()),
         "a_null_key" => Datacell::null(),
     };
-    let expected: MetaDict = into_dict!(
+    let expected: DictGeneric = into_dict!(
         "a_valid_key" => Datacell::new_uint(100)
     );
     let ret = dict::rflatten_metadata(generic_dict);
@@ -44,7 +44,7 @@ fn t_simple_flatten() {
 
 #[test]
 fn t_simple_patch() {
-    let mut current: MetaDict = into_dict! {
+    let mut current: DictGeneric = into_dict! {
         "a" => Datacell::new_uint(2),
         "b" => Datacell::new_uint(3),
         "z" => Datacell::new_sint(-100),
@@ -54,7 +54,7 @@ fn t_simple_patch() {
         "b" => Datacell::new_uint(2),
         "z" => Datacell::null(),
     };
-    let expected: MetaDict = into_dict! {
+    let expected: DictGeneric = into_dict! {
         "a" => Datacell::new_uint(1),
         "b" => Datacell::new_uint(2),
     };
@@ -64,7 +64,7 @@ fn t_simple_patch() {
 
 #[test]
 fn t_bad_patch() {
-    let mut current: MetaDict = into_dict! {
+    let mut current: DictGeneric = into_dict! {
         "a" => Datacell::new_uint(2),
         "b" => Datacell::new_uint(3),
         "z" => Datacell::new_sint(-100),
@@ -81,15 +81,15 @@ fn t_bad_patch() {
 
 #[test]
 fn patch_null_out_dict() {
-    let mut current: MetaDict = into_dict! {
+    let mut current: DictGeneric = into_dict! {
         "a" => Datacell::new_uint(2),
         "b" => Datacell::new_uint(3),
-        "z" => MetaDictEntry::Map(into_dict!(
+        "z" => DictEntryGeneric::Map(into_dict!(
             "c" => Datacell::new_uint(1),
             "d" => Datacell::new_uint(2)
         )),
     };
-    let expected: MetaDict = into_dict! {
+    let expected: DictGeneric = into_dict! {
         "a" => Datacell::new_uint(2),
         "b" => Datacell::new_uint(3),
     };

server/src/engine/ql/ddl/syn.rs

@@ -179,7 +179,7 @@ where
                             // UNSAFE(@ohsayan): we only switch to this when we've already read in a key
                             key.take().as_str().into()
                         },
-                        DictEntryGeneric::Lit(Datacell::null()),
+                        DictEntryGeneric::Data(Datacell::null()),
                     )
                     .is_none(),
                 );

server/src/engine/ql/tests.rs

@@ -81,7 +81,7 @@ pub trait NullableDictEntry {
 
 impl NullableDictEntry for Null {
     fn data(self) -> crate::engine::data::DictEntryGeneric {
-        crate::engine::data::DictEntryGeneric::Lit(Datacell::null())
+        crate::engine::data::DictEntryGeneric::Data(Datacell::null())
     }
 }
 

server/src/engine/storage/v1/inf/map.rs

@@ -211,7 +211,7 @@ impl PersistMapSpec for GenericDictSpec {
                 buf.extend(key.as_bytes());
                 enc_dict_into_buffer::<Self>(buf, map);
             }
-            DictEntryGeneric::Lit(dc) => {
+            DictEntryGeneric::Data(dc) => {
                 buf.push(
                     PersistDictEntryDscr::translate_from_class(dc.tag().tag_class()).value_u8()
                         * (!dc.is_null() as u8),
@@ -257,13 +257,13 @@ impl PersistMapSpec for GenericDictSpec {
             dg_top_element: bool,
         ) -> Option<DictEntryGeneric> {
             let r = match dscr {
-                PersistDictEntryDscr::Null => DictEntryGeneric::Lit(Datacell::null()),
+                PersistDictEntryDscr::Null => DictEntryGeneric::Data(Datacell::null()),
                 PersistDictEntryDscr::Bool => {
-                    DictEntryGeneric::Lit(Datacell::new_bool(scanner.next_byte() == 1))
+                    DictEntryGeneric::Data(Datacell::new_bool(scanner.next_byte() == 1))
                 }
                 PersistDictEntryDscr::UnsignedInt
                 | PersistDictEntryDscr::SignedInt
-                | PersistDictEntryDscr::Float => DictEntryGeneric::Lit(Datacell::new_qw(
+                | PersistDictEntryDscr::Float => DictEntryGeneric::Data(Datacell::new_qw(
                     scanner.next_u64_le(),
                     CUTag::new(
                         dscr.into_class(),
@@ -281,7 +281,7 @@ impl PersistMapSpec for GenericDictSpec {
                         return None;
                     }
                     let slc = scanner.next_chunk_variable(slc_len);
-                    DictEntryGeneric::Lit(if dscr == PersistDictEntryDscr::Str {
+                    DictEntryGeneric::Data(if dscr == PersistDictEntryDscr::Str {
                         if core::str::from_utf8(slc).is_err() {
                             return None;
                         }
@@ -306,14 +306,14 @@ impl PersistMapSpec for GenericDictSpec {
                                 PersistDictEntryDscr::from_raw(dscr),
                                 false,
                             ) {
-                                Some(DictEntryGeneric::Lit(l)) => l,
+                                Some(DictEntryGeneric::Data(l)) => l,
                                 None => return None,
                                 _ => unreachable!("found top-level dict item in datacell"),
                             },
                         );
                     }
                     if v.len() == list_len {
-                        DictEntryGeneric::Lit(Datacell::new_list(v))
+                        DictEntryGeneric::Data(Datacell::new_list(v))
                     } else {
                         return None;
                     }

server/src/engine/storage/v1/inf/mod.rs

@@ -74,7 +74,7 @@ impl PersistDictEntryDscr {
     pub fn new_from_dict_gen_entry(e: &DictEntryGeneric) -> Self {
         match e {
             DictEntryGeneric::Map(_) => Self::Dict,
-            DictEntryGeneric::Lit(dc) => Self::translate_from_class(dc.tag().tag_class()),
+            DictEntryGeneric::Data(dc) => Self::translate_from_class(dc.tag().tag_class()),
         }
     }
     /// The data in question is null (well, can we call that data afterall?)

server/src/engine/storage/v1/inf/obj.rs

@@ -28,7 +28,10 @@ use {
     super::{dec_md, map::FieldMapSpec, PersistObjectHlIO, PersistObjectMD, SimpleSizeMD, VecU8},
     crate::{
         engine::{
-            core::model::{Field, Layer, Model},
+            core::{
+                model::{Field, Layer, Model},
+                space::{Space, SpaceMeta},
+            },
             data::{
                 tag::{DataTag, FullTag, TagClass, TagSelector},
                 uuid::Uuid,
@@ -274,3 +277,52 @@ impl PersistObjectHlIO for ModelLayout {
         ))
     }
 }
+
+pub struct SpaceLayout;
+pub struct SpaceLayoutMD {
+    uuid: Uuid,
+}
+
+impl SpaceLayoutMD {
+    pub fn new(uuid: Uuid) -> Self {
+        Self { uuid }
+    }
+}
+
+impl PersistObjectMD for SpaceLayoutMD {
+    const MD_DEC_INFALLIBLE: bool = true;
+
+    fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool {
+        scanner.has_left(sizeof!(u128) + sizeof!(u64)) // u64 for props dict; we don't handle that directly
+    }
+    fn pretest_src_for_object_dec(&self, _: &BufferedScanner) -> bool {
+        true
+    }
+    unsafe fn dec_md_payload(scanner: &mut BufferedScanner) -> Option<Self> {
+        Some(Self::new(Uuid::from_bytes(scanner.next_chunk())))
+    }
+}
+
+impl PersistObjectHlIO for SpaceLayout {
+    const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool = false; // no need, since the MD only handles the UUID
+    type Type = Space;
+    type Metadata = SpaceLayoutMD;
+    fn pe_obj_hlio_enc(buf: &mut VecU8, v: &Self::Type) {
+        buf.extend(v.get_uuid().to_le_bytes());
+        super::enc_into_buf::<super::map::PersistMapImpl<super::map::GenericDictSpec>>(
+            buf,
+            &v.metadata().env().read(),
+        );
+    }
+    unsafe fn pe_obj_hlio_dec(
+        scanner: &mut BufferedScanner,
+        md: Self::Metadata,
+    ) -> SDSSResult<Self::Type> {
+        let space_meta =
+            super::dec::<super::map::PersistMapImpl<super::map::GenericDictSpec>>(scanner)?;
+        Ok(Space::new_restore_empty(
+            SpaceMeta::with_env(space_meta),
+            md.uuid,
+        ))
+    }
+}

server/src/engine/storage/v1/inf/tests.rs

@@ -25,7 +25,10 @@
 */
 
 use crate::engine::{
-    core::model::{Field, Layer, Model},
+    core::{
+        model::{Field, Layer, Model},
+        space::{Space, SpaceMeta},
+    },
     data::{
         cell::Datacell,
         dict::{DictEntryGeneric, DictGeneric},
@@ -109,3 +112,13 @@ fn model() {
     let dec = super::dec::<super::obj::ModelLayout>(&mut scanner).unwrap();
     assert_eq!(model, dec);
 }
+
+#[test]
+fn space() {
+    let uuid = Uuid::new();
+    let space = Space::new_with_uuid(Default::default(), SpaceMeta::default(), uuid);
+    let enc = super::enc::<super::obj::SpaceLayout>(&space);
+    let mut scanner = BufferedScanner::new(&enc);
+    let dec = super::dec::<super::obj::SpaceLayout>(&mut scanner).unwrap();
+    assert_eq!(space, dec);
+}