Unlink MD traits from map enc/dec impls

server/src/engine/storage/v1/inf/map.rs

@@ -26,8 +26,7 @@
 
 use {
     super::{
-        dec_md, obj::FieldMD, PersistDictEntryDscr, PersistMapSpec, PersistObjectHlIO,
-        PersistObjectMD, VecU8, VoidMetadata,
+        md::VoidMetadata, obj::FieldMD, PersistDictEntryDscr, PersistMapSpec, PersistObject, VecU8,
     },
     crate::{
         engine::{
@@ -50,7 +49,7 @@ use {
 /// This is more of a lazy hack than anything sensible. Just implement a spec and then use this wrapper for any enc/dec operations
 pub struct PersistMapImpl<M: PersistMapSpec>(PhantomData<M>);
 
-impl<M: PersistMapSpec> PersistObjectHlIO for PersistMapImpl<M>
+impl<M: PersistMapSpec> PersistObject for PersistMapImpl<M>
 where
     M::MapType: STIndex<M::Key, M::Value>,
 {
@@ -87,7 +86,7 @@ pub fn dec_dict<PM: PersistMapSpec>(scanner: &mut BufferedScanner) -> SDSSResult
 where
     PM::MapType: STIndex<PM::Key, PM::Value>,
 {
-    if !(PM::meta_dec_collection_pretest(scanner) & scanner.has_left(sizeof!(u64))) {
+    if !(PM::pretest_collection(scanner) & scanner.has_left(sizeof!(u64))) {
         return Err(SDSSError::InternalDecodeStructureCorrupted);
     }
     let size = unsafe {
@@ -95,12 +94,21 @@ where
         scanner.next_u64_le() as usize
     };
     let mut dict = PM::MapType::idx_init_cap(size);
-    while PM::meta_dec_entry_pretest(scanner) & (dict.st_len() != size) {
+    while PM::pretest_entry_metadata(scanner) & (dict.st_len() != size) {
         let md = unsafe {
             // pretest
-            dec_md::<PM::EntryMD, true>(scanner)?
+            match PM::entry_md_dec(scanner) {
+                Some(md) => md,
+                None => {
+                    if PM::ENTRYMETA_DEC_INFALLIBLE {
+                        impossible!()
+                    } else {
+                        return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
+                    }
+                }
+            }
         };
-        if PM::META_VERIFY_BEFORE_DEC && !md.pretest_src_for_object_dec(scanner) {
+        if PM::META_VERIFY_BEFORE_DEC && !PM::pretest_entry_data(scanner, &md) {
             return Err(SDSSError::InternalDecodeStructureCorrupted);
         }
         let key;
@@ -164,21 +172,6 @@ impl GenericDictEntryMD {
     }
 }
 
-impl PersistObjectMD for GenericDictEntryMD {
-    const MD_DEC_INFALLIBLE: bool = true;
-    fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool {
-        scanner.has_left(sizeof!(u64, 2) + 1)
-    }
-    unsafe fn dec_md_payload(scanner: &mut BufferedScanner) -> Option<Self> {
-        Some(Self::decode(scanner.next_chunk()))
-    }
-    fn pretest_src_for_object_dec(&self, scanner: &BufferedScanner) -> bool {
-        static EXPECT_ATLEAST: [u8; 4] = [0, 1, 8, 8]; // PAD to align
-        let lbound_rem = self.klen + EXPECT_ATLEAST[cmp::min(self.dscr, 3) as usize] as usize;
-        scanner.has_left(lbound_rem) & (self.dscr <= PersistDictEntryDscr::Dict.value_u8())
-    }
-}
-
 impl PersistMapSpec for GenericDictSpec {
     type MapIter<'a> = std::collections::hash_map::Iter<'a, Box<str>, DictEntryGeneric>;
     type MapType = DictGeneric;
@@ -188,15 +181,21 @@ impl PersistMapSpec for GenericDictSpec {
     const DEC_COUPLED: bool = false;
     const ENC_COUPLED: bool = true;
     const META_VERIFY_BEFORE_DEC: bool = true;
+    const ENTRYMETA_DEC_INFALLIBLE: bool = true;
     fn _get_iter<'a>(map: &'a Self::MapType) -> Self::MapIter<'a> {
         map.iter()
     }
-    fn meta_dec_collection_pretest(_: &BufferedScanner) -> bool {
+    fn pretest_collection(_: &BufferedScanner) -> bool {
         true
     }
-    fn meta_dec_entry_pretest(scanner: &BufferedScanner) -> bool {
+    fn pretest_entry_metadata(scanner: &BufferedScanner) -> bool {
         // we just need to see if we can decode the entry metadata
-        Self::EntryMD::pretest_src_for_metadata_dec(scanner)
+        scanner.has_left(9)
+    }
+    fn pretest_entry_data(scanner: &BufferedScanner, md: &Self::EntryMD) -> bool {
+        static EXPECT_ATLEAST: [u8; 4] = [0, 1, 8, 8]; // PAD to align
+        let lbound_rem = md.klen + EXPECT_ATLEAST[cmp::min(md.dscr, 3) as usize] as usize;
+        scanner.has_left(lbound_rem) & (md.dscr <= PersistDictEntryDscr::Dict.value_u8())
     }
     fn entry_md_enc(buf: &mut VecU8, key: &Self::Key, _: &Self::Value) {
         buf.extend(key.len().u64_bytes_le());
@@ -364,44 +363,27 @@ impl FieldMapEntryMD {
     }
 }
 
-impl PersistObjectMD for FieldMapEntryMD {
-    const MD_DEC_INFALLIBLE: bool = true;
-
-    fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool {
-        scanner.has_left(sizeof!(u64, 3) + 1)
-    }
-
-    fn pretest_src_for_object_dec(&self, scanner: &BufferedScanner) -> bool {
-        scanner.has_left(self.field_id_l as usize) // TODO(@ohsayan): we can enforce way more here such as atleast one field etc
-    }
-
-    unsafe fn dec_md_payload(scanner: &mut BufferedScanner) -> Option<Self> {
-        Some(Self::new(
-            u64::from_le_bytes(scanner.next_chunk()),
-            u64::from_le_bytes(scanner.next_chunk()),
-            u64::from_le_bytes(scanner.next_chunk()),
-            scanner.next_byte(),
-        ))
-    }
-}
-
 impl PersistMapSpec for FieldMapSpec {
     type MapIter<'a> = crate::engine::idx::IndexSTSeqDllIterOrdKV<'a, Box<str>, Field>;
     type MapType = IndexSTSeqCns<Self::Key, Self::Value>;
     type EntryMD = FieldMapEntryMD;
     type Key = Box<str>;
     type Value = Field;
+    const ENTRYMETA_DEC_INFALLIBLE: bool = true;
     const ENC_COUPLED: bool = false;
     const DEC_COUPLED: bool = false;
     const META_VERIFY_BEFORE_DEC: bool = true;
     fn _get_iter<'a>(m: &'a Self::MapType) -> Self::MapIter<'a> {
         m.stseq_ord_kv()
     }
-    fn meta_dec_collection_pretest(_: &BufferedScanner) -> bool {
+    fn pretest_collection(_: &BufferedScanner) -> bool {
         true
     }
-    fn meta_dec_entry_pretest(scanner: &BufferedScanner) -> bool {
-        FieldMapEntryMD::pretest_src_for_metadata_dec(scanner)
+    fn pretest_entry_metadata(scanner: &BufferedScanner) -> bool {
+        scanner.has_left(sizeof!(u64, 3) + 1)
+    }
+    fn pretest_entry_data(scanner: &BufferedScanner, md: &Self::EntryMD) -> bool {
+        scanner.has_left(md.field_id_l as usize) // TODO(@ohsayan): we can enforce way more here such as atleast one field etc
     }
     fn entry_md_enc(buf: &mut VecU8, key: &Self::Key, val: &Self::Value) {
         buf.extend(key.len().u64_bytes_le());
@@ -410,7 +392,12 @@ impl PersistMapSpec for FieldMapSpec {
         buf.push(val.is_nullable() as u8);
     }
     unsafe fn entry_md_dec(scanner: &mut BufferedScanner) -> Option<Self::EntryMD> {
-        FieldMapEntryMD::dec_md_payload(scanner)
+        Some(FieldMapEntryMD::new(
+            u64::from_le_bytes(scanner.next_chunk()),
+            u64::from_le_bytes(scanner.next_chunk()),
+            u64::from_le_bytes(scanner.next_chunk()),
+            scanner.next_byte(),
+        ))
     }
     fn enc_key(buf: &mut VecU8, key: &Self::Key) {
         buf.extend(key.as_bytes());

server/src/engine/storage/v1/inf/md.rs

@@ -0,0 +1,94 @@
+/*
+ * Created on Mon Aug 21 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use super::super::{rw::BufferedScanner, SDSSError, SDSSResult};
+
+/// metadata spec for a persist map entry
+pub trait PersistObjectMD: Sized {
+    /// set to true if decode is infallible once the MD payload has been verified
+    const MD_DEC_INFALLIBLE: bool;
+    /// returns true if the current buffered source can be used to decode the metadata (self)
+    fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool;
+    /// returns true if per the metadata and the current buffered source, the target object in question can be decoded
+    fn pretest_src_for_object_dec(&self, scanner: &BufferedScanner) -> bool;
+    /// decode the metadata
+    unsafe fn dec_md_payload(scanner: &mut BufferedScanner) -> Option<Self>;
+}
+
+/// Metadata for a simple size requirement
+pub struct SimpleSizeMD<const N: usize>;
+
+impl<const N: usize> PersistObjectMD for SimpleSizeMD<N> {
+    const MD_DEC_INFALLIBLE: bool = true;
+    fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool {
+        scanner.has_left(N)
+    }
+    fn pretest_src_for_object_dec(&self, _: &BufferedScanner) -> bool {
+        true
+    }
+    unsafe fn dec_md_payload(_: &mut BufferedScanner) -> Option<Self> {
+        Some(Self)
+    }
+}
+
+/// For wrappers and other complicated metadata handling, set this to the metadata type
+pub struct VoidMetadata;
+
+impl PersistObjectMD for VoidMetadata {
+    const MD_DEC_INFALLIBLE: bool = true;
+    fn pretest_src_for_metadata_dec(_: &BufferedScanner) -> bool {
+        true
+    }
+    fn pretest_src_for_object_dec(&self, _: &BufferedScanner) -> bool {
+        true
+    }
+    unsafe fn dec_md_payload(_: &mut BufferedScanner) -> Option<Self> {
+        Some(Self)
+    }
+}
+
+/// Decode metadata
+///
+/// ## Safety
+/// unsafe because you need to set whether you've already verified the metadata or not
+pub(super) unsafe fn dec_md<Md: PersistObjectMD, const ASSUME_PRETEST_PASS: bool>(
+    scanner: &mut BufferedScanner,
+) -> SDSSResult<Md> {
+    if ASSUME_PRETEST_PASS || Md::pretest_src_for_metadata_dec(scanner) {
+        match Md::dec_md_payload(scanner) {
+            Some(md) => Ok(md),
+            None => {
+                if Md::MD_DEC_INFALLIBLE {
+                    impossible!()
+                } else {
+                    Err(SDSSError::InternalDecodeStructureCorrupted)
+                }
+            }
+        }
+    } else {
+        Err(SDSSError::InternalDecodeStructureCorrupted)
+    }
+}

server/src/engine/storage/v1/inf/mod.rs

@@ -29,12 +29,14 @@
 use crate::engine::idx::STIndex;
 
 mod map;
+mod md;
 mod obj;
 // tests
 #[cfg(test)]
 mod tests;
 
 use {
+    self::md::PersistObjectMD,
     crate::engine::{
         data::{
             dict::DictEntryGeneric,
@@ -99,77 +101,6 @@ impl PersistDictEntryDscr {
     }
 }
 
-/*
-    md spec
-*/
-
-/// metadata spec for a persist map entry
-pub trait PersistObjectMD: Sized {
-    /// set to true if decode is infallible once the MD payload has been verified
-    const MD_DEC_INFALLIBLE: bool;
-    /// returns true if the current buffered source can be used to decode the metadata (self)
-    fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool;
-    /// returns true if per the metadata and the current buffered source, the target object in question can be decoded
-    fn pretest_src_for_object_dec(&self, scanner: &BufferedScanner) -> bool;
-    /// decode the metadata
-    unsafe fn dec_md_payload(scanner: &mut BufferedScanner) -> Option<Self>;
-}
-
-/// Metadata for a simple size requirement
-pub struct SimpleSizeMD<const N: usize>;
-
-impl<const N: usize> PersistObjectMD for SimpleSizeMD<N> {
-    const MD_DEC_INFALLIBLE: bool = true;
-    fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool {
-        scanner.has_left(N)
-    }
-    fn pretest_src_for_object_dec(&self, _: &BufferedScanner) -> bool {
-        true
-    }
-    unsafe fn dec_md_payload(_: &mut BufferedScanner) -> Option<Self> {
-        Some(Self)
-    }
-}
-
-/// For wrappers and other complicated metadata handling, set this to the metadata type
-pub struct VoidMetadata;
-
-impl PersistObjectMD for VoidMetadata {
-    const MD_DEC_INFALLIBLE: bool = true;
-    fn pretest_src_for_metadata_dec(_: &BufferedScanner) -> bool {
-        true
-    }
-    fn pretest_src_for_object_dec(&self, _: &BufferedScanner) -> bool {
-        true
-    }
-    unsafe fn dec_md_payload(_: &mut BufferedScanner) -> Option<Self> {
-        Some(Self)
-    }
-}
-
-/// Decode metadata
-///
-/// ## Safety
-/// unsafe because you need to set whether you've already verified the metadata or not
-unsafe fn dec_md<Md: PersistObjectMD, const ASSUME_PRETEST_PASS: bool>(
-    scanner: &mut BufferedScanner,
-) -> SDSSResult<Md> {
-    if ASSUME_PRETEST_PASS || Md::pretest_src_for_metadata_dec(scanner) {
-        match Md::dec_md_payload(scanner) {
-            Some(md) => Ok(md),
-            None => {
-                if Md::MD_DEC_INFALLIBLE {
-                    impossible!()
-                } else {
-                    Err(SDSSError::InternalDecodeStructureCorrupted)
-                }
-            }
-        }
-    } else {
-        Err(SDSSError::InternalDecodeStructureCorrupted)
-    }
-}
-
 /*
     obj spec
 */
@@ -177,12 +108,12 @@ unsafe fn dec_md<Md: PersistObjectMD, const ASSUME_PRETEST_PASS: bool>(
 /// Specification for any object that can be persisted
 ///
 /// To actuall enc/dec any object, use functions (and their derivatives) [`enc`] and [`dec`]
-pub trait PersistObjectHlIO {
+pub trait PersistObject {
     const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool;
     /// the actual type (we can have wrappers)
     type Type;
     /// the metadata type (use this to verify the buffered source)
-    type Metadata: PersistObjectMD;
+    type Metadata: md::PersistObjectMD;
     /// enc routine
     ///
     /// METADATA: handle yourself
@@ -195,33 +126,33 @@ pub trait PersistObjectHlIO {
 }
 
 /// enc the given object into a new buffer
-pub fn enc<Obj: PersistObjectHlIO>(obj: &Obj::Type) -> VecU8 {
+pub fn enc<Obj: PersistObject>(obj: &Obj::Type) -> VecU8 {
     let mut buf = vec![];
     Obj::pe_obj_hlio_enc(&mut buf, obj);
     buf
 }
 
 /// enc the object into the given buffer
-pub fn enc_into_buf<Obj: PersistObjectHlIO>(buf: &mut VecU8, obj: &Obj::Type) {
+pub fn enc_into_buf<Obj: PersistObject>(buf: &mut VecU8, obj: &Obj::Type) {
     Obj::pe_obj_hlio_enc(buf, obj)
 }
 
 /// enc the object into the given buffer
-pub fn enc_self_into_buf<Obj: PersistObjectHlIO<Type = Obj>>(buf: &mut VecU8, obj: &Obj) {
+pub fn enc_self_into_buf<Obj: PersistObject<Type = Obj>>(buf: &mut VecU8, obj: &Obj) {
     Obj::pe_obj_hlio_enc(buf, obj)
 }
 
 /// enc the object into a new buffer
-pub fn enc_self<Obj: PersistObjectHlIO<Type = Obj>>(obj: &Obj) -> VecU8 {
+pub fn enc_self<Obj: PersistObject<Type = Obj>>(obj: &Obj) -> VecU8 {
     enc::<Obj>(obj)
 }
 
 /// dec the object
-pub fn dec<Obj: PersistObjectHlIO>(scanner: &mut BufferedScanner) -> SDSSResult<Obj::Type> {
+pub fn dec<Obj: PersistObject>(scanner: &mut BufferedScanner) -> SDSSResult<Obj::Type> {
     if Obj::Metadata::pretest_src_for_metadata_dec(scanner) {
         let md = unsafe {
             // UNSAFE(@ohsaya): pretest
-            dec_md::<Obj::Metadata, true>(scanner)?
+            md::dec_md::<Obj::Metadata, true>(scanner)?
         };
         if Obj::ALWAYS_VERIFY_PAYLOAD_USING_MD && !md.pretest_src_for_object_dec(scanner) {
             return Err(SDSSError::InternalDecodeStructureCorrupted);
@@ -233,9 +164,7 @@ pub fn dec<Obj: PersistObjectHlIO>(scanner: &mut BufferedScanner) -> SDSSResult<
 }
 
 /// dec the object
-pub fn dec_self<Obj: PersistObjectHlIO<Type = Obj>>(
-    scanner: &mut BufferedScanner,
-) -> SDSSResult<Obj> {
+pub fn dec_self<Obj: PersistObject<Type = Obj>>(scanner: &mut BufferedScanner) -> SDSSResult<Obj> {
     dec::<Obj>(scanner)
 }
 
@@ -252,7 +181,7 @@ pub trait PersistMapSpec {
     where
         Self: 'a;
     /// metadata type
-    type EntryMD: PersistObjectMD;
+    type EntryMD;
     /// key type (NOTE: set this to the true key type; handle any differences using the spec unless you have an entirely different
     /// wrapper type)
     type Key: AsKey;
@@ -264,13 +193,17 @@ pub trait PersistMapSpec {
     const DEC_COUPLED: bool;
     /// verify the src using the given metadata
     const META_VERIFY_BEFORE_DEC: bool;
+    /// set to true if the entry meta, once pretested never fails to decode
+    const ENTRYMETA_DEC_INFALLIBLE: bool;
     // collection misc
     fn _get_iter<'a>(map: &'a Self::MapType) -> Self::MapIter<'a>;
     // collection meta
     /// pretest before jmp to routine for entire collection
-    fn meta_dec_collection_pretest(scanner: &BufferedScanner) -> bool;
+    fn pretest_collection(scanner: &BufferedScanner) -> bool;
     /// pretest before jmp to entry dec routine
-    fn meta_dec_entry_pretest(scanner: &BufferedScanner) -> bool;
+    fn pretest_entry_metadata(scanner: &BufferedScanner) -> bool;
+    /// pretest the src before jmp to entry data dec routine
+    fn pretest_entry_data(scanner: &BufferedScanner, md: &Self::EntryMD) -> bool;
     // entry meta
     /// enc the entry meta
     fn entry_md_enc(buf: &mut VecU8, key: &Self::Key, val: &Self::Value);

server/src/engine/storage/v1/inf/obj.rs

@@ -25,7 +25,11 @@
 */
 
 use {
-    super::{dec_md, map::FieldMapSpec, PersistObjectHlIO, PersistObjectMD, SimpleSizeMD, VecU8},
+    super::{
+        map::FieldMapSpec,
+        md::{dec_md, PersistObjectMD, SimpleSizeMD},
+        PersistObject, VecU8,
+    },
     crate::{
         engine::{
             core::{
@@ -52,7 +56,7 @@ use {
 
 struct POByteBlockFullTag(FullTag);
 
-impl PersistObjectHlIO for POByteBlockFullTag {
+impl PersistObject for POByteBlockFullTag {
     const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool = false;
     type Type = FullTag;
     type Metadata = SimpleSizeMD<{ sizeof!(u64) }>;
@@ -106,7 +110,7 @@ impl PersistObjectMD for LayerMD {
     }
 }
 
-impl PersistObjectHlIO for Layer {
+impl PersistObject for Layer {
     const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool = false;
     type Type = Layer;
     type Metadata = LayerMD;
@@ -166,7 +170,7 @@ impl PersistObjectMD for FieldMD {
     }
 }
 
-impl PersistObjectHlIO for Field {
+impl PersistObject for Field {
     const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool = false;
     type Type = Self;
     type Metadata = FieldMD;
@@ -187,7 +191,7 @@ impl PersistObjectHlIO for Field {
         let mut fin = false;
         while (!scanner.eof())
             & (layers.len() as u64 != md.layer_c)
-            & (<Layer as PersistObjectHlIO>::Metadata::pretest_src_for_metadata_dec(scanner))
+            & (<Layer as PersistObject>::Metadata::pretest_src_for_metadata_dec(scanner))
             & !fin
         {
             let layer_md = unsafe {
@@ -241,7 +245,7 @@ impl PersistObjectMD for ModelLayoutMD {
     }
 }
 
-impl PersistObjectHlIO for ModelLayout {
+impl PersistObject for ModelLayout {
     const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool = true;
     type Type = Model;
     type Metadata = ModelLayoutMD;
@@ -303,7 +307,7 @@ impl PersistObjectMD for SpaceLayoutMD {
     }
 }
 
-impl PersistObjectHlIO for SpaceLayout {
+impl PersistObject for SpaceLayout {
     const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool = false; // no need, since the MD only handles the UUID
     type Type = Space;
     type Metadata = SpaceLayoutMD;