diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs index 211bcecf..013d6080 100644 --- a/server/src/engine/idx/mod.rs +++ b/server/src/engine/idx/mod.rs @@ -225,6 +225,8 @@ pub trait STIndex: IndexBaseSpec { where Self: 'a, V: 'a; + /// returns the length of the idx + fn st_len(&self) -> usize; /// Attempts to compact the backing storage fn st_compact(&mut self) {} /// Clears all the entries in the STIndex diff --git a/server/src/engine/idx/stdhm.rs b/server/src/engine/idx/stdhm.rs index 4161d03b..0820ada7 100644 --- a/server/src/engine/idx/stdhm.rs +++ b/server/src/engine/idx/stdhm.rs @@ -95,6 +95,10 @@ where self.shrink_to_fit() } + fn st_len(&self) -> usize { + self.len() + } + fn st_clear(&mut self) { self.clear() } diff --git a/server/src/engine/idx/stord/mod.rs b/server/src/engine/idx/stord/mod.rs index 67ae4a93..f54d02f2 100644 --- a/server/src/engine/idx/stord/mod.rs +++ b/server/src/engine/idx/stord/mod.rs @@ -585,6 +585,10 @@ where self.vacuum_full(); } + fn st_len(&self) -> usize { + self.len() + } + fn st_clear(&mut self) { self._clear() } diff --git a/server/src/engine/storage/v1/inf/map.rs b/server/src/engine/storage/v1/inf/map.rs index f90b33f2..c853fdad 100644 --- a/server/src/engine/storage/v1/inf/map.rs +++ b/server/src/engine/storage/v1/inf/map.rs @@ -35,21 +35,26 @@ use { cell::Datacell, dict::DictEntryGeneric, tag::{CUTag, DataTag, TagClass, TagUnique}, + DictGeneric, }, + idx::{IndexBaseSpec, STIndex}, storage::v1::{rw::BufferedScanner, SDSSError, SDSSResult}, }, util::{copy_slice_to_array as memcpy, EndianQW}, }, core::marker::PhantomData, - std::{cmp, collections::HashMap}, + std::cmp, }; /// This is more of a lazy hack than anything sensible. Just implement a spec and then use this wrapper for any enc/dec operations pub struct PersistMapImpl(PhantomData); -impl PersistObjectHlIO for PersistMapImpl { +impl PersistObjectHlIO for PersistMapImpl +where + M::MapType: STIndex, +{ const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool = false; - type Type = HashMap; + type Type = M::MapType; type Metadata = VoidMetadata; fn pe_obj_hlio_enc(buf: &mut VecU8, v: &Self::Type) { enc_dict_into_buffer::(buf, v) @@ -63,12 +68,9 @@ impl PersistObjectHlIO for PersistMapImpl { } /// Encode the dict into the given buffer -pub fn enc_dict_into_buffer( - buf: &mut VecU8, - map: &HashMap, -) { - buf.extend(map.len().u64_bytes_le()); - for (key, val) in map { +pub fn enc_dict_into_buffer(buf: &mut VecU8, map: &PM::MapType) { + buf.extend(map.st_len().u64_bytes_le()); + for (key, val) in map.st_iter_kv() { PM::entry_md_enc(buf, key, val); if PM::ENC_COUPLED { PM::enc_entry(buf, key, val); @@ -80,9 +82,10 @@ pub fn enc_dict_into_buffer( } /// Decode the dict using the given buffered scanner -pub fn dec_dict( - scanner: &mut BufferedScanner, -) -> SDSSResult> { +pub fn dec_dict(scanner: &mut BufferedScanner) -> SDSSResult +where + PM::MapType: STIndex, +{ if !(PM::meta_dec_collection_pretest(scanner) & scanner.has_left(sizeof!(u64))) { return Err(SDSSError::InternalDecodeStructureCorrupted); } @@ -90,11 +93,11 @@ pub fn dec_dict( // UNSAFE(@ohsayan): pretest scanner.next_u64_le() as usize }; - let mut dict = HashMap::with_capacity(size); - while PM::meta_dec_entry_pretest(scanner) & (dict.len() != size) { + let mut dict = PM::MapType::idx_init_cap(size); + while PM::meta_dec_entry_pretest(scanner) & (dict.st_len() != size) { let md = unsafe { // pretest - dec_md::(scanner)? + dec_md::(scanner)? }; if PM::META_VERIFY_BEFORE_DEC && !md.pretest_src_for_object_dec(scanner) { return Err(SDSSError::InternalDecodeStructureCorrupted); @@ -122,11 +125,11 @@ pub fn dec_dict( } } } - if dict.insert(key, val).is_some() { + if !dict.st_insert(key, val) { return Err(SDSSError::InternalDecodeStructureIllegalData); } } - if dict.len() == size { + if dict.st_len() == size { Ok(dict) } else { Err(SDSSError::InternalDecodeStructureIllegalData) @@ -163,7 +166,7 @@ impl GenericDictEntryMD { impl PersistObjectMD for GenericDictEntryMD { const MD_DEC_INFALLIBLE: bool = true; fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool { - scanner.has_left((sizeof!(u64) * 2) + 1) + scanner.has_left(sizeof!(u64, 2) + 1) } unsafe fn dec_md_payload(scanner: &mut BufferedScanner) -> Option { Some(Self::decode(scanner.next_chunk())) @@ -176,9 +179,10 @@ impl PersistObjectMD for GenericDictEntryMD { } impl PersistMapSpec for GenericDictSpec { + type MapType = DictGeneric; type Key = Box; type Value = DictEntryGeneric; - type Metadata = GenericDictEntryMD; + type EntryMD = GenericDictEntryMD; const DEC_COUPLED: bool = false; const ENC_COUPLED: bool = true; const META_VERIFY_BEFORE_DEC: bool = true; @@ -187,13 +191,13 @@ impl PersistMapSpec for GenericDictSpec { } fn meta_dec_entry_pretest(scanner: &BufferedScanner) -> bool { // we just need to see if we can decode the entry metadata - Self::Metadata::pretest_src_for_metadata_dec(scanner) + Self::EntryMD::pretest_src_for_metadata_dec(scanner) } fn entry_md_enc(buf: &mut VecU8, key: &Self::Key, _: &Self::Value) { buf.extend(key.len().u64_bytes_le()); } - unsafe fn entry_md_dec(scanner: &mut BufferedScanner) -> Option { - Some(Self::Metadata::decode(scanner.next_chunk())) + unsafe fn entry_md_dec(scanner: &mut BufferedScanner) -> Option { + Some(Self::EntryMD::decode(scanner.next_chunk())) } fn enc_entry(buf: &mut VecU8, key: &Self::Key, val: &Self::Value) { match val { @@ -236,12 +240,12 @@ impl PersistMapSpec for GenericDictSpec { } } } - unsafe fn dec_key(scanner: &mut BufferedScanner, md: &Self::Metadata) -> Option { + unsafe fn dec_key(scanner: &mut BufferedScanner, md: &Self::EntryMD) -> Option { String::from_utf8(scanner.next_chunk_variable(md.klen).to_owned()) .map(|s| s.into_boxed_str()) .ok() } - unsafe fn dec_val(scanner: &mut BufferedScanner, md: &Self::Metadata) -> Option { + unsafe fn dec_val(scanner: &mut BufferedScanner, md: &Self::EntryMD) -> Option { unsafe fn decode_element( scanner: &mut BufferedScanner, dscr: PersistDictEntryDscr, @@ -330,7 +334,7 @@ impl PersistMapSpec for GenericDictSpec { } unsafe fn dec_entry( _: &mut BufferedScanner, - _: Self::Metadata, + _: Self::EntryMD, ) -> Option<(Self::Key, Self::Value)> { unimplemented!() } diff --git a/server/src/engine/storage/v1/inf/mod.rs b/server/src/engine/storage/v1/inf/mod.rs index 38e8448b..3320c533 100644 --- a/server/src/engine/storage/v1/inf/mod.rs +++ b/server/src/engine/storage/v1/inf/mod.rs @@ -26,6 +26,8 @@ //! High level interfaces +use crate::engine::idx::STIndex; + mod map; mod obj; // tests @@ -243,8 +245,9 @@ pub fn dec_self>( /// specification for a persist map pub trait PersistMapSpec { + type MapType: STIndex; /// metadata type - type Metadata: PersistObjectMD; + type EntryMD: PersistObjectMD; /// key type (NOTE: set this to the true key type; handle any differences using the spec unless you have an entirely different /// wrapper type) type Key: AsKey; @@ -266,22 +269,22 @@ pub trait PersistMapSpec { fn entry_md_enc(buf: &mut VecU8, key: &Self::Key, val: &Self::Value); /// dec the entry meta /// SAFETY: ensure that all pretests have passed (we expect the caller to not be stupid) - unsafe fn entry_md_dec(scanner: &mut BufferedScanner) -> Option; + unsafe fn entry_md_dec(scanner: &mut BufferedScanner) -> Option; // independent packing /// enc key (non-packed) fn enc_key(buf: &mut VecU8, key: &Self::Key); /// enc val (non-packed) fn enc_val(buf: &mut VecU8, key: &Self::Value); /// dec key (non-packed) - unsafe fn dec_key(scanner: &mut BufferedScanner, md: &Self::Metadata) -> Option; + unsafe fn dec_key(scanner: &mut BufferedScanner, md: &Self::EntryMD) -> Option; /// dec val (non-packed) - unsafe fn dec_val(scanner: &mut BufferedScanner, md: &Self::Metadata) -> Option; + unsafe fn dec_val(scanner: &mut BufferedScanner, md: &Self::EntryMD) -> Option; // coupled packing /// entry packed enc fn enc_entry(buf: &mut VecU8, key: &Self::Key, val: &Self::Value); /// entry packed dec unsafe fn dec_entry( scanner: &mut BufferedScanner, - md: Self::Metadata, + md: Self::EntryMD, ) -> Option<(Self::Key, Self::Value)>; } diff --git a/server/src/engine/storage/v1/inf/obj.rs b/server/src/engine/storage/v1/inf/obj.rs index 99689840..4934a22b 100644 --- a/server/src/engine/storage/v1/inf/obj.rs +++ b/server/src/engine/storage/v1/inf/obj.rs @@ -37,6 +37,13 @@ use { }, }; +/* + Full 8B tag block. Notes: + 1. 7B at this moment is currently unused but there's a lot of additional flags that we might want to store here + 2. If we end up deciding that this is indeed a waste of space, version this out and get rid of the 7B (or whatever we determine + to be the correct size.) +*/ + struct POByteBlockFullTag(FullTag); impl PersistObjectHlIO for POByteBlockFullTag { @@ -58,6 +65,10 @@ impl PersistObjectHlIO for POByteBlockFullTag { } } +/* + layer +*/ + #[derive(Debug)] pub struct LayerMD { type_selector: u64, @@ -76,7 +87,7 @@ impl LayerMD { impl PersistObjectMD for LayerMD { const MD_DEC_INFALLIBLE: bool = true; fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool { - scanner.has_left(sizeof!(u64) * 2) + scanner.has_left(sizeof!(u64, 2)) } fn pretest_src_for_object_dec(&self, _: &BufferedScanner) -> bool { true @@ -111,6 +122,10 @@ impl PersistObjectHlIO for Layer { } } +/* + field +*/ + pub struct FieldMD { prop_c: u64, layer_c: u64, @@ -130,7 +145,7 @@ impl FieldMD { impl PersistObjectMD for FieldMD { const MD_DEC_INFALLIBLE: bool = true; fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool { - scanner.has_left((sizeof!(u64) * 2) + 1) + scanner.has_left(sizeof!(u64, 2) + 1) } fn pretest_src_for_object_dec(&self, _: &BufferedScanner) -> bool { // nothing here really; we can't help much with the stuff ahead