gvsafronov
/
skytable
1
0
Fork 0
Code Pull Requests Activity

Remove metadata spec and simplify obj enc/dec

Browse Source
next
Sayan Nandan 1 year ago
parent d09df81757
commit 79b450160e
No known key found for this signature in database
GPG Key ID: 42EEDF4AE9D96B54
6 changed files with 349 additions and 413 deletions
Show Stats Download Patch File Download Diff File

4
server/src/engine/core/model/mod.rs
Unescape Escape View File

@ -25,7 +25,7 @@
*/
pub(super) mod alt;
mod delta;
pub(in crate::engine) mod delta;
#[cfg(test)]
use std::cell::RefCell;
@ -52,7 +52,7 @@ use {
};
pub(in crate::engine::core) use self::delta::{DeltaKind, DeltaState, DeltaVersion};
pub(in crate::engine::core) type Fields = IndexSTSeqCns<Box<str>, Field>;
pub(in crate::engine) type Fields = IndexSTSeqCns<Box<str>, Field>;
#[derive(Debug)]
pub struct Model {

184
server/src/engine/storage/v1/inf/map.rs
Unescape Escape View File

@ -25,12 +25,10 @@
*/
use {
super::{
md::VoidMetadata, obj::FieldMD, PersistDictEntryDscr, PersistMapSpec, PersistObject, VecU8,
},
super::{obj::FieldMD, PersistDictEntryDscr, PersistMapSpec, PersistObject, VecU8},
crate::{
engine::{
core::model::{Field, Layer},
core::model::Field,
data::{
cell::Datacell,
dict::DictEntryGeneric,
@ -46,103 +44,94 @@ use {
std::cmp,
};
#[derive(Debug, PartialEq, Eq, Clone, Copy, PartialOrd, Ord)]
pub struct MapIndexSizeMD(pub(super) usize);
/// This is more of a lazy hack than anything sensible. Just implement a spec and then use this wrapper for any enc/dec operations
pub struct PersistMapImpl<M: PersistMapSpec>(PhantomData<M>);
pub struct PersistMapImpl<'a, M: PersistMapSpec>(PhantomData<&'a M::MapType>);
impl<M: PersistMapSpec> PersistObject for PersistMapImpl<M>
impl<'a, M: PersistMapSpec> PersistObject for PersistMapImpl<'a, M>
where
M::MapType: STIndex<M::Key, M::Value>,
M::MapType: 'a + STIndex<M::Key, M::Value>,
{
const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool = false;
type Type = M::MapType;
type Metadata = VoidMetadata;
fn pe_obj_hlio_enc(buf: &mut VecU8, v: &Self::Type) {
enc_dict_into_buffer::<M>(buf, v)
}
unsafe fn pe_obj_hlio_dec(
scanner: &mut BufferedScanner,
_: VoidMetadata,
) -> SDSSResult<Self::Type> {
dec_dict::<M>(scanner)
type InputType = &'a M::MapType;
type OutputType = M::MapType;
type Metadata = MapIndexSizeMD;
fn pretest_can_dec_metadata(scanner: &BufferedScanner) -> bool {
scanner.has_left(sizeof!(u64))
}
fn pretest_can_dec_object(
s: &BufferedScanner,
MapIndexSizeMD(dict_size): &Self::Metadata,
) -> bool {
M::pretest_collection_using_size(s, *dict_size)
}
fn meta_enc(buf: &mut VecU8, data: Self::InputType) {
buf.extend(data.st_len().u64_bytes_le());
}
unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
Ok(MapIndexSizeMD(
u64::from_le_bytes(scanner.next_chunk()) as usize
))
}
}
/// Encode the dict into the given buffer
pub fn enc_dict_into_buffer<PM: PersistMapSpec>(buf: &mut VecU8, map: &PM::MapType) {
buf.extend(map.st_len().u64_bytes_le());
for (key, val) in PM::_get_iter(map) {
PM::entry_md_enc(buf, key, val);
if PM::ENC_COUPLED {
PM::enc_entry(buf, key, val);
} else {
PM::enc_key(buf, key);
PM::enc_val(buf, val);
fn obj_enc(buf: &mut VecU8, map: Self::InputType) {
for (key, val) in M::_get_iter(map) {
M::entry_md_enc(buf, key, val);
if M::ENC_COUPLED {
M::enc_entry(buf, key, val);
} else {
M::enc_key(buf, key);
M::enc_val(buf, val);
}
}
}
}
/// Decode the dict using the given buffered scanner
pub fn dec_dict<PM: PersistMapSpec>(scanner: &mut BufferedScanner) -> SDSSResult<PM::MapType>
where
PM::MapType: STIndex<PM::Key, PM::Value>,
{
if !(PM::pretest_collection(scanner) & scanner.has_left(sizeof!(u64))) {
return Err(SDSSError::InternalDecodeStructureCorrupted);
}
let size = unsafe {
// UNSAFE(@ohsayan): pretest
scanner.next_u64_le() as usize
};
let mut dict = PM::MapType::idx_init_cap(size);
while PM::pretest_entry_metadata(scanner) & (dict.st_len() != size) {
let md = unsafe {
// pretest
match PM::entry_md_dec(scanner) {
Some(md) => md,
None => {
if PM::ENTRYMETA_DEC_INFALLIBLE {
impossible!()
} else {
return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
}
}
unsafe fn obj_dec(
scanner: &mut BufferedScanner,
MapIndexSizeMD(dict_size): Self::Metadata,
) -> SDSSResult<Self::OutputType> {
let mut dict = M::MapType::idx_init();
while M::pretest_entry_metadata(scanner) & (dict.st_len() != dict_size) {
let md = unsafe {
// UNSAFE(@ohsayan): +pretest
M::entry_md_dec(scanner)
.ok_or(SDSSError::InternalDecodeStructureCorruptedPayload)?
};
if !M::pretest_entry_data(scanner, &md) {
return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
}
};
if PM::META_VERIFY_BEFORE_DEC && !PM::pretest_entry_data(scanner, &md) {
return Err(SDSSError::InternalDecodeStructureCorrupted);
}
let key;
let val;
unsafe {
if PM::DEC_COUPLED {
match PM::dec_entry(scanner, md) {
Some((_k, _v)) => {
key = _k;
val = _v;
let key;
let val;
unsafe {
if M::DEC_COUPLED {
match M::dec_entry(scanner, md) {
Some((_k, _v)) => {
key = _k;
val = _v;
}
None => return Err(SDSSError::InternalDecodeStructureCorruptedPayload),
}
None => return Err(SDSSError::InternalDecodeStructureCorruptedPayload),
}
} else {
let _k = PM::dec_key(scanner, &md);
let _v = PM::dec_val(scanner, &md);
match (_k, _v) {
(Some(_k), Some(_v)) => {
key = _k;
val = _v;
} else {
let _k = M::dec_key(scanner, &md);
let _v = M::dec_val(scanner, &md);
match (_k, _v) {
(Some(_k), Some(_v)) => {
key = _k;
val = _v;
}
_ => return Err(SDSSError::InternalDecodeStructureCorruptedPayload),
}
_ => return Err(SDSSError::InternalDecodeStructureCorruptedPayload),
}
}
if !dict.st_insert(key, val) {
return Err(SDSSError::InternalDecodeStructureIllegalData);
}
}
if !dict.st_insert(key, val) {
return Err(SDSSError::InternalDecodeStructureIllegalData);
if dict.st_len() == dict_size {
Ok(dict)
} else {
Err(SDSSError::InternalDecodeStructureIllegalData)
}
}
if dict.st_len() == size {
Ok(dict)
} else {
Err(SDSSError::InternalDecodeStructureIllegalData)
}
}
/// generic dict spec (simple spec for [DictGeneric](crate::engine::data::dict::DictGeneric))
@ -180,14 +169,9 @@ impl PersistMapSpec for GenericDictSpec {
type EntryMD = GenericDictEntryMD;
const DEC_COUPLED: bool = false;
const ENC_COUPLED: bool = true;
const META_VERIFY_BEFORE_DEC: bool = true;
const ENTRYMETA_DEC_INFALLIBLE: bool = true;
fn _get_iter<'a>(map: &'a Self::MapType) -> Self::MapIter<'a> {
map.iter()
}
fn pretest_collection(_: &BufferedScanner) -> bool {
true
}
fn pretest_entry_metadata(scanner: &BufferedScanner) -> bool {
// we just need to see if we can decode the entry metadata
scanner.has_left(9)
@ -208,7 +192,7 @@ impl PersistMapSpec for GenericDictSpec {
DictEntryGeneric::Map(map) => {
buf.push(PersistDictEntryDscr::Dict.value_u8());
buf.extend(key.as_bytes());
enc_dict_into_buffer::<Self>(buf, map);
<PersistMapImpl<Self> as PersistObject>::default_full_enc(buf, map);
}
DictEntryGeneric::Data(dc) => {
buf.push(
@ -319,7 +303,12 @@ impl PersistMapSpec for GenericDictSpec {
}
PersistDictEntryDscr::Dict => {
if dg_top_element {
DictEntryGeneric::Map(dec_dict::<GenericDictSpec>(scanner).ok()?)
DictEntryGeneric::Map(
<PersistMapImpl<GenericDictSpec> as PersistObject>::default_full_dec(
scanner,
)
.ok()?,
)
} else {
unreachable!("found top-level dict item in datacell")
}
@ -369,16 +358,11 @@ impl PersistMapSpec for FieldMapSpec {
type EntryMD = FieldMapEntryMD;
type Key = Box<str>;
type Value = Field;
const ENTRYMETA_DEC_INFALLIBLE: bool = true;
const ENC_COUPLED: bool = false;
const DEC_COUPLED: bool = false;
const META_VERIFY_BEFORE_DEC: bool = true;
fn _get_iter<'a>(m: &'a Self::MapType) -> Self::MapIter<'a> {
m.stseq_ord_kv()
}
fn pretest_collection(_: &BufferedScanner) -> bool {
true
}
fn pretest_entry_metadata(scanner: &BufferedScanner) -> bool {
scanner.has_left(sizeof!(u64, 3) + 1)
}
@ -404,7 +388,7 @@ impl PersistMapSpec for FieldMapSpec {
}
fn enc_val(buf: &mut VecU8, val: &Self::Value) {
for layer in val.layers() {
Layer::pe_obj_hlio_enc(buf, layer)
super::obj::LayerRef::default_full_enc(buf, super::obj::LayerRef(layer))
}
}
unsafe fn dec_key(scanner: &mut BufferedScanner, md: &Self::EntryMD) -> Option<Self::Key> {
@ -417,7 +401,7 @@ impl PersistMapSpec for FieldMapSpec {
.ok()
}
unsafe fn dec_val(scanner: &mut BufferedScanner, md: &Self::EntryMD) -> Option<Self::Value> {
Field::pe_obj_hlio_dec(
super::obj::FieldRef::obj_dec(
scanner,
FieldMD::new(md.field_prop_c, md.field_layer_c, md.null),
)

94
server/src/engine/storage/v1/inf/md.rs
Unescape Escape View File

@ -1,94 +0,0 @@
/*
* Created on Mon Aug 21 2023
*
* This file is a part of Skytable
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
* NoSQL database written by Sayan Nandan ("the Author") with the
* vision to provide flexibility in data modelling without compromising
* on performance, queryability or scalability.
*
* Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
use super::super::{rw::BufferedScanner, SDSSError, SDSSResult};
/// metadata spec for a persist map entry
pub trait PersistObjectMD: Sized {
/// set to true if decode is infallible once the MD payload has been verified
const MD_DEC_INFALLIBLE: bool;
/// returns true if the current buffered source can be used to decode the metadata (self)
fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool;
/// returns true if per the metadata and the current buffered source, the target object in question can be decoded
fn pretest_src_for_object_dec(&self, scanner: &BufferedScanner) -> bool;
/// decode the metadata
unsafe fn dec_md_payload(scanner: &mut BufferedScanner) -> Option<Self>;
}
/// Metadata for a simple size requirement
pub struct SimpleSizeMD<const N: usize>;
impl<const N: usize> PersistObjectMD for SimpleSizeMD<N> {
const MD_DEC_INFALLIBLE: bool = true;
fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool {
scanner.has_left(N)
}
fn pretest_src_for_object_dec(&self, _: &BufferedScanner) -> bool {
true
}
unsafe fn dec_md_payload(_: &mut BufferedScanner) -> Option<Self> {
Some(Self)
}
}
/// For wrappers and other complicated metadata handling, set this to the metadata type
pub struct VoidMetadata;
impl PersistObjectMD for VoidMetadata {
const MD_DEC_INFALLIBLE: bool = true;
fn pretest_src_for_metadata_dec(_: &BufferedScanner) -> bool {
true
}
fn pretest_src_for_object_dec(&self, _: &BufferedScanner) -> bool {
true
}
unsafe fn dec_md_payload(_: &mut BufferedScanner) -> Option<Self> {
Some(Self)
}
}
/// Decode metadata
///
/// ## Safety
/// unsafe because you need to set whether you've already verified the metadata or not
pub(super) unsafe fn dec_md<Md: PersistObjectMD, const ASSUME_PRETEST_PASS: bool>(
scanner: &mut BufferedScanner,
) -> SDSSResult<Md> {
if ASSUME_PRETEST_PASS || Md::pretest_src_for_metadata_dec(scanner) {
match Md::dec_md_payload(scanner) {
Some(md) => Ok(md),
None => {
if Md::MD_DEC_INFALLIBLE {
impossible!()
} else {
Err(SDSSError::InternalDecodeStructureCorrupted)
}
}
}
} else {
Err(SDSSError::InternalDecodeStructureCorrupted)
}
}

161
server/src/engine/storage/v1/inf/mod.rs
Unescape Escape View File

@ -29,14 +29,12 @@
use crate::engine::idx::STIndex;
mod map;
mod md;
mod obj;
// tests
#[cfg(test)]
mod tests;
use {
self::md::PersistObjectMD,
crate::engine::{
data::{
dict::DictEntryGeneric,
@ -105,69 +103,63 @@ impl PersistDictEntryDscr {
obj spec
*/
/// Specification for any object that can be persisted
///
/// To actuall enc/dec any object, use functions (and their derivatives) [`enc`] and [`dec`]
/// Any object that can be persisted
pub trait PersistObject {
const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool;
/// the actual type (we can have wrappers)
type Type;
/// the metadata type (use this to verify the buffered source)
type Metadata: md::PersistObjectMD;
/// enc routine
// types
/// Input type for enc operations
type InputType: Copy;
/// Output type for dec operations
type OutputType;
/// Metadata type
type Metadata;
// pretest
/// Pretest to see if the src has the required data for metadata dec
fn pretest_can_dec_metadata(scanner: &BufferedScanner) -> bool;
/// Pretest to see if the src has the required data for object dec
fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool;
// meta
/// metadata enc
fn meta_enc(buf: &mut VecU8, data: Self::InputType);
/// metadata dec
///
/// METADATA: handle yourself
fn pe_obj_hlio_enc(buf: &mut VecU8, v: &Self::Type);
/// dec routine
unsafe fn pe_obj_hlio_dec(
scanner: &mut BufferedScanner,
md: Self::Metadata,
) -> SDSSResult<Self::Type>;
}
/// enc the given object into a new buffer
pub fn enc<Obj: PersistObject>(obj: &Obj::Type) -> VecU8 {
let mut buf = vec![];
Obj::pe_obj_hlio_enc(&mut buf, obj);
buf
}
/// enc the object into the given buffer
pub fn enc_into_buf<Obj: PersistObject>(buf: &mut VecU8, obj: &Obj::Type) {
Obj::pe_obj_hlio_enc(buf, obj)
}
/// enc the object into the given buffer
pub fn enc_self_into_buf<Obj: PersistObject<Type = Obj>>(buf: &mut VecU8, obj: &Obj) {
Obj::pe_obj_hlio_enc(buf, obj)
}
/// enc the object into a new buffer
pub fn enc_self<Obj: PersistObject<Type = Obj>>(obj: &Obj) -> VecU8 {
enc::<Obj>(obj)
}
/// dec the object
pub fn dec<Obj: PersistObject>(scanner: &mut BufferedScanner) -> SDSSResult<Obj::Type> {
if Obj::Metadata::pretest_src_for_metadata_dec(scanner) {
/// ## Safety
///
/// Must pass the [`PersistObject::pretest_can_dec_metadata`] assertion
unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata>;
// obj
/// obj enc
fn obj_enc(buf: &mut VecU8, data: Self::InputType);
/// obj dec
///
/// ## Safety
///
/// Must pass the [`PersistObject::pretest_can_dec_object`] assertion
unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType>;
// default
/// Default routine to encode an object + its metadata
fn default_full_enc(buf: &mut VecU8, data: Self::InputType) {
Self::meta_enc(buf, data);
Self::obj_enc(buf, data);
}
/// Default routine to decode an object + its metadata (however, the metadata is used and not returned)
fn default_full_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::OutputType> {
if !Self::pretest_can_dec_metadata(scanner) {
return Err(SDSSError::InternalDecodeStructureCorrupted);
}
let md = unsafe {
// UNSAFE(@ohsaya): pretest
md::dec_md::<Obj::Metadata, true>(scanner)?
// UNSAFE(@ohsayan): +pretest
Self::meta_dec(scanner)?
};
if Obj::ALWAYS_VERIFY_PAYLOAD_USING_MD && !md.pretest_src_for_object_dec(scanner) {
return Err(SDSSError::InternalDecodeStructureCorrupted);
if !Self::pretest_can_dec_object(scanner, &md) {
return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
}
unsafe {
// UNSAFE(@ohsayan): +obj pretest
Self::obj_dec(scanner, md)
}
unsafe { Obj::pe_obj_hlio_dec(scanner, md) }
} else {
Err(SDSSError::InternalDecodeStructureCorrupted)
}
}
/// dec the object
pub fn dec_self<Obj: PersistObject<Type = Obj>>(scanner: &mut BufferedScanner) -> SDSSResult<Obj> {
dec::<Obj>(scanner)
}
/*
map spec
*/
@ -191,15 +183,13 @@ pub trait PersistMapSpec {
const ENC_COUPLED: bool;
/// coupled dec
const DEC_COUPLED: bool;
/// verify the src using the given metadata
const META_VERIFY_BEFORE_DEC: bool;
/// set to true if the entry meta, once pretested never fails to decode
const ENTRYMETA_DEC_INFALLIBLE: bool;
// collection misc
fn _get_iter<'a>(map: &'a Self::MapType) -> Self::MapIter<'a>;
// collection meta
/// pretest before jmp to routine for entire collection
fn pretest_collection(scanner: &BufferedScanner) -> bool;
fn pretest_collection_using_size(_: &BufferedScanner, _: usize) -> bool {
true
}
/// pretest before jmp to entry dec routine
fn pretest_entry_metadata(scanner: &BufferedScanner) -> bool;
/// pretest the src before jmp to entry data dec routine
@ -228,3 +218,50 @@ pub trait PersistMapSpec {
md: Self::EntryMD,
) -> Option<(Self::Key, Self::Value)>;
}
// enc
pub mod enc {
use super::{map, PersistMapSpec, PersistObject, VecU8};
pub fn enc_full<Obj: PersistObject>(obj: Obj::InputType) -> Vec<u8> {
let mut v = vec![];
enc_full_into_buffer::<Obj>(&mut v, obj);
v
}
pub fn enc_full_into_buffer<Obj: PersistObject>(buf: &mut VecU8, obj: Obj::InputType) {
Obj::default_full_enc(buf, obj)
}
pub fn enc_dict_full<PM: PersistMapSpec>(dict: &PM::MapType) -> Vec<u8> {
let mut v = vec![];
enc_dict_full_into_buffer::<PM>(&mut v, dict);
v
}
pub fn enc_dict_full_into_buffer<PM: PersistMapSpec>(buf: &mut VecU8, dict: &PM::MapType) {
<map::PersistMapImpl<PM> as PersistObject>::default_full_enc(buf, dict)
}
}
// dec
pub mod dec {
use {
super::{map, PersistMapSpec, PersistObject},
crate::engine::storage::v1::{rw::BufferedScanner, SDSSResult},
};
pub fn dec_full<Obj: PersistObject>(data: &[u8]) -> SDSSResult<Obj::OutputType> {
let mut scanner = BufferedScanner::new(data);
dec_full_from_scanner::<Obj>(&mut scanner)
}
pub fn dec_full_from_scanner<Obj: PersistObject>(
scanner: &mut BufferedScanner,
) -> SDSSResult<Obj::OutputType> {
Obj::default_full_dec(scanner)
}
pub fn dec_dict_full<PM: PersistMapSpec>(data: &[u8]) -> SDSSResult<PM::MapType> {
let mut scanner = BufferedScanner::new(data);
dec_dict_full_from_scanner::<PM>(&mut scanner)
}
fn dec_dict_full_from_scanner<PM: PersistMapSpec>(
scanner: &mut BufferedScanner,
) -> SDSSResult<PM::MapType> {
<map::PersistMapImpl<PM> as PersistObject>::default_full_dec(scanner)
}
}

255
server/src/engine/storage/v1/inf/obj.rs
Unescape Escape View File

@ -24,12 +24,10 @@
*
*/
use crate::engine::{core::model::delta::IRModel, data::DictGeneric};
use {
super::{
map::FieldMapSpec,
md::{dec_md, PersistObjectMD, SimpleSizeMD},
PersistObject, VecU8,
},
super::{PersistObject, VecU8},
crate::{
engine::{
core::{
@ -37,7 +35,7 @@ use {
space::{Space, SpaceMeta},
},
data::{
tag::{DataTag, FullTag, TagClass, TagSelector},
tag::{DataTag, TagClass, TagSelector},
uuid::Uuid,
},
mem::VInline,
@ -54,27 +52,6 @@ use {
to be the correct size.)
*/
struct POByteBlockFullTag(FullTag);
impl PersistObject for POByteBlockFullTag {
const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool = false;
type Type = FullTag;
type Metadata = SimpleSizeMD<{ sizeof!(u64) }>;
fn pe_obj_hlio_enc(buf: &mut VecU8, slf: &Self::Type) {
buf.extend(slf.tag_selector().d().u64_bytes_le())
}
unsafe fn pe_obj_hlio_dec(
scanner: &mut BufferedScanner,
_: Self::Metadata,
) -> SDSSResult<FullTag> {
let dscr = scanner.next_u64_le();
if dscr > TagSelector::max_dscr() as u64 {
return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
}
Ok(TagSelector::from_raw(dscr as u8).into_full())
}
}
/*
layer
*/
@ -94,35 +71,35 @@ impl LayerMD {
}
}
impl PersistObjectMD for LayerMD {
const MD_DEC_INFALLIBLE: bool = true;
fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool {
#[derive(Clone, Copy)]
pub struct LayerRef<'a>(pub &'a Layer);
impl<'a> From<&'a Layer> for LayerRef<'a> {
fn from(value: &'a Layer) -> Self {
Self(value)
}
}
impl<'a> PersistObject for LayerRef<'a> {
type InputType = LayerRef<'a>;
type OutputType = Layer;
type Metadata = LayerMD;
fn pretest_can_dec_metadata(scanner: &BufferedScanner) -> bool {
scanner.has_left(sizeof!(u64, 2))
}
fn pretest_src_for_object_dec(&self, _: &BufferedScanner) -> bool {
fn pretest_can_dec_object(_: &BufferedScanner, _: &Self::Metadata) -> bool {
true
}
unsafe fn dec_md_payload(scanner: &mut BufferedScanner) -> Option<Self> {
Some(Self::new(
fn meta_enc(buf: &mut VecU8, LayerRef(layer): Self::InputType) {
buf.extend(layer.tag().tag_selector().d().u64_bytes_le());
buf.extend(0u64.to_le_bytes());
}
unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
Ok(LayerMD::new(
u64::from_le_bytes(scanner.next_chunk()),
u64::from_le_bytes(scanner.next_chunk()),
))
}
}
impl PersistObject for Layer {
const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool = false;
type Type = Layer;
type Metadata = LayerMD;
fn pe_obj_hlio_enc(buf: &mut VecU8, slf: &Self::Type) {
// [8B: type sig][8B: empty property set]
POByteBlockFullTag::pe_obj_hlio_enc(buf, &slf.tag());
buf.extend(0u64.to_le_bytes());
}
unsafe fn pe_obj_hlio_dec(
_: &mut BufferedScanner,
md: Self::Metadata,
) -> SDSSResult<Self::Type> {
fn obj_enc(_: &mut VecU8, _: Self::InputType) {}
unsafe fn obj_dec(_: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
if (md.type_selector > TagSelector::List.d() as u64) | (md.prop_set_arity != 0) {
return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
}
@ -152,53 +129,56 @@ impl FieldMD {
}
}
impl PersistObjectMD for FieldMD {
const MD_DEC_INFALLIBLE: bool = true;
fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool {
pub struct FieldRef<'a>(&'a Field);
impl<'a> From<&'a Field> for FieldRef<'a> {
fn from(f: &'a Field) -> Self {
Self(f)
}
}
impl<'a> PersistObject for FieldRef<'a> {
type InputType = &'a Field;
type OutputType = Field;
type Metadata = FieldMD;
fn pretest_can_dec_metadata(scanner: &BufferedScanner) -> bool {
scanner.has_left(sizeof!(u64, 2) + 1)
}
fn pretest_src_for_object_dec(&self, _: &BufferedScanner) -> bool {
// nothing here really; we can't help much with the stuff ahead
fn pretest_can_dec_object(_: &BufferedScanner, _: &Self::Metadata) -> bool {
true
}
unsafe fn dec_md_payload(scanner: &mut BufferedScanner) -> Option<Self> {
Some(Self::new(
fn meta_enc(buf: &mut VecU8, slf: Self::InputType) {
// [prop_c][layer_c][null]
buf.extend(0u64.to_le_bytes());
buf.extend(slf.layers().len().u64_bytes_le());
buf.push(slf.is_nullable() as u8);
}
unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
Ok(FieldMD::new(
u64::from_le_bytes(scanner.next_chunk()),
u64::from_le_bytes(scanner.next_chunk()),
scanner.next_byte(),
))
}
}
impl PersistObject for Field {
const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool = false;
type Type = Self;
type Metadata = FieldMD;
fn pe_obj_hlio_enc(buf: &mut VecU8, slf: &Self::Type) {
// [prop_c][layer_c][null]
buf.extend(0u64.to_le_bytes());
buf.extend(slf.layers().len().u64_bytes_le());
buf.push(slf.is_nullable() as u8);
fn obj_enc(buf: &mut VecU8, slf: Self::InputType) {
for layer in slf.layers() {
Layer::pe_obj_hlio_enc(buf, layer);
LayerRef::default_full_enc(buf, LayerRef(layer));
}
}
unsafe fn pe_obj_hlio_dec(
unsafe fn obj_dec(
scanner: &mut BufferedScanner,
md: Self::Metadata,
) -> SDSSResult<Self::Type> {
) -> SDSSResult<Self::OutputType> {
let mut layers = VInline::new();
let mut fin = false;
while (!scanner.eof())
& (layers.len() as u64 != md.layer_c)
& (<Layer as PersistObject>::Metadata::pretest_src_for_metadata_dec(scanner))
& (LayerRef::pretest_can_dec_metadata(scanner))
& !fin
{
let layer_md = unsafe {
// UNSAFE(@ohsayan): pretest
dec_md::<_, true>(scanner)?
LayerRef::meta_dec(scanner)?
};
let l = Layer::pe_obj_hlio_dec(scanner, layer_md)?;
let l = LayerRef::obj_dec(scanner, layer_md)?;
fin = l.tag().tag_class() != TagClass::List;
layers.push(l);
}
@ -216,58 +196,78 @@ pub struct ModelLayoutMD {
model_uuid: Uuid,
p_key_len: u64,
p_key_tag: u64,
field_c: u64,
}
impl ModelLayoutMD {
pub(super) const fn new(model_uuid: Uuid, p_key_len: u64, p_key_tag: u64) -> Self {
pub(super) const fn new(
model_uuid: Uuid,
p_key_len: u64,
p_key_tag: u64,
field_c: u64,
) -> Self {
Self {
model_uuid,
p_key_len,
p_key_tag,
field_c,
}
}
}
impl PersistObjectMD for ModelLayoutMD {
const MD_DEC_INFALLIBLE: bool = true;
fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool {
scanner.has_left(sizeof!(u64, 3) + sizeof!(u128)) // u64,3 since the fieldmap len is also there, but we don't handle it directly
#[derive(Clone, Copy)]
pub struct ModelLayoutRef<'a>(pub(super) &'a Model, pub(super) &'a IRModel<'a>);
impl<'a> From<(&'a Model, &'a IRModel<'a>)> for ModelLayoutRef<'a> {
fn from((mdl, irm): (&'a Model, &'a IRModel<'a>)) -> Self {
Self(mdl, irm)
}
}
impl<'a> PersistObject for ModelLayoutRef<'a> {
type InputType = ModelLayoutRef<'a>;
type OutputType = Model;
type Metadata = ModelLayoutMD;
fn pretest_can_dec_metadata(scanner: &BufferedScanner) -> bool {
scanner.has_left(sizeof!(u128) + sizeof!(u64, 3))
}
fn pretest_src_for_object_dec(&self, scanner: &BufferedScanner) -> bool {
scanner.has_left(self.p_key_len as usize)
fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool {
scanner.has_left(md.p_key_len as usize)
}
fn meta_enc(buf: &mut VecU8, ModelLayoutRef(v, irm): Self::InputType) {
buf.extend(v.get_uuid().to_le_bytes());
buf.extend(v.p_key().len().u64_bytes_le());
buf.extend(v.p_tag().tag_selector().d().u64_bytes_le());
buf.extend(irm.fields().len().u64_bytes_le());
}
unsafe fn dec_md_payload(scanner: &mut BufferedScanner) -> Option<Self> {
Some(Self::new(
unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
Ok(ModelLayoutMD::new(
Uuid::from_bytes(scanner.next_chunk()),
u64::from_le_bytes(scanner.next_chunk()),
u64::from_le_bytes(scanner.next_chunk()),
u64::from_le_bytes(scanner.next_chunk()),
))
}
}
impl PersistObject for ModelLayout {
const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool = true;
type Type = Model;
type Metadata = ModelLayoutMD;
fn pe_obj_hlio_enc(buf: &mut VecU8, v: &Self::Type) {
let irm = v.intent_read_model();
buf.extend(v.get_uuid().to_le_bytes());
buf.extend(v.p_key().len().u64_bytes_le());
buf.extend(v.p_tag().tag_selector().d().u64_bytes_le());
buf.extend(v.p_key().as_bytes());
super::map::enc_dict_into_buffer::<FieldMapSpec>(buf, irm.fields())
fn obj_enc(buf: &mut VecU8, ModelLayoutRef(mdl, irm): Self::InputType) {
buf.extend(mdl.p_key().as_bytes());
<super::map::PersistMapImpl<super::map::FieldMapSpec> as PersistObject>::obj_enc(
buf,
irm.fields(),
)
}
unsafe fn pe_obj_hlio_dec(
unsafe fn obj_dec(
scanner: &mut BufferedScanner,
md: Self::Metadata,
) -> SDSSResult<Self::Type> {
) -> SDSSResult<Self::OutputType> {
let key = String::from_utf8(
scanner
.next_chunk_variable(md.p_key_len as usize)
.to_owned(),
)
.map_err(|_| SDSSError::InternalDecodeStructureCorruptedPayload)?;
let fieldmap = super::map::dec_dict::<FieldMapSpec>(scanner)?;
let fieldmap =
<super::map::PersistMapImpl<super::map::FieldMapSpec> as PersistObject>::obj_dec(
scanner,
super::map::MapIndexSizeMD(md.field_c as usize),
)?;
let ptag = if md.p_key_tag > TagSelector::max_dscr() as u64 {
return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
} else {
@ -285,45 +285,56 @@ impl PersistObject for ModelLayout {
pub struct SpaceLayout;
pub struct SpaceLayoutMD {
uuid: Uuid,
prop_c: usize,
}
impl SpaceLayoutMD {
pub fn new(uuid: Uuid) -> Self {
Self { uuid }
pub fn new(uuid: Uuid, prop_c: usize) -> Self {
Self { uuid, prop_c }
}
}
impl PersistObjectMD for SpaceLayoutMD {
const MD_DEC_INFALLIBLE: bool = true;
fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool {
scanner.has_left(sizeof!(u128) + sizeof!(u64)) // u64 for props dict; we don't handle that directly
#[derive(Clone, Copy)]
pub struct SpaceLayoutRef<'a>(&'a Space, &'a DictGeneric);
impl<'a> From<(&'a Space, &'a DictGeneric)> for SpaceLayoutRef<'a> {
fn from((spc, spc_meta): (&'a Space, &'a DictGeneric)) -> Self {
Self(spc, spc_meta)
}
fn pretest_src_for_object_dec(&self, _: &BufferedScanner) -> bool {
}
impl<'a> PersistObject for SpaceLayoutRef<'a> {
type InputType = SpaceLayoutRef<'a>;
type OutputType = Space;
type Metadata = SpaceLayoutMD;
fn pretest_can_dec_metadata(scanner: &BufferedScanner) -> bool {
scanner.has_left(sizeof!(u128) + sizeof!(u64)) // u64 for props dict
}
fn pretest_can_dec_object(_: &BufferedScanner, _: &Self::Metadata) -> bool {
true
}
unsafe fn dec_md_payload(scanner: &mut BufferedScanner) -> Option<Self> {
Some(Self::new(Uuid::from_bytes(scanner.next_chunk())))
fn meta_enc(buf: &mut VecU8, SpaceLayoutRef(space, space_meta): Self::InputType) {
buf.extend(space.get_uuid().to_le_bytes());
buf.extend(space_meta.len().u64_bytes_le());
}
}
impl PersistObject for SpaceLayout {
const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool = false; // no need, since the MD only handles the UUID
type Type = Space;
type Metadata = SpaceLayoutMD;
fn pe_obj_hlio_enc(buf: &mut VecU8, v: &Self::Type) {
buf.extend(v.get_uuid().to_le_bytes());
super::enc_into_buf::<super::map::PersistMapImpl<super::map::GenericDictSpec>>(
buf,
&v.metadata().env().read(),
);
unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
Ok(SpaceLayoutMD::new(
Uuid::from_bytes(scanner.next_chunk()),
u64::from_le_bytes(scanner.next_chunk()) as usize,
))
}
fn obj_enc(buf: &mut VecU8, SpaceLayoutRef(_, space_meta): Self::InputType) {
<super::map::PersistMapImpl<super::map::GenericDictSpec> as PersistObject>::obj_enc(
buf, space_meta,
)
}
unsafe fn pe_obj_hlio_dec(
unsafe fn obj_dec(
scanner: &mut BufferedScanner,
md: Self::Metadata,
) -> SDSSResult<Self::Type> {
) -> SDSSResult<Self::OutputType> {
let space_meta =
super::dec::<super::map::PersistMapImpl<super::map::GenericDictSpec>>(scanner)?;
<super::map::PersistMapImpl<super::map::GenericDictSpec> as PersistObject>::obj_dec(
scanner,
super::map::MapIndexSizeMD(md.prop_c),
)?;
Ok(Space::new_restore_empty(
SpaceMeta::with_env(space_meta),
md.uuid,

64
server/src/engine/storage/v1/inf/tests.rs
Unescape Escape View File

@ -24,19 +24,21 @@
*
*/
use crate::engine::{
core::{
model::{Field, Layer, Model},
space::{Space, SpaceMeta},
},
data::{
cell::Datacell,
dict::{DictEntryGeneric, DictGeneric},
tag::TagSelector,
uuid::Uuid,
use {
super::obj,
crate::engine::{
core::{
model::{Field, Layer, Model},
space::{Space, SpaceMeta},
},
data::{
cell::Datacell,
dict::{DictEntryGeneric, DictGeneric},
tag::TagSelector,
uuid::Uuid,
},
idx::{IndexBaseSpec, IndexSTSeqCns, STIndex, STIndexSeq},
},
idx::{IndexBaseSpec, IndexSTSeqCns, STIndex, STIndexSeq},
storage::v1::rw::BufferedScanner,
};
#[test]
@ -49,29 +51,24 @@ fn dict() {
"and a null" => Datacell::null(),
))
};
let encoded = super::enc::<super::map::PersistMapImpl<super::map::GenericDictSpec>>(&dict);
let mut scanner = BufferedScanner::new(&encoded);
let decoded =
super::dec::<super::map::PersistMapImpl<super::map::GenericDictSpec>>(&mut scanner)
.unwrap();
let encoded = super::enc::enc_dict_full::<super::map::GenericDictSpec>(&dict);
let decoded = super::dec::dec_dict_full::<super::map::GenericDictSpec>(&encoded).unwrap();
assert_eq!(dict, decoded);
}
#[test]
fn layer() {
let layer = Layer::list();
let encoded = super::enc_self(&layer);
let mut scanner = BufferedScanner::new(&encoded);
let dec = super::dec_self::<Layer>(&mut scanner).unwrap();
let encoded = super::enc::enc_full::<obj::LayerRef>(obj::LayerRef(&layer));
let dec = super::dec::dec_full::<obj::LayerRef>(&encoded).unwrap();
assert_eq!(layer, dec);
}
#[test]
fn field() {
let field = Field::new([Layer::list(), Layer::uint64()].into(), true);
let encoded = super::enc_self(&field);
let mut scanner = BufferedScanner::new(&encoded);
let dec = super::dec_self::<Field>(&mut scanner).unwrap();
let encoded = super::enc::enc_full::<obj::FieldRef>((&field).into());
let dec = super::dec::dec_full::<obj::FieldRef>(&encoded).unwrap();
assert_eq!(field, dec);
}
@ -83,10 +80,8 @@ fn fieldmap() {
"profile_pic".into(),
Field::new([Layer::bin()].into(), true),
);
let enc = super::enc::<super::map::PersistMapImpl<super::map::FieldMapSpec>>(&fields);
let mut scanner = BufferedScanner::new(&enc);
let dec =
super::dec::<super::map::PersistMapImpl<super::map::FieldMapSpec>>(&mut scanner).unwrap();
let enc = super::enc::enc_dict_full::<super::map::FieldMapSpec>(&fields);
let dec = super::dec::dec_dict_full::<super::map::FieldMapSpec>(&enc).unwrap();
for ((orig_field_id, orig_field), (restored_field_id, restored_field)) in
fields.stseq_ord_kv().zip(dec.stseq_ord_kv())
{
@ -107,9 +102,9 @@ fn model() {
"profile_pic" => Field::new([Layer::bin()].into(), true),
},
);
let enc = super::enc::<super::obj::ModelLayout>(&model);
let mut scanner = BufferedScanner::new(&enc);
let dec = super::dec::<super::obj::ModelLayout>(&mut scanner).unwrap();
let model_irm = model.intent_read_model();
let enc = super::enc::enc_full::<obj::ModelLayoutRef>(obj::ModelLayoutRef(&model, &model_irm));
let dec = super::dec::dec_full::<obj::ModelLayoutRef>(&enc).unwrap();
assert_eq!(model, dec);
}
@ -117,8 +112,11 @@ fn model() {
fn space() {
let uuid = Uuid::new();
let space = Space::new_with_uuid(Default::default(), SpaceMeta::default(), uuid);
let enc = super::enc::<super::obj::SpaceLayout>(&space);
let mut scanner = BufferedScanner::new(&enc);
let dec = super::dec::<super::obj::SpaceLayout>(&mut scanner).unwrap();
let space_meta_read = space.metadata().env().read();
let enc = super::enc::enc_full::<obj::SpaceLayoutRef>(obj::SpaceLayoutRef::from((
&space,
&*space_meta_read,
)));
let dec = super::dec::dec_full::<obj::SpaceLayoutRef>(&enc).unwrap();
assert_eq!(space, dec);
}

Write Preview
Loading…
Cancel
Save