gvsafronov
/
skytable
1
0
Fork 0
Code Pull Requests Activity

Preserve type in objects and fix DML validation

Browse Source
next
Sayan Nandan 11 months ago
parent 3e981f3dcb
commit e412cea7eb
No known key found for this signature in database
GPG Key ID: 42EEDF4AE9D96B54
24 changed files with 591 additions and 531 deletions
Show Stats Download Patch File Download Diff File

9
server/src/engine/core/dml/del.rs
Unescape Escape View File

@ -29,11 +29,18 @@ use crate::engine::{
error::{QueryError, QueryResult},
fractal::GlobalInstanceLike,
idx::MTIndex,
net::protocol::Response,
ql::dml::del::DeleteStatement,
sync,
};
#[allow(unused)]
pub fn delete_resp(
global: &impl GlobalInstanceLike,
delete: DeleteStatement,
) -> QueryResult<Response> {
self::delete(global, delete).map(|_| Response::Empty)
}
pub fn delete(global: &impl GlobalInstanceLike, mut delete: DeleteStatement) -> QueryResult<()> {
core::with_model_for_data_update(global, delete.entity(), |model| {
let g = sync::atm::cpin();

17
server/src/engine/core/dml/ins.rs
Unescape Escape View File

@ -34,11 +34,18 @@ use crate::engine::{
error::{QueryError, QueryResult},
fractal::GlobalInstanceLike,
idx::{IndexBaseSpec, MTIndex, STIndex, STIndexSeq},
net::protocol::Response,
ql::dml::ins::{InsertData, InsertStatement},
sync::atm::cpin,
};
#[allow(unused)]
pub fn insert_resp(
global: &impl GlobalInstanceLike,
insert: InsertStatement,
) -> QueryResult<Response> {
self::insert(global, insert).map(|_| Response::Empty)
}
pub fn insert(global: &impl GlobalInstanceLike, insert: InsertStatement) -> QueryResult<()> {
core::with_model_for_data_update(global, insert.entity(), |mdl| {
let irmwd = mdl.intent_write_new_data();
@ -77,7 +84,7 @@ fn prepare_insert(
let mut fields = fields.stseq_ord_kv();
let mut tuple = tuple.into_iter();
while (tuple.len() != 0) & okay {
let data;
let mut data;
let field;
unsafe {
// UNSAFE(@ohsayan): safe because of invariant
@ -86,14 +93,14 @@ fn prepare_insert(
field = fields.next().unwrap_unchecked();
}
let (field_id, field) = field;
okay &= field.validate_data_fpath(&data);
okay &= field.vt_data_fpath(&mut data);
okay &= prepared_data.st_insert(field_id.clone(), data);
}
}
InsertData::Map(map) => {
let mut map = map.into_iter();
while (map.len() != 0) & okay {
let (field_id, field_data) = unsafe {
let (field_id, mut field_data) = unsafe {
// UNSAFE(@ohsayan): safe because of loop invariant
map.next().unwrap_unchecked()
};
@ -101,7 +108,7 @@ fn prepare_insert(
okay = false;
break;
};
okay &= field.validate_data_fpath(&field_data);
okay &= field.vt_data_fpath(&mut field_data);
prepared_data.st_insert(field_id.boxed_str(), field_data);
}
}

7
server/src/engine/core/dml/mod.rs
Unescape Escape View File

@ -41,7 +41,12 @@ use crate::{
#[cfg(test)]
pub use upd::collect_trace_path as update_flow_trace;
pub use {del::delete, ins::insert, sel::select_custom, upd::update};
pub use {
del::{delete, delete_resp},
ins::{insert, insert_resp},
sel::select_custom,
upd::{update, update_resp},
};
impl Model {
pub(self) fn resolve_where<'a>(

8
server/src/engine/core/dml/sel.rs
Unescape Escape View File

@ -30,10 +30,18 @@ use crate::engine::{
error::{QueryError, QueryResult},
fractal::GlobalInstanceLike,
idx::{STIndex, STIndexSeq},
net::protocol::Response,
ql::dml::sel::SelectStatement,
sync,
};
pub fn select_resp(
global: &impl GlobalInstanceLike,
select: SelectStatement,
) -> QueryResult<Response> {
todo!()
}
#[allow(unused)]
pub fn select_custom<F>(
global: &impl GlobalInstanceLike,

89
server/src/engine/core/dml/upd.rs
Unescape Escape View File

@ -37,11 +37,12 @@ use {
data::{
cell::Datacell,
lit::Lit,
tag::{DataTag, TagClass},
tag::{DataTag, FloatSpec, SIntSpec, TagClass, UIntSpec},
},
error::{QueryError, QueryResult},
fractal::GlobalInstanceLike,
idx::STIndex,
net::protocol::Response,
ql::dml::upd::{AssignmentExpression, UpdateStatement},
sync,
},
@ -59,44 +60,56 @@ unsafe fn dc_op_bool_ass(_: &Datacell, rhs: Lit) -> (bool, Datacell) {
(true, Datacell::new_bool(rhs.bool()))
}
// uint
unsafe fn dc_op_uint_ass(_: &Datacell, rhs: Lit) -> (bool, Datacell) {
(true, Datacell::new_uint(rhs.uint()))
unsafe fn dc_op_uint_ass(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
let uint = rhs.uint();
let kind = UIntSpec::from_full(dc.tag());
(kind.check(uint), Datacell::new_uint(uint, kind))
}
unsafe fn dc_op_uint_add(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
let (sum, of) = dc.read_uint().overflowing_add(rhs.uint());
(of, Datacell::new_uint(sum))
let kind = UIntSpec::from_full(dc.tag());
let (uint, did_of) = dc.uint().overflowing_add(rhs.uint());
(kind.check(uint) & !did_of, Datacell::new_uint(uint, kind))
}
unsafe fn dc_op_uint_sub(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
let (diff, of) = dc.read_uint().overflowing_sub(rhs.uint());
(of, Datacell::new_uint(diff))
let kind = UIntSpec::from_full(dc.tag());
let (uint, did_of) = dc.uint().overflowing_sub(rhs.uint());
(kind.check(uint) & !did_of, Datacell::new_uint(uint, kind))
}
unsafe fn dc_op_uint_mul(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
let (prod, of) = dc.read_uint().overflowing_mul(rhs.uint());
(of, Datacell::new_uint(prod))
let kind = UIntSpec::from_full(dc.tag());
let (uint, did_of) = dc.uint().overflowing_mul(rhs.uint());
(kind.check(uint) & !did_of, Datacell::new_uint(uint, kind))
}
unsafe fn dc_op_uint_div(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
let (quo, of) = dc.read_uint().overflowing_div(rhs.uint());
(of, Datacell::new_uint(quo))
let kind = UIntSpec::from_full(dc.tag());
let (uint, did_of) = dc.uint().overflowing_div(rhs.uint());
(kind.check(uint) & !did_of, Datacell::new_uint(uint, kind))
}
// sint
unsafe fn dc_op_sint_ass(_: &Datacell, rhs: Lit) -> (bool, Datacell) {
(true, Datacell::new_sint(rhs.sint()))
unsafe fn dc_op_sint_ass(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
let sint = rhs.sint();
let kind = SIntSpec::from_full(dc.tag());
(kind.check(sint), Datacell::new_sint(sint, kind))
}
unsafe fn dc_op_sint_add(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
let (sum, of) = dc.read_sint().overflowing_add(rhs.sint());
(of, Datacell::new_sint(sum))
let kind = SIntSpec::from_full(dc.tag());
let (sint, did_of) = dc.sint().overflowing_add(rhs.sint());
(kind.check(sint) & !did_of, Datacell::new_sint(sint, kind))
}
unsafe fn dc_op_sint_sub(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
let (diff, of) = dc.read_sint().overflowing_sub(rhs.sint());
(of, Datacell::new_sint(diff))
let kind = SIntSpec::from_full(dc.tag());
let (sint, did_of) = dc.sint().overflowing_sub(rhs.sint());
(kind.check(sint) & !did_of, Datacell::new_sint(sint, kind))
}
unsafe fn dc_op_sint_mul(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
let (prod, of) = dc.read_sint().overflowing_mul(rhs.sint());
(of, Datacell::new_sint(prod))
let kind = SIntSpec::from_full(dc.tag());
let (sint, did_of) = dc.sint().overflowing_mul(rhs.sint());
(kind.check(sint) & !did_of, Datacell::new_sint(sint, kind))
}
unsafe fn dc_op_sint_div(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
let (quo, of) = dc.read_sint().overflowing_div(rhs.sint());
(of, Datacell::new_sint(quo))
let kind = SIntSpec::from_full(dc.tag());
let (sint, did_of) = dc.sint().overflowing_div(rhs.sint());
(kind.check(sint) & !did_of, Datacell::new_sint(sint, kind))
}
/*
float
@ -108,24 +121,30 @@ unsafe fn dc_op_sint_div(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
--
TODO(@ohsayan): account for float32 overflow
*/
unsafe fn dc_op_float_ass(_: &Datacell, rhs: Lit) -> (bool, Datacell) {
(true, Datacell::new_float(rhs.float()))
unsafe fn dc_op_float_ass(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
let float = rhs.float();
let kind = FloatSpec::from_full(dc.tag());
(kind.check(float), Datacell::new_float(float, kind))
}
unsafe fn dc_op_float_add(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
let sum = dc.read_float() + rhs.float();
(true, Datacell::new_float(sum))
let result = dc.read_float() + rhs.float();
let kind = FloatSpec::from_full(dc.tag());
(kind.check(result), Datacell::new_float(result, kind))
}
unsafe fn dc_op_float_sub(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
let diff = dc.read_float() - rhs.float();
(true, Datacell::new_float(diff))
let result = dc.read_float() - rhs.float();
let kind = FloatSpec::from_full(dc.tag());
(kind.check(result), Datacell::new_float(result, kind))
}
unsafe fn dc_op_float_mul(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
let prod = dc.read_float() - rhs.float();
(true, Datacell::new_float(prod))
let result = dc.read_float() * rhs.float();
let kind = FloatSpec::from_full(dc.tag());
(kind.check(result), Datacell::new_float(result, kind))
}
unsafe fn dc_op_float_div(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
let quo = dc.read_float() * rhs.float();
(true, Datacell::new_float(quo))
let result = dc.read_float() / rhs.float();
let kind = FloatSpec::from_full(dc.tag());
(kind.check(result), Datacell::new_float(result, kind))
}
// binary
unsafe fn dc_op_bin_ass(_dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
@ -235,7 +254,13 @@ pub fn collect_trace_path() -> Vec<&'static str> {
ROUTE_TRACE.with(|v| v.borrow().iter().cloned().collect())
}
#[allow(unused)]
pub fn update_resp(
global: &impl GlobalInstanceLike,
update: UpdateStatement,
) -> QueryResult<Response> {
self::update(global, update).map(|_| Response::Empty)
}
pub fn update(global: &impl GlobalInstanceLike, mut update: UpdateStatement) -> QueryResult<()> {
core::with_model_for_data_update(global, update.entity(), |mdl| {
let mut ret = Ok(QueryExecMeta::zero());

11
server/src/engine/core/exec.rs
Unescape Escape View File

@ -171,14 +171,14 @@ fn run_nb(
stmt: KeywordStmt,
) -> QueryResult<Response> {
let stmt = stmt.value_u8() - KeywordStmt::Use.value_u8();
static F: [fn(&Global, &mut State<'static, InplaceData>) -> QueryResult<()>; 8] = [
static F: [fn(&Global, &mut State<'static, InplaceData>) -> QueryResult<Response>; 8] = [
|_, _| panic!("use not implemented"),
|_, _| panic!("inspect not implemented"),
|_, _| panic!("describe not implemented"),
|g, s| _call(g, s, dml::insert),
|g, s| _call(g, s, dml::insert_resp),
|_, _| panic!("select not implemented"),
|g, s| _call(g, s, dml::update),
|g, s| _call(g, s, dml::delete),
|g, s| _call(g, s, dml::update_resp),
|g, s| _call(g, s, dml::delete_resp),
|_, _| panic!("exists not implemented"),
];
{
@ -186,7 +186,6 @@ fn run_nb(
// UNSAFE(@ohsayan): this is a lifetime issue with the token handle
core::mem::transmute(state)
};
F[stmt as usize](global, &mut state)?;
F[stmt as usize](global, &mut state)
}
Ok(Response::Empty)
}

2
server/src/engine/core/index/key.rs
Unescape Escape View File

@ -345,7 +345,7 @@ fn check_pk_lit_eq_hash() {
#[test]
fn check_pk_extremes() {
let state = test_utils::randomstate();
let d1 = PrimaryIndexKey::try_from_dc(Datacell::new_uint(u64::MAX)).unwrap();
let d1 = PrimaryIndexKey::try_from_dc(Datacell::new_uint_default(u64::MAX)).unwrap();
let d2 = PrimaryIndexKey::try_from_dc(Datacell::from(Lit::new_uint(u64::MAX))).unwrap();
assert_eq!(d1, d2);
assert_eq!(d1.uint().unwrap(), u64::MAX);

89
server/src/engine/core/model/mod.rs
Unescape Escape View File

@ -36,7 +36,7 @@ use {
crate::engine::{
data::{
cell::Datacell,
tag::{DataTag, FullTag, TagClass, TagSelector},
tag::{DataTag, FloatSpec, FullTag, SIntSpec, TagClass, TagSelector, UIntSpec},
uuid::Uuid,
},
error::{QueryError, QueryResult},
@ -379,44 +379,39 @@ impl Field {
dc.kind().value_word()
}
}
pub fn validate_data_fpath(&self, data: &Datacell) -> bool {
// if someone sends a PR with an added check, I'll personally come to your house and throw a brick on your head
if (self.layers.len() == 1) | data.is_null() {
pub fn vt_data_fpath(&self, data: &mut Datacell) -> bool {
if (self.layers.len() == 1) | (data.is_null()) {
layertrace("fpath");
unsafe {
// UNSAFE(@ohsayan): checked for non-null, and used correct class
LVERIFY[self.compute_index(data)](self.layers()[0], data)
}
unsafe { VTFN[self.compute_index(data)](self.layers()[0], data) }
} else {
Self::rverify_layers(self.layers(), data)
Self::rvt_data(self.layers(), data)
}
}
// TODO(@ohsayan): improve algo with dfs
fn rverify_layers(layers: &[Layer], data: &Datacell) -> bool {
fn rvt_data(layers: &[Layer], data: &mut Datacell) -> bool {
let layer = layers[0];
let layers = &layers[1..];
match (layer.tag.tag_class(), data.kind()) {
(TagClass::List, TagClass::List) if !layers.is_empty() => {
match (layer.tag().tag_class(), data.kind()) {
(TagClass::List, TagClass::List) => {
let mut okay = unsafe {
// UNSAFE(@ohsayan): we've verified this
LVERIFY[TagClass::List.value_word()](layer, data)
// UNSAFE(@ohsayan): +tagck
VTFN[TagClass::List.value_word()](layer, data)
};
let list = unsafe {
// UNSAFE(@ohsayan): we verified tags
// UNSAFE(@ohsayan): +tagck
data.read_list()
};
let lread = list.read();
let mut lread = list.write();
let mut i = 0;
while (i < lread.len()) & okay {
okay &= Self::rverify_layers(layers, &lread[i]);
okay &= Self::rvt_data(layers, &mut lread[i]);
i += 1;
}
okay
}
(tag_a, tag_b) if tag_a == tag_b => {
unsafe {
// UNSAFE(@ohsayan): same tags; not-null for now so no extra handling required here
LVERIFY[tag_a.value_word()](layer, data)
// UNSAFE(@ohsayan): same tags and lists have non-null elements
VTFN[tag_a.value_word()](layer, data)
}
}
_ => false,
@ -510,17 +505,6 @@ impl Layer {
}
}
static LVERIFY: [unsafe fn(Layer, &Datacell) -> bool; 8] = [
lverify_bool,
lverify_uint,
lverify_sint,
lverify_float,
lverify_bin,
lverify_str,
lverify_list,
|_, _| false,
];
#[cfg(test)]
thread_local! {
static LAYER_TRACE: RefCell<Vec<Box<str>>> = RefCell::new(Vec::new());
@ -544,41 +528,44 @@ pub(super) fn layer_traces() -> Box<[Box<str>]> {
})
}
unsafe fn lverify_bool(_: Layer, _: &Datacell) -> bool {
static VTFN: [unsafe fn(Layer, &mut Datacell) -> bool; 8] = [
vt_bool,
vt_uint,
vt_sint,
vt_float,
vt_bin,
vt_str,
vt_list,
|_, _| false,
];
unsafe fn vt_bool(_: Layer, _: &mut Datacell) -> bool {
layertrace("bool");
true
}
unsafe fn lverify_uint(l: Layer, d: &Datacell) -> bool {
unsafe fn vt_uint(l: Layer, dc: &mut Datacell) -> bool {
layertrace("uint");
const MX: [u64; 4] = [u8::MAX as _, u16::MAX as _, u32::MAX as _, u64::MAX];
d.read_uint() <= MX[l.tag.tag_selector().value_word() - 1]
dc.set_tag(l.tag());
UIntSpec::from_full(l.tag()).check(dc.read_uint())
}
unsafe fn lverify_sint(l: Layer, d: &Datacell) -> bool {
unsafe fn vt_sint(l: Layer, dc: &mut Datacell) -> bool {
layertrace("sint");
const MN_MX: [(i64, i64); 4] = [
(i8::MIN as _, i8::MAX as _),
(i16::MIN as _, i16::MAX as _),
(i32::MIN as _, i32::MAX as _),
(i64::MIN, i64::MAX),
];
let (mn, mx) = MN_MX[l.tag.tag_selector().value_word() - 5];
(d.read_sint() >= mn) & (d.read_sint() <= mx)
dc.set_tag(l.tag());
SIntSpec::from_full(l.tag()).check(dc.read_sint())
}
unsafe fn lverify_float(l: Layer, d: &Datacell) -> bool {
unsafe fn vt_float(l: Layer, dc: &mut Datacell) -> bool {
layertrace("float");
const MN_MX: [(f64, f64); 2] = [(f32::MIN as _, f32::MAX as _), (f64::MIN, f64::MAX)];
let (mn, mx) = MN_MX[l.tag.tag_selector().value_word() - 9];
(d.read_float() >= mn) & (d.read_float() <= mx)
dc.set_tag(l.tag());
FloatSpec::from_full(l.tag()).check(dc.read_float())
}
unsafe fn lverify_bin(_: Layer, _: &Datacell) -> bool {
unsafe fn vt_bin(_: Layer, _: &mut Datacell) -> bool {
layertrace("binary");
true
}
unsafe fn lverify_str(_: Layer, _: &Datacell) -> bool {
unsafe fn vt_str(_: Layer, _: &mut Datacell) -> bool {
layertrace("string");
true
}
unsafe fn lverify_list(_: Layer, _: &Datacell) -> bool {
unsafe fn vt_list(_: Layer, _: &mut Datacell) -> bool {
layertrace("list");
true
}

68
server/src/engine/core/tests/ddl_model/layer.rs
Unescape Escape View File

@ -84,9 +84,9 @@ mod layer_data_validation {
};
#[test]
fn bool() {
let dc = Datacell::new_bool(true);
let mut dc = Datacell::new_bool(true);
let layer = layerview("bool").unwrap();
assert!(layer.validate_data_fpath(&dc));
assert!(layer.vt_data_fpath(&mut dc));
assert_vecstreq_exact!(model::layer_traces(), ["fpath", "bool"]);
}
#[test]
@ -102,17 +102,17 @@ mod layer_data_validation {
.enumerate()
.for_each(|(i, (layer, max))| {
let this_layer = layerview(layer).unwrap();
let dc = Datacell::new_uint(max);
assert!(this_layer.validate_data_fpath(&dc), "{:?}", this_layer);
let mut dc = Datacell::new_uint_default(max);
assert!(this_layer.vt_data_fpath(&mut dc), "{:?}", this_layer);
assert_vecstreq_exact!(model::layer_traces(), ["fpath", "uint"]);
for (lower, _) in targets[..i].iter() {
let layer = layerview(lower).unwrap();
assert!(!layer.validate_data_fpath(&dc), "{:?}", layer);
assert!(!layer.vt_data_fpath(&mut dc), "{:?}", layer);
assert_vecstreq_exact!(model::layer_traces(), ["fpath", "uint"]);
}
for (higher, _) in targets[i + 1..].iter() {
let layer = layerview(higher).unwrap();
assert!(layer.validate_data_fpath(&dc), "{:?}", layer);
assert!(layer.vt_data_fpath(&mut dc), "{:?}", layer);
assert_vecstreq_exact!(model::layer_traces(), ["fpath", "uint"]);
}
});
@ -130,15 +130,15 @@ mod layer_data_validation {
.enumerate()
.for_each(|(i, (layer, (min, max)))| {
let this_layer = layerview(layer).unwrap();
let dc_min = Datacell::new_sint(min);
let dc_max = Datacell::new_sint(max);
assert!(this_layer.validate_data_fpath(&dc_min), "{:?}", this_layer);
assert!(this_layer.validate_data_fpath(&dc_max), "{:?}", this_layer);
let mut dc_min = Datacell::new_sint_default(min);
let mut dc_max = Datacell::new_sint_default(max);
assert!(this_layer.vt_data_fpath(&mut dc_min), "{:?}", this_layer);
assert!(this_layer.vt_data_fpath(&mut dc_max), "{:?}", this_layer);
assert_vecstreq_exact!(model::layer_traces(), ["fpath", "sint", "fpath", "sint"]);
for (lower, _) in targets[..i].iter() {
let layer = layerview(lower).unwrap();
assert!(!layer.validate_data_fpath(&dc_min), "{:?}", layer);
assert!(!layer.validate_data_fpath(&dc_max), "{:?}", layer);
assert!(!layer.vt_data_fpath(&mut dc_min), "{:?}", layer);
assert!(!layer.vt_data_fpath(&mut dc_max), "{:?}", layer);
assert_vecstreq_exact!(
model::layer_traces(),
["fpath", "sint", "fpath", "sint"]
@ -146,8 +146,8 @@ mod layer_data_validation {
}
for (higher, _) in targets[i + 1..].iter() {
let layer = layerview(higher).unwrap();
assert!(layer.validate_data_fpath(&dc_min), "{:?}", layer);
assert!(layer.validate_data_fpath(&dc_max), "{:?}", layer);
assert!(layer.vt_data_fpath(&mut dc_min), "{:?}", layer);
assert!(layer.vt_data_fpath(&mut dc_max), "{:?}", layer);
assert_vecstreq_exact!(
model::layer_traces(),
["fpath", "sint", "fpath", "sint"]
@ -161,46 +161,46 @@ mod layer_data_validation {
let f32_l = layerview("float32").unwrap();
let f64_l = layerview("float64").unwrap();
// dc
let f32_dc_min = Datacell::new_float(f32::MIN as _);
let f32_dc_max = Datacell::new_float(f32::MAX as _);
let f64_dc_min = Datacell::new_float(f64::MIN as _);
let f64_dc_max = Datacell::new_float(f64::MAX as _);
let f32_dc_min = Datacell::new_float_default(f32::MIN as _);
let f32_dc_max = Datacell::new_float_default(f32::MAX as _);
let f64_dc_min = Datacell::new_float_default(f64::MIN as _);
let f64_dc_max = Datacell::new_float_default(f64::MAX as _);
// check (32)
assert!(f32_l.validate_data_fpath(&f32_dc_min));
assert!(f32_l.validate_data_fpath(&f32_dc_max));
assert!(f32_l.vt_data_fpath(&mut f32_dc_min.clone()));
assert!(f32_l.vt_data_fpath(&mut f32_dc_max.clone()));
assert_vecstreq_exact!(model::layer_traces(), ["fpath", "float", "fpath", "float"]);
assert!(f64_l.validate_data_fpath(&f32_dc_min));
assert!(f64_l.validate_data_fpath(&f32_dc_max));
assert!(f64_l.vt_data_fpath(&mut f32_dc_min.clone()));
assert!(f64_l.vt_data_fpath(&mut f32_dc_max.clone()));
assert_vecstreq_exact!(model::layer_traces(), ["fpath", "float", "fpath", "float"]);
// check (64)
assert!(!f32_l.validate_data_fpath(&f64_dc_min));
assert!(!f32_l.validate_data_fpath(&f64_dc_max));
assert!(!f32_l.vt_data_fpath(&mut f64_dc_min.clone()));
assert!(!f32_l.vt_data_fpath(&mut f64_dc_max.clone()));
assert_vecstreq_exact!(model::layer_traces(), ["fpath", "float", "fpath", "float"]);
assert!(f64_l.validate_data_fpath(&f64_dc_min));
assert!(f64_l.validate_data_fpath(&f64_dc_max));
assert!(f64_l.vt_data_fpath(&mut f64_dc_min.clone()));
assert!(f64_l.vt_data_fpath(&mut f64_dc_max.clone()));
assert_vecstreq_exact!(model::layer_traces(), ["fpath", "float", "fpath", "float"]);
}
#[test]
fn bin() {
let layer = layerview("binary").unwrap();
assert!(layer.validate_data_fpath(&Datacell::from("hello".as_bytes())));
assert!(layer.vt_data_fpath(&mut Datacell::from("hello".as_bytes())));
assert_vecstreq_exact!(model::layer_traces(), ["fpath", "binary"]);
}
#[test]
fn str() {
let layer = layerview("string").unwrap();
assert!(layer.validate_data_fpath(&Datacell::from("hello")));
assert!(layer.vt_data_fpath(&mut Datacell::from("hello")));
assert_vecstreq_exact!(model::layer_traces(), ["fpath", "string"]);
}
#[test]
fn list_simple() {
let layer = layerview("list { type: string }").unwrap();
let dc = Datacell::new_list(vec![
let mut dc = Datacell::new_list(vec![
Datacell::from("I"),
Datacell::from("love"),
Datacell::from("cats"),
]);
assert!(layer.validate_data_fpath(&dc));
assert!(layer.vt_data_fpath(&mut dc));
assert_vecstreq_exact!(
model::layer_traces(),
["list", "string", "string", "string"]
@ -209,12 +209,12 @@ mod layer_data_validation {
#[test]
fn list_nested_l1() {
let layer = layerview("list { type: list { type: string } }").unwrap();
let dc = Datacell::new_list(vec![
let mut dc = Datacell::new_list(vec![
Datacell::new_list(vec![Datacell::from("hello_11"), Datacell::from("hello_12")]),
Datacell::new_list(vec![Datacell::from("hello_21"), Datacell::from("hello_22")]),
Datacell::new_list(vec![Datacell::from("hello_31"), Datacell::from("hello_32")]),
]);
assert!(layer.validate_data_fpath(&dc));
assert!(layer.vt_data_fpath(&mut dc));
assert_vecstreq_exact!(
model::layer_traces(),
[
@ -228,13 +228,13 @@ mod layer_data_validation {
#[test]
fn nullval_fpath() {
let layer = layerview_nullable("string", true).unwrap();
assert!(layer.validate_data_fpath(&Datacell::null()));
assert!(layer.vt_data_fpath(&mut Datacell::null()));
assert_vecstreq_exact!(model::layer_traces(), ["fpath", "bool"]);
}
#[test]
fn nullval_nested_but_fpath() {
let layer = layerview_nullable("list { type: string }", true).unwrap();
assert!(layer.validate_data_fpath(&Datacell::null()));
assert!(layer.vt_data_fpath(&mut Datacell::null()));
assert_vecstreq_exact!(model::layer_traces(), ["fpath", "bool"]);
}
}

10
server/src/engine/core/tests/ddl_space/alter.rs
Unescape Escape View File

@ -43,7 +43,7 @@ fn alter_add_prop_env_var() {
space,
&Space::new_with_uuid(
into_dict!(),
SpaceMeta::with_env(into_dict! ("MY_NEW_PROP" => Datacell::new_uint(100))),
SpaceMeta::with_env(into_dict! ("MY_NEW_PROP" => Datacell::new_uint_default(100))),
space.get_uuid()
)
);
@ -62,7 +62,7 @@ fn alter_update_prop_env_var() {
let rl = space.meta.dict().read();
assert_eq!(
SpaceMeta::get_env(&rl).get("MY_NEW_PROP").unwrap(),
&(Datacell::new_uint(100).into())
&(Datacell::new_uint_default(100).into())
)
},
)
@ -75,7 +75,7 @@ fn alter_update_prop_env_var() {
space,
&Space::new_with_uuid(
into_dict!(),
SpaceMeta::with_env(into_dict! ("MY_NEW_PROP" => Datacell::new_uint(200))),
SpaceMeta::with_env(into_dict! ("MY_NEW_PROP" => Datacell::new_uint_default(200))),
uuid,
)
)
@ -94,7 +94,7 @@ fn alter_remove_prop_env_var() {
let rl = space.meta.dict().read();
assert_eq!(
SpaceMeta::get_env(&rl).get("MY_NEW_PROP").unwrap(),
&(Datacell::new_uint(100).into())
&(Datacell::new_uint_default(100).into())
)
},
)
@ -136,7 +136,7 @@ fn alter_remove_all_env() {
let rl = space.meta.dict().read();
assert_eq!(
SpaceMeta::get_env(&rl).get("MY_NEW_PROP").unwrap(),
&(Datacell::new_uint(100).into())
&(Datacell::new_uint_default(100).into())
)
},
)

2
server/src/engine/core/tests/ddl_space/create.rs
Unescape Escape View File

@ -58,7 +58,7 @@ fn exec_create_space_with_env() {
&Space::new_with_uuid(
into_dict! {},
SpaceMeta::with_env(into_dict! {
"MAX_MODELS" => Datacell::new_uint(100)
"MAX_MODELS" => Datacell::new_uint_default(100)
}),
space.get_uuid()
)

76
server/src/engine/data/cell.rs
Unescape Escape View File

@ -24,24 +24,28 @@
*
*/
use std::{marker::PhantomData, ops::Deref};
use {
crate::engine::{
self,
data::{
lit::Lit,
tag::{CUTag, DataTag, TagClass},
tag::{DataTag, FloatSpec, FullTag, SIntSpec, TagClass, UIntSpec},
},
mem::{DwordNN, DwordQN, NativeQword, SpecialPaddedWord, WordIO},
},
core::{fmt, mem, mem::ManuallyDrop, slice, str},
core::{
fmt,
marker::PhantomData,
mem::{self, ManuallyDrop},
ops::Deref,
slice, str,
},
parking_lot::RwLock,
};
pub struct Datacell {
init: bool,
tag: CUTag,
tag: FullTag,
data: DataRaw,
}
@ -51,7 +55,7 @@ impl Datacell {
unsafe {
// UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
Self::new(
CUTag::BOOL,
FullTag::BOOL,
DataRaw::word(SpecialPaddedWord::store(b).dwordqn_promote()),
)
}
@ -69,15 +73,18 @@ impl Datacell {
self.try_bool().unwrap()
}
// uint
pub fn new_uint(u: u64) -> Self {
pub fn new_uint(u: u64, kind: UIntSpec) -> Self {
unsafe {
// UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
Self::new(
CUTag::UINT,
kind.into(),
DataRaw::word(SpecialPaddedWord::store(u).dwordqn_promote()),
)
}
}
pub fn new_uint_default(u: u64) -> Self {
Self::new_uint(u, UIntSpec::DEFAULT)
}
pub unsafe fn read_uint(&self) -> u64 {
self.load_word()
}
@ -101,15 +108,18 @@ impl Datacell {
}
}
// sint
pub fn new_sint(i: i64) -> Self {
pub fn new_sint(i: i64, kind: SIntSpec) -> Self {
unsafe {
// UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
Self::new(
CUTag::SINT,
kind.into(),
DataRaw::word(SpecialPaddedWord::store(i).dwordqn_promote()),
)
}
}
pub fn new_sint_default(s: i64) -> Self {
Self::new_sint(s, SIntSpec::DEFAULT)
}
pub unsafe fn read_sint(&self) -> i64 {
self.load_word()
}
@ -123,15 +133,18 @@ impl Datacell {
self.try_sint().unwrap()
}
// float
pub fn new_float(f: f64) -> Self {
pub fn new_float(f: f64, spec: FloatSpec) -> Self {
unsafe {
// UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
Self::new(
CUTag::FLOAT,
spec.into(),
DataRaw::word(SpecialPaddedWord::store(f).dwordqn_promote()),
)
}
}
pub fn new_float_default(f: f64) -> Self {
Self::new_float(f, FloatSpec::DEFAULT)
}
pub unsafe fn read_float(&self) -> f64 {
self.load_word()
}
@ -150,7 +163,7 @@ impl Datacell {
unsafe {
// UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
Self::new(
CUTag::BIN,
FullTag::BIN,
DataRaw::word(WordIO::store((md.len(), md.as_mut_ptr()))),
)
}
@ -189,7 +202,7 @@ impl Datacell {
unsafe {
// UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
Self::new(
CUTag::STR,
FullTag::STR,
DataRaw::word(WordIO::store((md.len(), md.as_mut_ptr()))),
)
}
@ -211,7 +224,7 @@ impl Datacell {
pub fn new_list(l: Vec<Self>) -> Self {
unsafe {
// UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
Self::new(CUTag::LIST, DataRaw::rwl(RwLock::new(l)))
Self::new(FullTag::LIST, DataRaw::rwl(RwLock::new(l)))
}
}
pub unsafe fn read_list(&self) -> &RwLock<Vec<Self>> {
@ -237,21 +250,23 @@ impl Datacell {
Some(ManuallyDrop::into_inner(rwl).into_inner())
}
}
pub unsafe fn new_qw(qw: u64, tag: CUTag) -> Datacell {
pub unsafe fn new_qw(qw: u64, tag: FullTag) -> Datacell {
Self::new(
tag,
DataRaw::word(SpecialPaddedWord::store(qw).dwordqn_promote()),
)
}
pub unsafe fn set_tag(&mut self, tag: FullTag) {
self.tag = tag;
}
}
direct_from! {
Datacell => {
bool as new_bool,
u64 as new_uint,
i64 as new_sint,
f64 as new_float,
f32 as new_float,
u64 as new_uint_default,
i64 as new_sint_default,
f64 as new_float_default,
Vec<u8> as new_bin,
Box<[u8]> as new_bin,
&'static [u8] as new_bin,
@ -269,7 +284,7 @@ impl<'a> From<Lit<'a>> for Datacell {
tag if tag < TagClass::Bin => unsafe {
// UNSAFE(@ohsayan): Correct because we are using the same tag, and in this case the type doesn't need any advanced construction
Datacell::new(
CUTag::from(l.kind()),
l.kind(),
// DO NOT RELY ON the payload's bit pattern; it's padded
DataRaw::word(l.data().dwordqn_promote()),
)
@ -278,7 +293,7 @@ impl<'a> From<Lit<'a>> for Datacell {
// UNSAFE(@ohsayan): Correct because we are using the same tag, and in this case the type requires a new heap for construction
let mut bin = ManuallyDrop::new(l.bin().to_owned().into_boxed_slice());
Datacell::new(
CUTag::from(l.kind()),
l.kind(),
DataRaw::word(DwordQN::dwordqn_store_qw_nw(
bin.len() as u64,
bin.as_mut_ptr() as usize,
@ -297,9 +312,9 @@ impl<'a> From<Lit<'a>> for Datacell {
impl From<i32> for Datacell {
fn from(i: i32) -> Self {
if i.is_negative() {
Self::new_sint(i as _)
Self::new_sint_default(i as _)
} else {
Self::new_uint(i as _)
Self::new_uint_default(i as _)
}
}
}
@ -311,7 +326,7 @@ impl<const N: usize> From<[Datacell; N]> for Datacell {
}
impl Datacell {
pub const fn tag(&self) -> CUTag {
pub const fn tag(&self) -> FullTag {
self.tag
}
pub fn kind(&self) -> TagClass {
@ -321,7 +336,7 @@ impl Datacell {
unsafe {
// UNSAFE(@ohsayan): This is a hack. It's safe because we set init to false
Self::_new(
CUTag::BOOL,
FullTag::BOOL,
DataRaw::word(NativeQword::dwordnn_store_qw(0)),
false,
)
@ -339,10 +354,10 @@ impl Datacell {
{
self.data.word.load()
}
unsafe fn _new(tag: CUTag, data: DataRaw, init: bool) -> Self {
unsafe fn _new(tag: FullTag, data: DataRaw, init: bool) -> Self {
Self { init, tag, data }
}
unsafe fn new(tag: CUTag, data: DataRaw) -> Self {
unsafe fn new(tag: FullTag, data: DataRaw) -> Self {
Self::_new(tag, data, true)
}
fn checked_tag<T>(&self, tag: TagClass, f: impl FnOnce() -> T) -> Option<T> {
@ -482,10 +497,7 @@ impl<'a> VirtualDatacell<'a> {
Self {
dc: ManuallyDrop::new(unsafe {
// UNSAFE(@ohsayan): this is a "reference" to a "virtual" aka fake DC. this just works because of memory layouts
Datacell::new(
CUTag::from(lit.kind()),
DataRaw::word(lit.data().dwordqn_promote()),
)
Datacell::new(lit.kind(), DataRaw::word(lit.data().dwordqn_promote()))
}),
_lt: PhantomData,
}

105
server/src/engine/data/tag.rs
Unescape Escape View File

@ -36,23 +36,6 @@ pub enum TagClass {
List = 6,
}
impl TagClass {
pub const unsafe fn from_raw(v: u8) -> Self {
core::mem::transmute(v)
}
pub const fn tag_unique(&self) -> TagUnique {
[
TagUnique::Illegal,
TagUnique::UnsignedInt,
TagUnique::SignedInt,
TagUnique::Illegal,
TagUnique::Bin,
TagUnique::Str,
TagUnique::Illegal,
][self.value_word()]
}
}
#[repr(u8)]
#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord, sky_macros::EnumMethods)]
pub enum TagSelector {
@ -206,57 +189,69 @@ impl DataTag for FullTag {
}
}
#[derive(Debug, Clone, Copy)]
pub struct CUTag {
class: TagClass,
unique: TagUnique,
}
impl PartialEq for CUTag {
fn eq(&self, other: &Self) -> bool {
self.class == other.class
#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)]
#[repr(transparent)]
pub struct UIntSpec(FullTag);
impl UIntSpec {
pub const LIM_MAX: [u64; 4] = as_array![u8::MAX, u16::MAX, u32::MAX, u64::MAX];
pub const DEFAULT: Self = Self::UINT64;
pub const UINT64: Self = Self(FullTag::new_uint(TagSelector::UInt64));
pub const unsafe fn from_full(f: FullTag) -> Self {
Self(f)
}
pub fn check(&self, v: u64) -> bool {
v <= Self::LIM_MAX[self.0.tag_selector().value_word() - 1]
}
}
macro_rules! cutag {
($class:ident, $unique:ident) => {
CUTag::new(TagClass::$class, TagUnique::$unique)
};
($class:ident) => {
CUTag::new(TagClass::$class, TagUnique::Illegal)
};
impl From<UIntSpec> for FullTag {
fn from(value: UIntSpec) -> Self {
value.0
}
}
impl CUTag {
pub const fn new(class: TagClass, unique: TagUnique) -> Self {
Self { class, unique }
#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)]
#[repr(transparent)]
pub struct SIntSpec(FullTag);
impl SIntSpec {
pub const LIM_MIN: [i64; 4] = as_array![i8::MIN, i16::MIN, i32::MIN, i64::MIN];
pub const LIM_MAX: [i64; 4] = as_array![i8::MAX, i16::MAX, i32::MAX, i64::MAX];
pub const DEFAULT: Self = Self::SINT64;
pub const SINT64: Self = Self(FullTag::new_sint(TagSelector::SInt64));
pub const unsafe fn from_full(f: FullTag) -> Self {
Self(f)
}
pub fn check(&self, i: i64) -> bool {
let tag = self.0.tag_selector().value_word() - 5;
(i >= Self::LIM_MIN[tag]) & (i <= Self::LIM_MAX[tag])
}
}
impl DataTag for CUTag {
const BOOL: Self = cutag!(Bool);
const UINT: Self = cutag!(UnsignedInt, UnsignedInt);
const SINT: Self = cutag!(SignedInt, SignedInt);
const FLOAT: Self = cutag!(Float);
const BIN: Self = cutag!(Bin, Bin);
const STR: Self = cutag!(Str, Str);
const LIST: Self = cutag!(List);
fn tag_class(&self) -> TagClass {
self.class
impl From<SIntSpec> for FullTag {
fn from(value: SIntSpec) -> Self {
value.0
}
}
fn tag_selector(&self) -> TagSelector {
unimplemented!()
#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)]
#[repr(transparent)]
pub struct FloatSpec(FullTag);
impl FloatSpec {
pub const LIM_MIN: [f64; 2] = as_array![f32::MIN, f64::MIN];
pub const LIM_MAX: [f64; 2] = as_array![f32::MAX, f64::MAX];
pub const DEFAULT: Self = Self::F64;
pub const F64: Self = Self(FullTag::new_float(TagSelector::Float64));
pub const unsafe fn from_full(f: FullTag) -> Self {
Self(f)
}
fn tag_unique(&self) -> TagUnique {
self.unique
pub fn check(&self, f: f64) -> bool {
let tag = self.0.tag_selector().value_word() - 9;
(f >= Self::LIM_MIN[tag]) & (f <= Self::LIM_MAX[tag])
}
}
impl From<FullTag> for CUTag {
fn from(f: FullTag) -> Self {
Self::new(f.tag_class(), f.tag_unique())
impl From<FloatSpec> for FullTag {
fn from(value: FloatSpec) -> Self {
value.0
}
}

42
server/src/engine/data/tests/md_dict_tests.rs
Unescape Escape View File

@ -36,7 +36,7 @@ fn t_simple_flatten() {
"a_null_key" => Datacell::null(),
};
let expected: DictGeneric = into_dict!(
"a_valid_key" => Datacell::new_uint(100)
"a_valid_key" => Datacell::new_uint_default(100)
);
let ret = dict::rflatten_metadata(generic_dict);
assert_eq!(ret, expected);
@ -45,18 +45,18 @@ fn t_simple_flatten() {
#[test]
fn t_simple_patch() {
let mut current: DictGeneric = into_dict! {
"a" => Datacell::new_uint(2),
"b" => Datacell::new_uint(3),
"z" => Datacell::new_sint(-100),
"a" => Datacell::new_uint_default(2),
"b" => Datacell::new_uint_default(3),
"z" => Datacell::new_sint_default(-100),
};
let new: DictGeneric = into_dict! {
"a" => Datacell::new_uint(1),
"b" => Datacell::new_uint(2),
"a" => Datacell::new_uint_default(1),
"b" => Datacell::new_uint_default(2),
"z" => Datacell::null(),
};
let expected: DictGeneric = into_dict! {
"a" => Datacell::new_uint(1),
"b" => Datacell::new_uint(2),
"a" => Datacell::new_uint_default(1),
"b" => Datacell::new_uint_default(2),
};
assert!(dict::rmerge_metadata(&mut current, new));
assert_eq!(current, expected);
@ -65,14 +65,14 @@ fn t_simple_patch() {
#[test]
fn t_bad_patch() {
let mut current: DictGeneric = into_dict! {
"a" => Datacell::new_uint(2),
"b" => Datacell::new_uint(3),
"z" => Datacell::new_sint(-100),
"a" => Datacell::new_uint_default(2),
"b" => Datacell::new_uint_default(3),
"z" => Datacell::new_sint_default(-100),
};
let backup = current.clone();
let new: DictGeneric = into_dict! {
"a" => Datacell::new_uint(1),
"b" => Datacell::new_uint(2),
"a" => Datacell::new_uint_default(1),
"b" => Datacell::new_uint_default(2),
"z" => Datacell::new_str("omg".into()),
};
assert!(!dict::rmerge_metadata(&mut current, new));
@ -82,20 +82,20 @@ fn t_bad_patch() {
#[test]
fn patch_null_out_dict() {
let mut current: DictGeneric = into_dict! {
"a" => Datacell::new_uint(2),
"b" => Datacell::new_uint(3),
"a" => Datacell::new_uint_default(2),
"b" => Datacell::new_uint_default(3),
"z" => DictEntryGeneric::Map(into_dict!(
"c" => Datacell::new_uint(1),
"d" => Datacell::new_uint(2)
"c" => Datacell::new_uint_default(1),
"d" => Datacell::new_uint_default(2)
)),
};
let expected: DictGeneric = into_dict! {
"a" => Datacell::new_uint(2),
"b" => Datacell::new_uint(3),
"a" => Datacell::new_uint_default(2),
"b" => Datacell::new_uint_default(3),
};
let new: DictGeneric = into_dict! {
"a" => Datacell::new_uint(2),
"b" => Datacell::new_uint(3),
"a" => Datacell::new_uint_default(2),
"b" => Datacell::new_uint_default(3),
"z" => Datacell::null(),
};
assert!(dict::rmerge_metadata(&mut current, new));

4
server/src/engine/macros.rs
Unescape Escape View File

@ -29,6 +29,10 @@ macro_rules! into_array {
($($e:expr),* $(,)?) => { [$($e.into()),*] };
}
macro_rules! as_array {
($($e:expr),* $(,)?) => { [$($e as _),*] };
}
macro_rules! extract {
($src:expr, $what:pat => $ret:expr) => {
if let $what = $src {

2
server/src/engine/ql/dml/ins.rs
Unescape Escape View File

@ -72,7 +72,7 @@ fn pfnbase_uuid() -> Uuid {
// impl
#[inline(always)]
fn pfn_timesec() -> Datacell {
Datacell::new_uint(pfnbase_time().as_secs())
Datacell::new_uint_default(pfnbase_time().as_secs())
}
#[inline(always)]
fn pfn_uuidstr() -> Datacell {

55
server/src/engine/storage/v1/batch_jrnl/persist.rs
Unescape Escape View File

@ -24,6 +24,8 @@
*
*/
use crate::engine::storage::v1::inf::obj::cell;
use {
super::{
MARKER_ACTUAL_BATCH_EVENT, MARKER_BATCH_CLOSED, MARKER_BATCH_REOPEN, MARKER_END_OF_BATCH,
@ -40,14 +42,11 @@ use {
},
data::{
cell::Datacell,
tag::{DataTag, TagClass, TagUnique},
tag::{DataTag, TagUnique},
},
error::{RuntimeResult, StorageError},
idx::STIndexSeq,
storage::v1::{
inf::PersistTypeDscr,
rw::{RawFSInterface, SDSSFileIO, SDSSFileTrackedWriter},
},
storage::v1::rw::{RawFSInterface, SDSSFileIO, SDSSFileTrackedWriter},
},
util::EndianQW,
},
@ -224,49 +223,9 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
}
/// Encode a single cell
fn encode_cell(&mut self, value: &Datacell) -> RuntimeResult<()> {
let ref mut buf = self.f;
buf.unfsynced_write(&[
PersistTypeDscr::translate_from_class(value.tag().tag_class()).value_u8(),
])?;
match value.tag().tag_class() {
TagClass::Bool if value.is_null() => {}
TagClass::Bool => {
let bool = unsafe {
// UNSAFE(@ohsayan): +tagck
value.read_bool()
} as u8;
buf.unfsynced_write(&[bool])?;
}
TagClass::SignedInt | TagClass::UnsignedInt | TagClass::Float => {
let chunk = unsafe {
// UNSAFE(@ohsayan): +tagck
value.read_uint()
}
.to_le_bytes();
buf.unfsynced_write(&chunk)?;
}
TagClass::Str | TagClass::Bin => {
let slice = unsafe {
// UNSAFE(@ohsayan): +tagck
value.read_bin()
};
let slice_l = slice.len().u64_bytes_le();
buf.unfsynced_write(&slice_l)?;
buf.unfsynced_write(slice)?;
}
TagClass::List => {
let list = unsafe {
// UNSAFE(@ohsayan): +tagck
value.read_list()
}
.read();
let list_l = list.len().u64_bytes_le();
buf.unfsynced_write(&list_l)?;
for item in list.iter() {
self.encode_cell(item)?;
}
}
}
let mut buf = vec![];
cell::encode(&mut buf, value);
self.f.unfsynced_write(&buf)?;
Ok(())
}
/// Encode row data

102
server/src/engine/storage/v1/batch_jrnl/restore.rs
Unescape Escape View File

@ -24,6 +24,11 @@
*
*/
use crate::engine::storage::v1::inf::{
obj::cell::{self, StorageCellTypeID},
DataSource,
};
use {
super::{
MARKER_ACTUAL_BATCH_EVENT, MARKER_BATCH_CLOSED, MARKER_BATCH_REOPEN, MARKER_END_OF_BATCH,
@ -34,16 +39,10 @@ use {
index::{DcFieldIndex, PrimaryIndexKey, Row},
model::{delta::DeltaVersion, Model},
},
data::{
cell::Datacell,
tag::{CUTag, TagClass, TagUnique},
},
data::{cell::Datacell, tag::TagUnique},
error::{RuntimeResult, StorageError},
idx::{MTIndex, STIndex, STIndexSeq},
storage::v1::{
inf::PersistTypeDscr,
rw::{RawFSInterface, SDSSFileIO, SDSSFileTrackedReader},
},
storage::v1::rw::{RawFSInterface, SDSSFileIO, SDSSFileTrackedReader},
},
crossbeam_epoch::pin,
std::{
@ -500,58 +499,43 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
})
}
fn decode_cell(&mut self) -> RuntimeResult<Datacell> {
let cell_type_sig = self.f.read_byte()?;
let Some(cell_type) = PersistTypeDscr::try_from_raw(cell_type_sig) else {
let Some(dscr) = StorageCellTypeID::try_from_raw(self.f.read_byte()?) else {
return Err(StorageError::DataBatchRestoreCorruptedEntry.into());
};
Ok(match cell_type {
PersistTypeDscr::Null => Datacell::null(),
PersistTypeDscr::Bool => {
let bool = self.f.read_byte()?;
if bool > 1 {
return Err(StorageError::DataBatchRestoreCorruptedEntry.into());
}
Datacell::new_bool(bool == 1)
}
PersistTypeDscr::UnsignedInt | PersistTypeDscr::SignedInt | PersistTypeDscr::Float => {
let qw = self.f.read_u64_le()?;
unsafe {
// UNSAFE(@ohsayan): choosing the correct type and tag
let tc = TagClass::from_raw(cell_type.value_u8() - 1);
Datacell::new_qw(qw, CUTag::new(tc, tc.tag_unique()))
}
}
PersistTypeDscr::Str | PersistTypeDscr::Bin => {
let len = self.f.read_u64_le()? as usize;
let mut data = vec![0; len];
self.f.read_into_buffer(&mut data)?;
unsafe {
// UNSAFE(@ohsayan): +tagck
if cell_type == PersistTypeDscr::Str {
if core::str::from_utf8(&data).is_err() {
return Err(StorageError::DataBatchRestoreCorruptedEntry.into());
}
Datacell::new_str(String::from_utf8_unchecked(data).into_boxed_str())
} else {
Datacell::new_bin(data.into_boxed_slice())
}
}
}
PersistTypeDscr::List => {
let len = self.f.read_u64_le()?;
let mut list = Vec::new();
while !self.f.is_eof() && list.len() as u64 != len {
list.push(self.decode_cell()?);
}
if len != list.len() as u64 {
return Err(StorageError::DataBatchRestoreCorruptedEntry.into());
}
Datacell::new_list(list)
}
PersistTypeDscr::Dict => {
// we don't support dicts just yet
return Err(StorageError::DataBatchRestoreCorruptedEntry.into());
}
})
unsafe { cell::decode_element::<Datacell, SDSSFileTrackedReader<F>>(&mut self.f, dscr) }
.map_err(|e| e.0)
}
}
pub struct ErrorHack(crate::engine::fractal::error::Error);
impl From<crate::engine::fractal::error::Error> for ErrorHack {
fn from(value: crate::engine::fractal::error::Error) -> Self {
Self(value)
}
}
impl From<()> for ErrorHack {
fn from(_: ()) -> Self {
Self(StorageError::DataBatchRestoreCorruptedEntry.into())
}
}
impl<F: RawFSInterface> DataSource for SDSSFileTrackedReader<F> {
const RELIABLE_SOURCE: bool = false;
type Error = ErrorHack;
fn has_remaining(&self, cnt: usize) -> bool {
self.remaining() >= cnt as u64
}
unsafe fn read_next_byte(&mut self) -> Result<u8, Self::Error> {
Ok(self.read_byte()?)
}
unsafe fn read_next_block<const N: usize>(&mut self) -> Result<[u8; N], Self::Error> {
Ok(self.read_block()?)
}
unsafe fn read_next_u64_le(&mut self) -> Result<u64, Self::Error> {
Ok(self.read_u64_le()?)
}
unsafe fn read_next_variable_block(&mut self, size: usize) -> Result<Vec<u8>, Self::Error> {
let mut buf = vec![0; size];
self.read_into_buffer(&mut buf)?;
Ok(buf)
}
}

118
server/src/engine/storage/v1/inf/map.rs
Unescape Escape View File

@ -26,18 +26,16 @@
use {
super::{
obj::{self, FieldMD},
PersistMapSpec, PersistObject, PersistTypeDscr, VecU8,
obj::{
cell::{self, CanYieldDict, StorageCellTypeID},
FieldMD,
},
PersistMapSpec, PersistObject, VecU8,
},
crate::{
engine::{
core::model::Field,
data::{
cell::Datacell,
dict::DictEntryGeneric,
tag::{CUTag, TagUnique},
DictGeneric,
},
data::{dict::DictEntryGeneric, DictGeneric},
error::{RuntimeResult, StorageError},
idx::{IndexBaseSpec, IndexSTSeqCns, STIndex, STIndexSeq},
mem::BufferedScanner,
@ -175,9 +173,8 @@ impl PersistMapSpec for GenericDictSpec {
scanner.has_left(9)
}
fn pretest_entry_data(scanner: &BufferedScanner, md: &Self::EntryMD) -> bool {
static EXPECT_ATLEAST: [u8; 4] = [0, 1, 8, 8]; // PAD to align
let lbound_rem = md.klen + EXPECT_ATLEAST[md.dscr.min(3) as usize] as usize;
scanner.has_left(lbound_rem) & (md.dscr <= PersistTypeDscr::Dict.value_u8())
StorageCellTypeID::is_valid(md.dscr)
& scanner.has_left(StorageCellTypeID::expect_atleast(md.dscr))
}
fn entry_md_enc(buf: &mut VecU8, key: &Self::Key, _: &Self::Value) {
buf.extend(key.len().u64_bytes_le());
@ -188,14 +185,14 @@ impl PersistMapSpec for GenericDictSpec {
fn enc_entry(buf: &mut VecU8, key: &Self::Key, val: &Self::Value) {
match val {
DictEntryGeneric::Map(map) => {
buf.push(PersistTypeDscr::Dict.value_u8());
buf.push(StorageCellTypeID::Dict.value_u8());
buf.extend(key.as_bytes());
<PersistMapImpl<Self> as PersistObject>::default_full_enc(buf, map);
}
DictEntryGeneric::Data(dc) => {
obj::encode_datacell_tag(buf, dc);
buf.push(cell::encode_tag(dc));
buf.extend(key.as_bytes());
obj::encode_element(buf, dc);
cell::encode_cell(buf, dc);
}
}
}
@ -205,85 +202,20 @@ impl PersistMapSpec for GenericDictSpec {
.ok()
}
unsafe fn dec_val(scanner: &mut BufferedScanner, md: &Self::EntryMD) -> Option<Self::Value> {
unsafe fn decode_element(
scanner: &mut BufferedScanner,
dscr: PersistTypeDscr,
dg_top_element: bool,
) -> Option<DictEntryGeneric> {
let r = match dscr {
PersistTypeDscr::Null => DictEntryGeneric::Data(Datacell::null()),
PersistTypeDscr::Bool => {
DictEntryGeneric::Data(Datacell::new_bool(scanner.next_byte() == 1))
}
PersistTypeDscr::UnsignedInt
| PersistTypeDscr::SignedInt
| PersistTypeDscr::Float => DictEntryGeneric::Data(Datacell::new_qw(
scanner.next_u64_le(),
CUTag::new(
dscr.into_class(),
[
TagUnique::UnsignedInt,
TagUnique::SignedInt,
TagUnique::Illegal,
TagUnique::Illegal, // pad
][(dscr.value_u8() - 2) as usize],
),
)),
PersistTypeDscr::Str | PersistTypeDscr::Bin => {
let slc_len = scanner.next_u64_le() as usize;
if !scanner.has_left(slc_len) {
return None;
}
let slc = scanner.next_chunk_variable(slc_len);
DictEntryGeneric::Data(if dscr == PersistTypeDscr::Str {
if core::str::from_utf8(slc).is_err() {
return None;
}
Datacell::new_str(
String::from_utf8_unchecked(slc.to_owned()).into_boxed_str(),
)
} else {
Datacell::new_bin(slc.to_owned().into_boxed_slice())
})
}
PersistTypeDscr::List => {
let list_len = scanner.next_u64_le() as usize;
let mut v = Vec::with_capacity(list_len);
while (!scanner.eof()) & (v.len() < list_len) {
let dscr = scanner.next_byte();
if dscr > PersistTypeDscr::Dict.value_u8() {
return None;
}
v.push(
match decode_element(scanner, PersistTypeDscr::from_raw(dscr), false) {
Some(DictEntryGeneric::Data(l)) => l,
None => return None,
_ => unreachable!("found top-level dict item in datacell"),
},
);
}
if v.len() == list_len {
DictEntryGeneric::Data(Datacell::new_list(v))
} else {
return None;
}
}
PersistTypeDscr::Dict => {
if dg_top_element {
DictEntryGeneric::Map(
<PersistMapImpl<GenericDictSpec> as PersistObject>::default_full_dec(
scanner,
)
.ok()?,
)
} else {
unreachable!("found top-level dict item in datacell")
}
}
};
Some(r)
}
decode_element(scanner, PersistTypeDscr::from_raw(md.dscr), true)
Some(
match cell::decode_element::<CanYieldDict, BufferedScanner>(
scanner,
StorageCellTypeID::from_raw(md.dscr),
)
.ok()?
{
CanYieldDict::Data(d) => DictEntryGeneric::Data(d),
CanYieldDict::Dict => DictEntryGeneric::Map(
<PersistMapImpl<GenericDictSpec> as PersistObject>::default_full_dec(scanner)
.ok()?,
),
},
)
}
// not implemented
fn enc_key(_: &mut VecU8, _: &Self::Key) {

79
server/src/engine/storage/v1/inf/mod.rs
Unescape Escape View File

@ -26,77 +26,46 @@
//! High level interfaces
use crate::engine::idx::STIndex;
pub mod map;
pub mod obj;
// tests
#[cfg(test)]
mod tests;
use {
crate::engine::{
data::tag::TagClass,
error::{RuntimeResult, StorageError},
idx::{AsKey, AsValue},
mem::BufferedScanner,
},
std::mem,
use crate::engine::{
error::{RuntimeResult, StorageError},
idx::{AsKey, AsValue, STIndex},
mem::BufferedScanner,
};
type VecU8 = Vec<u8>;
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, sky_macros::EnumMethods)]
#[repr(u8)]
#[allow(unused)]
/// Disambiguation for data
pub enum PersistTypeDscr {
Null = 0,
Bool = 1,
UnsignedInt = 2,
SignedInt = 3,
Float = 4,
Bin = 5,
Str = 6,
List = 7,
Dict = 8,
pub trait DataSource {
type Error;
const RELIABLE_SOURCE: bool = true;
fn has_remaining(&self, cnt: usize) -> bool;
unsafe fn read_next_byte(&mut self) -> Result<u8, Self::Error>;
unsafe fn read_next_block<const N: usize>(&mut self) -> Result<[u8; N], Self::Error>;
unsafe fn read_next_u64_le(&mut self) -> Result<u64, Self::Error>;
unsafe fn read_next_variable_block(&mut self, size: usize) -> Result<Vec<u8>, Self::Error>;
}
#[allow(unused)]
impl PersistTypeDscr {
/// translates the tag class definition into the dscr definition
pub const fn translate_from_class(class: TagClass) -> Self {
unsafe { Self::from_raw(class.value_u8() + 1) }
}
pub const fn try_from_raw(v: u8) -> Option<Self> {
if v > Self::MAX {
None
} else {
unsafe { Some(Self::from_raw(v)) }
}
}
pub const unsafe fn from_raw(v: u8) -> Self {
core::mem::transmute(v)
}
/// The data in question is null (well, can we call that data afterall?)
pub const fn is_null(&self) -> bool {
self.value_u8() == Self::Null.value_u8()
impl<'a> DataSource for BufferedScanner<'a> {
type Error = ();
fn has_remaining(&self, cnt: usize) -> bool {
self.has_left(cnt)
}
/// The data in question is a scalar
pub const fn is_scalar(&self) -> bool {
self.value_u8() <= Self::Float.value_u8()
unsafe fn read_next_byte(&mut self) -> Result<u8, Self::Error> {
Ok(self.next_byte())
}
/// The data is composite
pub const fn is_composite(&self) -> bool {
self.value_u8() > Self::Float.value_u8()
unsafe fn read_next_block<const N: usize>(&mut self) -> Result<[u8; N], Self::Error> {
Ok(self.next_chunk())
}
/// Recursive data
pub const fn is_recursive(&self) -> bool {
self.value_u8() >= Self::List.value_u8()
unsafe fn read_next_u64_le(&mut self) -> Result<u64, Self::Error> {
Ok(self.next_u64_le())
}
fn into_class(&self) -> TagClass {
debug_assert!(*self != Self::Null);
unsafe { mem::transmute(self.value_u8() - 1) }
unsafe fn read_next_variable_block(&mut self, size: usize) -> Result<Vec<u8>, Self::Error> {
Ok(self.next_chunk_variable(size).into())
}
}

217
server/src/engine/storage/v1/inf/obj.rs
Unescape Escape View File

@ -25,7 +25,7 @@
*/
use {
super::{PersistObject, PersistTypeDscr, VecU8},
super::{PersistObject, VecU8},
crate::{
engine::{
core::{
@ -33,7 +33,6 @@ use {
space::{Space, SpaceMeta},
},
data::{
cell::Datacell,
tag::{DataTag, TagClass, TagSelector},
uuid::Uuid,
DictGeneric,
@ -46,34 +45,202 @@ use {
},
};
pub fn encode_element(buf: &mut VecU8, dc: &Datacell) {
unsafe {
use TagClass::*;
match dc.tag().tag_class() {
Bool if dc.is_init() => buf.push(dc.read_bool() as u8),
Bool => {}
UnsignedInt | SignedInt | Float => buf.extend(dc.read_uint().to_le_bytes()),
Str | Bin => {
let slc = dc.read_bin();
buf.extend(slc.len().u64_bytes_le());
buf.extend(slc);
/*
generic cells
*/
pub mod cell {
use crate::{
engine::{
data::{
cell::Datacell,
tag::{DataTag, TagClass, TagSelector},
},
storage::v1::inf::{DataSource, VecU8},
},
util::EndianQW,
};
#[derive(Debug, PartialEq, Eq, Clone, Copy, PartialOrd, Ord, Hash, sky_macros::EnumMethods)]
#[repr(u8)]
#[allow(dead_code)]
pub enum StorageCellTypeID {
Null = 0x00,
Bool = 0x01,
UInt8 = 0x02,
UInt16 = 0x03,
UInt32 = 0x04,
UInt64 = 0x05,
SInt8 = 0x06,
SInt16 = 0x07,
SInt32 = 0x08,
SInt64 = 0x09,
Float32 = 0x0A,
Float64 = 0x0B,
Bin = 0x0C,
Str = 0x0D,
List = 0x0E,
Dict = 0x0F,
}
impl StorageCellTypeID {
pub const unsafe fn from_raw(v: u8) -> Self {
core::mem::transmute(v)
}
pub const fn try_from_raw(v: u8) -> Option<Self> {
if Self::is_valid(v) {
Some(unsafe { Self::from_raw(v) })
} else {
None
}
List => {
let lst = dc.read_list().read();
buf.extend(lst.len().u64_bytes_le());
for item in lst.iter() {
encode_element(buf, item);
}
#[inline(always)]
pub const fn is_valid(d: u8) -> bool {
d <= Self::MAX
}
const unsafe fn into_selector(self) -> TagSelector {
debug_assert!(self.value_u8() != Self::Null.value_u8());
core::mem::transmute(self.value_u8() - 1)
}
#[inline(always)]
pub fn expect_atleast(d: u8) -> usize {
[0u8, 1, 8, 8][d.min(3) as usize] as usize
}
}
pub fn encode(buf: &mut VecU8, dc: &Datacell) {
buf.push(encode_tag(dc));
encode_cell(buf, dc)
}
pub fn encode_tag(dc: &Datacell) -> u8 {
(dc.tag().tag_selector().value_u8() + 1) * (dc.is_init() as u8)
}
pub fn encode_cell(buf: &mut VecU8, dc: &Datacell) {
if dc.is_null() {
return;
}
unsafe {
use TagClass::*;
match dc.tag().tag_class() {
Bool if dc.is_init() => buf.push(dc.read_bool() as u8),
Bool => {}
UnsignedInt | SignedInt | Float => buf.extend(dc.read_uint().to_le_bytes()),
Str | Bin => {
let slc = dc.read_bin();
buf.extend(slc.len().u64_bytes_le());
buf.extend(slc);
}
List => {
let lst = dc.read_list().read();
buf.extend(lst.len().u64_bytes_le());
for item in lst.iter() {
encode(buf, item);
}
}
}
}
}
}
pub fn encode_datacell_tag(buf: &mut VecU8, dc: &Datacell) {
buf.push(
PersistTypeDscr::translate_from_class(dc.tag().tag_class()).value_u8()
* (!dc.is_null() as u8),
)
pub trait ElementYield {
type Yield;
type Error;
const CAN_YIELD_DICT: bool = false;
fn yield_data(dc: Datacell) -> Result<Self::Yield, Self::Error>;
fn yield_dict() -> Result<Self::Yield, Self::Error> {
panic!()
}
fn error() -> Result<Self::Yield, Self::Error>;
}
impl ElementYield for Datacell {
type Yield = Self;
type Error = ();
fn yield_data(dc: Datacell) -> Result<Self::Yield, Self::Error> {
Ok(dc)
}
fn error() -> Result<Self::Yield, Self::Error> {
Err(())
}
}
#[derive(Debug, PartialEq)]
pub enum CanYieldDict {
Data(Datacell),
Dict,
}
impl ElementYield for CanYieldDict {
type Yield = Self;
type Error = ();
const CAN_YIELD_DICT: bool = true;
fn error() -> Result<Self::Yield, Self::Error> {
Err(())
}
fn yield_data(dc: Datacell) -> Result<Self::Yield, Self::Error> {
Ok(CanYieldDict::Data(dc))
}
fn yield_dict() -> Result<Self::Yield, Self::Error> {
Ok(CanYieldDict::Dict)
}
}
pub unsafe fn decode_element<EY: ElementYield, DS: DataSource>(
s: &mut DS,
dscr: StorageCellTypeID,
) -> Result<EY::Yield, DS::Error>
where
DS::Error: From<EY::Error>,
DS::Error: From<()>,
{
if dscr == StorageCellTypeID::Dict {
if EY::CAN_YIELD_DICT {
return Ok(EY::yield_dict()?);
} else {
return Ok(EY::error()?);
}
}
if dscr == StorageCellTypeID::Null {
return Ok(EY::yield_data(Datacell::null())?);
}
let tag = dscr.into_selector().into_full();
let d = match tag.tag_class() {
TagClass::Bool => {
let nx = s.read_next_byte()?;
if nx > 1 {
return Ok(EY::error()?);
}
Datacell::new_bool(nx == 1)
}
TagClass::UnsignedInt | TagClass::SignedInt | TagClass::Float => {
let nx = s.read_next_u64_le()?;
Datacell::new_qw(nx, tag)
}
TagClass::Bin | TagClass::Str => {
let len = s.read_next_u64_le()? as usize;
let block = s.read_next_variable_block(len)?;
if tag.tag_class() == TagClass::Str {
match String::from_utf8(block).map(|s| Datacell::new_str(s.into_boxed_str())) {
Ok(s) => s,
Err(_) => return Ok(EY::error()?),
}
} else {
Datacell::new_bin(block.into())
}
}
TagClass::List => {
let len = s.read_next_u64_le()? as usize;
let mut l = vec![];
while (l.len() != len) & s.has_remaining(1) {
let Some(dscr) = StorageCellTypeID::try_from_raw(s.read_next_byte()?) else {
return Ok(EY::error()?);
};
// FIXME(@ohsayan): right now, a list cannot contain a dict!
if !s.has_remaining(StorageCellTypeID::expect_atleast(dscr.value_u8())) {
return Ok(EY::error()?);
}
l.push(self::decode_element::<Datacell, DS>(s, dscr)?);
}
if l.len() != len {
return Ok(EY::error()?);
}
Datacell::new_list(l)
}
};
Ok(EY::yield_data(d)?)
}
}
/*

2
server/src/engine/storage/v1/inf/tests.rs
Unescape Escape View File

@ -47,7 +47,7 @@ fn dict() {
"hello" => Datacell::new_str("world".into()),
"omg a null?" => Datacell::null(),
"a big fat dict" => DictEntryGeneric::Map(into_dict!(
"with a value" => Datacell::new_uint(1002),
"with a value" => Datacell::new_uint_default(1002),
"and a null" => Datacell::null(),
))
};

4
server/src/engine/storage/v1/sysdb.rs
Unescape Escape View File

@ -149,8 +149,8 @@ pub fn sync_system_database_to<Fs: RawFSInterface>(
// prepare our flat file
let mut map: DictGeneric = into_dict!(
SYS_KEY_SYS => DictEntryGeneric::Map(into_dict!(
SYS_KEY_SYS_SETTINGS_VERSION => Datacell::new_uint(cfg.host_data().settings_version() as _),
SYS_KEY_SYS_STARTUP_COUNTER => Datacell::new_uint(cfg.host_data().startup_counter() as _),
SYS_KEY_SYS_SETTINGS_VERSION => Datacell::new_uint_default(cfg.host_data().settings_version() as _),
SYS_KEY_SYS_STARTUP_COUNTER => Datacell::new_uint_default(cfg.host_data().startup_counter() as _),
)),
SYS_KEY_AUTH => DictGeneric::new(),
);

4
server/src/engine/txn/gns/tests/full_chain.rs
Unescape Escape View File

@ -80,7 +80,7 @@ fn create_space() {
global.namespace().spaces().read().get("myspace").unwrap(),
&Space::new_restore_empty(
SpaceMeta::with_env(
into_dict!("SAYAN_MAX" => DictEntryGeneric::Data(Datacell::new_uint(65536)))
into_dict!("SAYAN_MAX" => DictEntryGeneric::Data(Datacell::new_uint_default(65536)))
),
uuid
)
@ -108,7 +108,7 @@ fn alter_space() {
global.namespace().spaces().read().get("myspace").unwrap(),
&Space::new_restore_empty(
SpaceMeta::with_env(
into_dict!("SAYAN_MAX" => DictEntryGeneric::Data(Datacell::new_uint(65536)))
into_dict!("SAYAN_MAX" => DictEntryGeneric::Data(Datacell::new_uint_default(65536)))
),
uuid
)

Write Preview
Loading…
Cancel
Save